From f1d694b47b6169af290d158e9b25d709e2b0cd49 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 20 Oct 2026 18:58:42 -0500 Subject: [PATCH 001/197] Move the emitter's loop optimization state into loop-state.ts LlEmitter's field list and constructor are where most concurrent edits to emitter.ts collide. A feature now defines its state in its own file with defineModuleState (emitter-state.ts) and reads it through the returned key, so it no longer edits the class. The loop optimizations' twelve fields and their per-function setup move to loop-state.ts; the emitted IR is unchanged. scripts/split holds the transforms used for the emitter and lowering splits, and rehome-split.mjs, which carries an open branch across a split. --- .../src/backend/llvm/emit-function.ts | 24 +- .../src/backend/llvm/emit-statements.ts | 55 +- .../src/backend/llvm/emitter-state.ts | 24 + packages/compiler/src/backend/llvm/emitter.ts | 34 +- .../compiler/src/backend/llvm/loop-state.ts | 76 +++ scripts/split/README.md | 22 + .../configs/loop-state/emitter-state.template | 24 + .../configs/loop-state/loop-state.config.json | 29 + .../configs/loop-state/loop-state.template | 34 ++ scripts/split/move-state.mjs | 202 +++++++ scripts/split/rehome-split.mjs | 311 ++++++++++ scripts/split/split-methods.mjs | 414 ++++++++++++++ scripts/split/split-module.mjs | 539 ++++++++++++++++++ 13 files changed, 1714 insertions(+), 74 deletions(-) create mode 100644 packages/compiler/src/backend/llvm/emitter-state.ts create mode 100644 packages/compiler/src/backend/llvm/loop-state.ts create mode 100644 scripts/split/README.md create mode 100644 scripts/split/configs/loop-state/emitter-state.template create mode 100644 scripts/split/configs/loop-state/loop-state.config.json create mode 100644 scripts/split/configs/loop-state/loop-state.template create mode 100644 scripts/split/move-state.mjs create mode 100644 scripts/split/rehome-split.mjs create mode 100644 scripts/split/split-methods.mjs create mode 100644 scripts/split/split-module.mjs diff --git a/packages/compiler/src/backend/llvm/emit-function.ts b/packages/compiler/src/backend/llvm/emit-function.ts index 9e50bea6d..ace696061 100644 --- a/packages/compiler/src/backend/llvm/emit-function.ts +++ b/packages/compiler/src/backend/llvm/emit-function.ts @@ -16,9 +16,7 @@ import { withInitializerBindings, withInitializerLocals, } from "../../ir/initializer-bindings.js"; -import { findUnaliasedLocalArrays } from "../../ir/array-length-stability.js"; import { constructorClass } from "../../ir/final-fields.js"; -import { findInitializedByteLoopBindings } from "./byte-windows.js"; import { findLoopArrayBorrows } from "./loop-array-borrows.js"; import { findCallArrayReads, findLocalArrayReads } from "./local-array-reads.js"; import { findScalarStringSlices, findScratchStringSlices } from "./string-slices.js"; @@ -54,6 +52,7 @@ import { INT32_ABI_SUFFIX, SCALAR_UNION_RESULT } from "./int32-abi.js"; import { f64Lit } from "./common.js"; import { RESULT_ENTRY_PREFIX } from "./record-results.js"; import { emitIntegerClone, emitIntegerDispatcher } from "./integer-clones.js"; +import { beginFunctionLoops, LOOPS } from "./loop-state.js"; export function emitFunction(emitter: LlEmitter, fn: IrFunction): string { const B = new BlockBuilder(); @@ -95,7 +94,6 @@ export function emitFunction(emitter: LlEmitter, fn: IrFunction): string { const initializerBindings = emitter.initializerBindings.get(fn.name) ?? []; const numericFn = withInitializerBindings(fn, initializerBindings); emitter.numericLocals = new Map(numericFn.locals.map((l) => [l.id, l])); - emitter.unaliasedArrays = findUnaliasedLocalArrays(fn); emitter.borrowedParameters.clear(); emitter.referenceWrites?.enter(fn); emitter.reboundOwnerSlots.clear(); @@ -126,8 +124,8 @@ export function emitFunction(emitter: LlEmitter, fn: IrFunction): string { emitter.stableCaptureIds = fn.captures !== undefined ? emitter.lazyCaptures.stableCaptures(fn.name) : new Set(); emitter.integerLoopBindings.clear(); - emitter.countedLoopsEnabled = emitter.debug === null && !fn.async && !fn.generator; - emitter.speculativeLoopsEnabled = emitter.countedLoopsEnabled; + beginFunctionLoops(emitter, fn, numericFn); + emitter.speculativeLoopsEnabled = LOOPS.of(emitter).countedLoopsEnabled; emitter.speculativeFunction = fn.name; emitter.speculation = null; emitter.speculationResume = null; @@ -136,20 +134,6 @@ export function emitFunction(emitter: LlEmitter, fn: IrFunction): string { !fn.generator && constructorClass(fn.name) === null && emitter.finalFields.size > 0; - emitter.byteWindowsEnabled = emitter.countedLoopsEnabled; - emitter.bytesLoopVersionsEnabled = emitter.countedLoopsEnabled; - emitter.bytesLoopEntries = null; - emitter.bytesLoopFunction = fn; - emitter.denseLoopsEnabled = emitter.countedLoopsEnabled; - emitter.denseLoop = null; - emitter.denseLoopEntries = - emitter.denseLoopsEnabled && numericFn.locals.some((l) => l.type.kind === "array") - ? findInitializedByteLoopBindings(fn) - : new Map(); - emitter.byteWindowEntries = - emitter.byteWindowsEnabled && numericFn.locals.some((l) => l.type.kind === "bytes") - ? findInitializedByteLoopBindings(fn) - : new Map(); emitter.integerViews.clear(); emitter.fieldPointerTags.clear(); emitter.integerArrayBindings.clear(); @@ -403,7 +387,7 @@ export function emitFunction(emitter: LlEmitter, fn: IrFunction): string { if (local.boxed || emitter.captureIds.has(local.id)) slots.add(`%${mangleLocal(local.id)}`); B.volatileSlots = slots; // Without optimization a versioned copy only adds code. - emitter.bytesLoopVersionsEnabled = false; + LOOPS.of(emitter).bytesLoopVersionsEnabled = false; } // Captured bindings come in through the environment — borrowed for the // whole call (the closure owns them): bound here, never released here. diff --git a/packages/compiler/src/backend/llvm/emit-statements.ts b/packages/compiler/src/backend/llvm/emit-statements.ts index da2ffbcab..e00abb8d1 100644 --- a/packages/compiler/src/backend/llvm/emit-statements.ts +++ b/packages/compiler/src/backend/llvm/emit-statements.ts @@ -67,6 +67,7 @@ import { stringParts, } from "./string-construction.js"; import { emitStringInputs } from "./string-lifetimes.js"; +import { LOOPS } from "./loop-state.js"; export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { const B = emitter.B; @@ -855,26 +856,26 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { integerArrayLoop ?? (emitter.debug === null ? matchIntegerBytesForLoop(s, emitter.numericLocals) : null); const countedLoop = - !integerLoop && emitter.countedLoopsEnabled + !integerLoop && LOOPS.of(emitter).countedLoopsEnabled ? matchIntegerCountedForLoop( s, emitter.numericLocals, emitter.integerRanges, - emitter.unaliasedArrays, + LOOPS.of(emitter).unaliasedArrays, ) : null; const window = - countedLoop && emitter.byteWindowsEnabled + countedLoop && LOOPS.of(emitter).byteWindowsEnabled ? matchByteWindow( s, countedLoop, emitter.numericLocals, emitter.captureIds, emitter.integerRanges, - emitter.byteWindowEntries.get(s) ?? new Set(), + LOOPS.of(emitter).byteWindowEntries.get(s) ?? new Set(), ) : null; - const denseShape = !emitter.denseLoopsEnabled + const denseShape = !LOOPS.of(emitter).denseLoopsEnabled ? null : integerArrayLoop ? { @@ -891,8 +892,8 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { s, denseShape, emitter.numericLocals, - emitter.unaliasedArrays, - emitter.denseLoopEntries.get(s) ?? new Set(), + LOOPS.of(emitter).unaliasedArrays, + LOOPS.of(emitter).denseLoopEntries.get(s) ?? new Set(), ) : null; if (dense) { @@ -901,18 +902,18 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { slow = B.newLabel("dense.loop.slow"), done = B.newLabel("dense.loop.done"); B.condBr(valid, fast, slow); - emitter.denseLoopsEnabled = false; + LOOPS.of(emitter).denseLoopsEnabled = false; try { B.startBlock(slow); emitter.emitStmt(s); B.br(done); B.startBlock(fast); - emitter.denseLoop = dense; + LOOPS.of(emitter).denseLoop = dense; emitter.emitStmt(s); B.br(done); } finally { - emitter.denseLoop = null; - emitter.denseLoopsEnabled = true; + LOOPS.of(emitter).denseLoop = null; + LOOPS.of(emitter).denseLoopsEnabled = true; } B.startBlock(done); break; @@ -925,7 +926,7 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { B.condBr(valid, fast, slow); const originalRanges = emitter.integerRanges, originalBounds = emitter.bytesBounds; - emitter.byteWindowsEnabled = false; + LOOPS.of(emitter).byteWindowsEnabled = false; try { B.startBlock(slow); emitter.emitStmt(s); @@ -938,15 +939,15 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { } finally { emitter.integerRanges = originalRanges; emitter.bytesBounds = originalBounds; - emitter.byteWindowsEnabled = true; + LOOPS.of(emitter).byteWindowsEnabled = true; } B.startBlock(done); break; } const version = - emitter.bytesLoopVersionsEnabled && - emitter.bytesLoopVersioning.size < MAX_BYTES_LOOP_VERSION_DEPTH && - !emitter.bytesLoopVersioning.has(s) + LOOPS.of(emitter).bytesLoopVersionsEnabled && + LOOPS.of(emitter).bytesLoopVersioning.size < MAX_BYTES_LOOP_VERSION_DEPTH && + !LOOPS.of(emitter).bytesLoopVersioning.has(s) ? emitter.matchBytesLoopVersion(s, countedLoop) : null; if (version) { @@ -956,7 +957,7 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { emitBytesLoopVersionGuard(emitter, version, fast, slow); const originalBounds = emitter.bytesBounds, originalRanges = emitter.integerRanges; - emitter.bytesLoopVersioning.add(s); + LOOPS.of(emitter).bytesLoopVersioning.add(s); try { B.startBlock(slow); emitter.emitStmt(s); @@ -965,29 +966,29 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { emitter.bytesBounds = new Set([...originalBounds, ...version.bounds]); if (version.ranges.size > 0) emitter.integerRanges = new Map([...originalRanges, ...version.ranges]); - if (version.counted?.guarded) emitter.provenCountedGuards.add(s); + if (version.counted?.guarded) LOOPS.of(emitter).provenCountedGuards.add(s); emitter.emitStmt(s); B.br(done); } finally { emitter.bytesBounds = originalBounds; emitter.integerRanges = originalRanges; - emitter.provenCountedGuards.delete(s); - emitter.bytesLoopVersioning.delete(s); + LOOPS.of(emitter).provenCountedGuards.delete(s); + LOOPS.of(emitter).bytesLoopVersioning.delete(s); } B.startBlock(done); break; } let countedJoin: string | null = null; - if (countedLoop?.guarded && !emitter.provenCountedGuards.has(s)) { + if (countedLoop?.guarded && !LOOPS.of(emitter).provenCountedGuards.has(s)) { const safe = emitCountedLoopSafety(emitter, countedLoop); const fast = B.newLabel("counted.fast"); const slow = B.newLabel("counted.slow"); countedJoin = B.newLabel("counted.done"); B.condBr(safe, fast, slow); B.startBlock(slow); - emitter.countedLoopsEnabled = false; + LOOPS.of(emitter).countedLoopsEnabled = false; emitter.emitStmt(s); - emitter.countedLoopsEnabled = true; + LOOPS.of(emitter).countedLoopsEnabled = true; B.br(countedJoin); B.startBlock(fast); } @@ -1015,8 +1016,8 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { slot: integerSlot, type: emitter.sizeType, range: - emitter.denseLoop?.counter === integerLoop.localId - ? emitter.denseLoop.range + LOOPS.of(emitter).denseLoop?.counter === integerLoop.localId + ? LOOPS.of(emitter).denseLoop!.range : { min: 0, max: integerArrayLoop ? 4294967294 : Number.MAX_SAFE_INTEGER - 1 }, }); } else if (countedLoop) { @@ -1033,8 +1034,8 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { type: "i64", signed: true, range: - emitter.denseLoop?.counter === countedLoop.localId - ? emitter.denseLoop.range + LOOPS.of(emitter).denseLoop?.counter === countedLoop.localId + ? LOOPS.of(emitter).denseLoop!.range : countedLoop.range, }); countedLimit = emitCountedLoopLimit(emitter, countedLoop); diff --git a/packages/compiler/src/backend/llvm/emitter-state.ts b/packages/compiler/src/backend/llvm/emitter-state.ts new file mode 100644 index 000000000..62dc47b4e --- /dev/null +++ b/packages/compiler/src/backend/llvm/emitter-state.ts @@ -0,0 +1,24 @@ +/* Per-feature emitter state. A feature defines its state in its own file + * with defineModuleState and reads it through the returned key, so adding + * or extending a feature does not edit LlEmitter's field list or + * constructor. */ +import type { LlEmitter } from "./emitter.js"; + +/** A feature's state on an emitter, read with `key.of(emitter)`. */ +export interface EmitterStateKey { + of(emitter: LlEmitter): T; +} + +let moduleSlots = 0; + +/** State that lives for the whole module, created on first read. */ +export function defineModuleState(create: (emitter: LlEmitter) => T): EmitterStateKey { + const slot = moduleSlots++; + return { + of(emitter) { + const states = emitter.moduleStates; + if (slot >= states.length || states[slot] === undefined) states[slot] = create(emitter); + return states[slot] as T; + }, + }; +} diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 8a4b16fc5..9400f7aa3 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -59,7 +59,6 @@ import { moduleUsesMockTimers, } from "../../ir/runtime-features.js"; import { captureFreeFunctions, observesStacks } from "../stack-observations.js"; -import { type DenseArrayLoop } from "./dense-array-loops.js"; import { findInitializedByteLoopBindings } from "./byte-windows.js"; import { matchBytesLoopVersion } from "./bytes-loop-versions.js"; import { globalSlotAlias } from "./bytes-alias.js"; @@ -179,6 +178,7 @@ import { PublishWalkers } from "./publish.js"; import { computeImmutableFields, stableGlobals, type ImmutableFields } from "../immutability.js"; import { computePublishedTypes, type PublishedTypes } from "../publication.js"; import { emitExpr } from "./expr-dispatch.js"; +import { LOOPS } from "./loop-state.js"; import { dynKind, @@ -570,6 +570,8 @@ function mentionsLocal(e: IrExpr, localId: string): boolean { } export class LlEmitter { + /** Per-feature state (emitter-state.ts), indexed by each key's slot. */ + readonly moduleStates: unknown[] = []; localArrayReads = new Map(); loopArrayBorrows: ReadonlySet = new Set(); scalarStringSlices = new Map(); @@ -626,7 +628,7 @@ export class LlEmitter { * prefix: the counter of a dense array loop's fast copy indexing one of * the arrays its entry test covered. */ provenDenseIndex(arr: IrExpr, index: IrExpr): boolean { - const loop = this.denseLoop; + const loop = LOOPS.of(this).denseLoop; return ( loop !== null && arr.kind === "varRef" && @@ -1000,21 +1002,12 @@ export class LlEmitter { * immortal from construction (classes.ts programLifetimeClass). */ readonly programLifetime: ReadonlySet; numericLocals = new Map(); - /** Local arrays of the current function that no other binding can hold - * (ir/array-length-stability.ts). */ - unaliasedArrays: ReadonlySet = new Set(); - /** Dense array loops (dense-array-loops.ts) run in this function. */ - denseLoopsEnabled = false; - denseLoopEntries: ReturnType = new Map(); - /** The fast copy of a dense array loop being emitted. */ - denseLoop: DenseArrayLoop | null = null; captureIds = new Set(); /** Active induction bindings retain their width independently of size_t. */ integerLoopBindings = new Map< string, { slot: string; type: "i32" | "i64"; signed?: boolean; range?: { min: number; max: number } } >(); - countedLoopsEnabled = false; /** Speculative integer loops (speculative-integers.ts) run in this function. */ speculativeLoopsEnabled = false; /** The fast copy of a speculative integer loop being emitted: its @@ -1042,19 +1035,6 @@ export class LlEmitter { int32Elements: ((arr: IrExpr) => boolean) | null = null; /** Sums whose result only feeds 32-bit conversions (wrapping-sums.ts). */ wrappingSums = new Set(); - byteWindowsEnabled = true; - byteWindowEntries: ReturnType = new Map(); - /** Typed-array loop versioning (bytes-loop-versions.ts) for the current - * function: off in debug and suspending functions and inside a copy. */ - bytesLoopVersionsEnabled = false; - /** Loops being versioned around the statement being emitted: a loop is - * never versioned inside its own copies, and versions nest boundedly. */ - readonly bytesLoopVersioning = new Set(); - bytesLoopEntries: ReturnType | null = null; - bytesLoopFunction: IrFunction | null = null; - /** Counted loops whose integer-form safety test an enclosing version - * guard already passed. */ - readonly provenCountedGuards = new Set(); bytesBounds: ReadonlySet = new Set(); integerViews = new Map(); /** Module globals the current function owns outright (initializer @@ -3391,10 +3371,10 @@ export class LlEmitter { s: IrStmt & { kind: "for" }, counted: IntegerCountedForLoop | null, ): ReturnType { - const fn = this.bytesLoopFunction; + const fn = LOOPS.of(this).bytesLoopFunction; if (!fn) return null; - this.bytesLoopEntries ??= findInitializedByteLoopBindings(fn); - const initialized = this.bytesLoopEntries.get(s); + LOOPS.of(this).bytesLoopEntries ??= findInitializedByteLoopBindings(fn); + const initialized = LOOPS.of(this).bytesLoopEntries!.get(s); if (!initialized) return null; const invariantGlobals = new Set(this.stableGlobals); for (const global of this.initializerBindings.get(fn.name) ?? []) diff --git a/packages/compiler/src/backend/llvm/loop-state.ts b/packages/compiler/src/backend/llvm/loop-state.ts new file mode 100644 index 000000000..ce9189a45 --- /dev/null +++ b/packages/compiler/src/backend/llvm/loop-state.ts @@ -0,0 +1,76 @@ +/* The loop optimizations' per-function state: counted loops, dense array + * loops, byte windows and typed-array loop versioning, and the unaliased + * local arrays they rely on. */ +import { findUnaliasedLocalArrays } from "../../ir/array-length-stability.js"; +import type { IrFunction, IrStmt } from "../../ir/ir.js"; +import { findInitializedByteLoopBindings } from "./byte-windows.js"; +import type { DenseArrayLoop } from "./dense-array-loops.js"; +import type { LlEmitter } from "./emitter.js"; +import { defineModuleState } from "./emitter-state.js"; + +/** Sets the loop state for `fn` at the start of its emission; `numericFn` + * is `fn` with its initializer bindings (emitFunction). */ +export function beginFunctionLoops( + emitter: LlEmitter, + fn: IrFunction, + numericFn: IrFunction, +): void { + const loops = LOOPS.of(emitter); + loops.unaliasedArrays = findUnaliasedLocalArrays(fn); + loops.countedLoopsEnabled = emitter.debug === null && !fn.async && !fn.generator; + loops.byteWindowsEnabled = loops.countedLoopsEnabled; + loops.bytesLoopVersionsEnabled = loops.countedLoopsEnabled; + loops.bytesLoopEntries = null; + loops.bytesLoopFunction = fn; + loops.denseLoopsEnabled = loops.countedLoopsEnabled; + loops.denseLoop = null; + loops.denseLoopEntries = + loops.denseLoopsEnabled && numericFn.locals.some((l) => l.type.kind === "array") + ? findInitializedByteLoopBindings(fn) + : new Map(); + loops.byteWindowEntries = + loops.byteWindowsEnabled && numericFn.locals.some((l) => l.type.kind === "bytes") + ? findInitializedByteLoopBindings(fn) + : new Map(); +} + +/** The state LOOPS holds on each emitter. */ +export interface LoopState { + /** Local arrays of the current function that no other binding can hold + * (ir/array-length-stability.ts). */ + unaliasedArrays: ReadonlySet; + /** Dense array loops (dense-array-loops.ts) run in this function. */ + denseLoopsEnabled: boolean; + denseLoopEntries: ReturnType; + /** The fast copy of a dense array loop being emitted. */ + denseLoop: DenseArrayLoop | null; + countedLoopsEnabled: boolean; + byteWindowsEnabled: boolean; + byteWindowEntries: ReturnType; + /** Typed-array loop versioning (bytes-loop-versions.ts) for the current + * function: off in debug and suspending functions and inside a copy. */ + bytesLoopVersionsEnabled: boolean; + /** Loops being versioned around the statement being emitted: a loop is + * never versioned inside its own copies, and versions nest boundedly. */ + readonly bytesLoopVersioning: Set; + bytesLoopEntries: ReturnType | null; + bytesLoopFunction: IrFunction | null; + /** Counted loops whose integer-form safety test an enclosing version + * guard already passed. */ + readonly provenCountedGuards: Set; +} + +export const LOOPS = defineModuleState(() => ({ + unaliasedArrays: new Set(), + denseLoopsEnabled: false, + denseLoopEntries: new Map(), + denseLoop: null, + countedLoopsEnabled: false, + byteWindowsEnabled: true, + byteWindowEntries: new Map(), + bytesLoopVersionsEnabled: false, + bytesLoopVersioning: new Set(), + bytesLoopEntries: null, + bytesLoopFunction: null, + provenCountedGuards: new Set(), +})); diff --git a/scripts/split/README.md b/scripts/split/README.md new file mode 100644 index 000000000..7d79b35c1 --- /dev/null +++ b/scripts/split/README.md @@ -0,0 +1,22 @@ +# Split tools + +These scripts split a large source file into smaller files without changing behavior, and carry open branches across the split. Each transform is deterministic: the same config gives the same result on any version of the tree, and rehoming depends on that. + +Run them from a checkout with `node_modules` installed. That checkout is the tools directory (`--tools`) and supplies `typescript` and `oxfmt`. + +## Transforms +- `split-module.mjs plan|apply ` moves top-level declarations into new files. `plan` computes a manifest from seed declarations, and `apply` does the move. +- `split-methods.mjs apply ` moves class methods into exported functions that take the instance as a parameter (`emitter`). The class keeps a one-line stub with the same signature, so callers don't change. +- `move-state.mjs apply ` moves a group of class fields into a feature's state object, defined in the feature's file with `defineModuleState` (`packages/compiler/src/backend/llvm/emitter-state.ts`). It rewrites `this.` and `emitter.` to `KEY.of(this).` and creates files from templates. It also applies the change's other edits as literal replacements, in order. A replacement whose text isn't found the expected number of times is reported and skipped. + +Each transform takes `--repo ` (the tree to rewrite) and `--tools `, and formats what it writes with oxfmt. `configs/` holds the configs of splits that have landed. For example, `configs/loop-state/` moves the loop optimizations' state out of `LlEmitter`. + +## Rehoming an open branch +```sh +git fetch origin pull//head:pr- +node scripts/split/rehome-split.mjs --branch pr- --onto origin/main --config scripts/split/configs//.json +``` + +The result is the branch `pr--rehomed`, a merge of `origin/main` into the branch. + +The script applies the split's transform to the branch's merge base and to its tip, then cherry-picks the difference onto `--onto`. It records the result as a merge commit with parents (branch tip, `--onto`), so it can be pushed without rewriting the branch; `--squash` makes one ordinary commit instead. Conflicts made only of import lines in the split's files resolve to both sides' imports. Any other conflict stops the script, which lists the files and leaves its scratch worktree for you to resolve. diff --git a/scripts/split/configs/loop-state/emitter-state.template b/scripts/split/configs/loop-state/emitter-state.template new file mode 100644 index 000000000..62dc47b4e --- /dev/null +++ b/scripts/split/configs/loop-state/emitter-state.template @@ -0,0 +1,24 @@ +/* Per-feature emitter state. A feature defines its state in its own file + * with defineModuleState and reads it through the returned key, so adding + * or extending a feature does not edit LlEmitter's field list or + * constructor. */ +import type { LlEmitter } from "./emitter.js"; + +/** A feature's state on an emitter, read with `key.of(emitter)`. */ +export interface EmitterStateKey { + of(emitter: LlEmitter): T; +} + +let moduleSlots = 0; + +/** State that lives for the whole module, created on first read. */ +export function defineModuleState(create: (emitter: LlEmitter) => T): EmitterStateKey { + const slot = moduleSlots++; + return { + of(emitter) { + const states = emitter.moduleStates; + if (slot >= states.length || states[slot] === undefined) states[slot] = create(emitter); + return states[slot] as T; + }, + }; +} diff --git a/scripts/split/configs/loop-state/loop-state.config.json b/scripts/split/configs/loop-state/loop-state.config.json new file mode 100644 index 000000000..e749a4b39 --- /dev/null +++ b/scripts/split/configs/loop-state/loop-state.config.json @@ -0,0 +1,29 @@ +{ + "module": "packages/compiler/src/backend/llvm/emitter.ts", + "class": "LlEmitter", + "dir": "packages/compiler/src/backend/llvm", + "create": { + "emitter-state.ts": "emitter-state.template", + "loop-state.ts": "loop-state.template" + }, + "groups": [ + { + "file": "loop-state.ts", + "key": "LOOPS", + "interfaceName": "LoopState", + "fields": ["unaliasedArrays", "denseLoopsEnabled", "denseLoopEntries", "denseLoop", "countedLoopsEnabled", "byteWindowsEnabled", "byteWindowEntries", "bytesLoopVersionsEnabled", "bytesLoopVersioning", "bytesLoopEntries", "bytesLoopFunction", "provenCountedGuards"] + } + ], + "replacements": [ + { "file": "emitter.ts", "find": "export class LlEmitter {\n", "replace": "export class LlEmitter {\n /** Per-feature state (emitter-state.ts), indexed by each key's slot. */\n readonly moduleStates: unknown[] = [];\n" }, + { "file": "emitter.ts", "find": "import { type DenseArrayLoop } from \"./dense-array-loops.js\";\n", "replace": "" }, + { "file": "emitter.ts", "find": " const initialized = LOOPS.of(this).bytesLoopEntries.get(s);\n", "replace": " const initialized = LOOPS.of(this).bytesLoopEntries!.get(s);\n" }, + { "file": "emit-statements.ts", "find": " ? LOOPS.of(emitter).denseLoop.range\n", "replace": " ? LOOPS.of(emitter).denseLoop!.range\n", "count": 2 }, + { "file": "emit-function.ts", "find": " LOOPS.of(emitter).unaliasedArrays = findUnaliasedLocalArrays(fn);\n", "replace": "" }, + { "file": "emit-function.ts", "find": " LOOPS.of(emitter).countedLoopsEnabled = emitter.debug === null && !fn.async && !fn.generator;\n", "replace": " beginFunctionLoops(emitter, fn, numericFn);\n" }, + { "file": "emit-function.ts", "find": " LOOPS.of(emitter).byteWindowsEnabled = LOOPS.of(emitter).countedLoopsEnabled;\n LOOPS.of(emitter).bytesLoopVersionsEnabled = LOOPS.of(emitter).countedLoopsEnabled;\n LOOPS.of(emitter).bytesLoopEntries = null;\n LOOPS.of(emitter).bytesLoopFunction = fn;\n LOOPS.of(emitter).denseLoopsEnabled = LOOPS.of(emitter).countedLoopsEnabled;\n LOOPS.of(emitter).denseLoop = null;\n LOOPS.of(emitter).denseLoopEntries =\n LOOPS.of(emitter).denseLoopsEnabled && numericFn.locals.some((l) => l.type.kind === \"array\")\n ? findInitializedByteLoopBindings(fn)\n : new Map();\n LOOPS.of(emitter).byteWindowEntries =\n LOOPS.of(emitter).byteWindowsEnabled && numericFn.locals.some((l) => l.type.kind === \"bytes\")\n ? findInitializedByteLoopBindings(fn)\n : new Map();\n", "replace": "" }, + { "file": "emit-function.ts", "find": "import { LOOPS } from \"./loop-state.js\";\n", "replace": "import { beginFunctionLoops, LOOPS } from \"./loop-state.js\";\n" }, + { "file": "emit-function.ts", "find": "import { findUnaliasedLocalArrays } from \"../../ir/array-length-stability.js\";\n", "replace": "" }, + { "file": "emit-function.ts", "find": "import { findInitializedByteLoopBindings } from \"./byte-windows.js\";\n", "replace": "" } + ] +} diff --git a/scripts/split/configs/loop-state/loop-state.template b/scripts/split/configs/loop-state/loop-state.template new file mode 100644 index 000000000..ae9673267 --- /dev/null +++ b/scripts/split/configs/loop-state/loop-state.template @@ -0,0 +1,34 @@ +/* The loop optimizations' per-function state: counted loops, dense array + * loops, byte windows and typed-array loop versioning, and the unaliased + * local arrays they rely on. */ +import { findUnaliasedLocalArrays } from "../../ir/array-length-stability.js"; +import type { IrFunction, IrStmt } from "../../ir/ir.js"; +import { findInitializedByteLoopBindings } from "./byte-windows.js"; +import type { DenseArrayLoop } from "./dense-array-loops.js"; +import type { LlEmitter } from "./emitter.js"; + +/** Sets the loop state for `fn` at the start of its emission; `numericFn` + * is `fn` with its initializer bindings (emitFunction). */ +export function beginFunctionLoops( + emitter: LlEmitter, + fn: IrFunction, + numericFn: IrFunction, +): void { + const loops = LOOPS.of(emitter); + loops.unaliasedArrays = findUnaliasedLocalArrays(fn); + loops.countedLoopsEnabled = emitter.debug === null && !fn.async && !fn.generator; + loops.byteWindowsEnabled = loops.countedLoopsEnabled; + loops.bytesLoopVersionsEnabled = loops.countedLoopsEnabled; + loops.bytesLoopEntries = null; + loops.bytesLoopFunction = fn; + loops.denseLoopsEnabled = loops.countedLoopsEnabled; + loops.denseLoop = null; + loops.denseLoopEntries = + loops.denseLoopsEnabled && numericFn.locals.some((l) => l.type.kind === "array") + ? findInitializedByteLoopBindings(fn) + : new Map(); + loops.byteWindowEntries = + loops.byteWindowsEnabled && numericFn.locals.some((l) => l.type.kind === "bytes") + ? findInitializedByteLoopBindings(fn) + : new Map(); +} diff --git a/scripts/split/move-state.mjs b/scripts/split/move-state.mjs new file mode 100644 index 000000000..5da787ac2 --- /dev/null +++ b/scripts/split/move-state.mjs @@ -0,0 +1,202 @@ +#!/usr/bin/env node +/* Move a group of LlEmitter fields into a feature's state object, as a + * deterministic rewrite (the same config gives the same result on any + * version of the tree, which the rehoming tool relies on). + * + * node move-state.mjs apply [--repo ] [--tools ] + * + * : + * { "module": "packages/compiler/src/backend/llvm/emitter.ts", + * "class": "LlEmitter", + * "dir": "packages/compiler/src/backend/llvm", + * "groups": [{ "file": "loop-state.ts", "key": "LOOPS", + * "fields": ["denseLoop", ...] }], + * "create": { "loop-state.ts": "loop-state.template.ts" }, + * "replacements": [{ "file": "emit-function.ts", "find": "...", "replace": "...", "count": 1 }], + * "receivers": ["this", "emitter"] } + * + * For each group: the fields' declarations leave the class (their doc + * comments and initializers move into the state object's interface and + * factory in the group's file), every `this.` and `emitter.` + * (or the config's receivers) in the directory's sources becomes + * `.of(this).` / `.of(emitter).`, and each + * rewritten file imports the key. + * The group's file must already exist and hold everything else (header, + * setup function); the tool appends the interface and key to it. */ +import { existsSync, readFileSync, writeFileSync, readdirSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { createRequire } from "node:module"; +import { execFileSync } from "node:child_process"; + +const [command, configPath, ...flags] = process.argv.slice(2); +if (command !== "apply") throw new Error("usage: move-state.mjs apply "); +const config = JSON.parse(readFileSync(configPath, "utf8")); +const flag = (name) => { + const i = flags.indexOf(name); + return i >= 0 ? flags[i + 1] : undefined; +}; +const repo = resolve(flag("--repo") ?? process.cwd()); +const toolsDir = resolve(flag("--tools") ?? process.cwd()); +const require = createRequire(join(toolsDir, "package.json")); +const ts = require("typescript"); +const modulePath = join(repo, config.module); +const dir = join(repo, config.dir); + +// Adds an import line after the file's first block of imports (up to a +// blank line or other code). +const addImport = (text, line) => { + const sf = ts.createSourceFile("x.ts", text, ts.ScriptTarget.Latest, true); + let last; + for (const s of sf.statements) { + if (!ts.isImportDeclaration(s)) break; + if (last && /\n[ \t]*\n/.test(text.slice(last.getEnd(), s.getStart()))) break; + last = s; + } + const at = last ? last.getEnd() + 1 : 0; + return text.slice(0, at) + line + "\n" + text.slice(at); +}; +const configDir = dirname(resolve(configPath)); +// The names the class's instance goes by in the directory's sources. +const receivers = config.receivers ?? ["this", "emitter"]; +const touched = new Set(); +// Files the change creates, from templates next to the config (kept when a +// branch already has the file). +for (const [file, template] of Object.entries(config.create ?? {})) { + const path = join(dir, file); + if (existsSync(path)) continue; + writeFileSync(path, readFileSync(join(configDir, template), "utf8")); + touched.add(path); +} +let moduleText = readFileSync(modulePath, "utf8"); +for (const group of config.groups) { + const sf = ts.createSourceFile(modulePath, moduleText, ts.ScriptTarget.Latest, true); + const cls = sf.statements.find((s) => ts.isClassDeclaration(s) && s.name?.text === config.class); + if (!cls) throw new Error(`no class ${config.class}`); + const wanted = new Set(group.fields); + const members = []; + for (const m of cls.members) { + if (ts.isPropertyDeclaration(m) && ts.isIdentifier(m.name) && wanted.has(m.name.text)) + members.push(m); + } + // A field the tree does not have yet (an older merge base) is left out, so + // the move still applies there. + const missing = group.fields.filter((f) => !members.some((m) => m.name.text === f)); + if (missing.length) + console.warn(`${group.file}: fields not in ${config.class}: ${missing.join(", ")}; skipped`); + group.present = members.map((m) => m.name.text); + // Interface members and factory entries, in the class's order. + const iface = []; + const factory = []; + for (const m of members) { + const lead = moduleText.slice(m.getFullStart(), m.getStart()).replace(/^\s*\n/, ""); + const readonly = m.modifiers?.some((x) => x.kind === ts.SyntaxKind.ReadonlyKeyword) + ? "readonly " + : ""; + const name = m.name.text; + if (!m.initializer) + throw new Error( + `${name}: a field without an initializer needs its constructor line moved by hand`, + ); + // An unannotated field takes its initializer's type: a boolean, + // number or string literal, or `new C(...)`. + const init = m.initializer; + const type = m.type + ? m.type.getText(sf) + : init.kind === ts.SyntaxKind.TrueKeyword || init.kind === ts.SyntaxKind.FalseKeyword + ? "boolean" + : ts.isNumericLiteral(init) + ? "number" + : ts.isStringLiteral(init) + ? "string" + : ts.isNewExpression(init) && init.typeArguments?.length + ? `${init.expression.getText(sf)}<${init.typeArguments.map((t) => t.getText(sf)).join(", ")}>` + : null; + if (type === null) throw new Error(`${name}: give the field a type annotation first`); + iface.push(`${lead} ${readonly}${name}: ${type};`); + factory.push(` ${name}: ${m.initializer.getText(sf)},`); + } + // Remove the declarations (with their leading comments) from the class. + const cuts = members.map((m) => [m.getFullStart(), m.getEnd()]).sort((a, b) => b[0] - a[0]); + for (const [s, e] of cuts) moduleText = moduleText.slice(0, s) + moduleText.slice(e); + const iName = group.interface ?? `${group.key[0]}${group.key.slice(1).toLowerCase()}State`; + const stateText = + `\n/** The state ${group.key} holds on each emitter. */\n` + + `export interface ${group.interfaceName ?? iName} {\n${iface.join("\n")}\n}\n\n` + + `export const ${group.key} = defineModuleState<${group.interfaceName ?? iName}>(() => ({\n${factory.join("\n")}\n}));\n`; + const groupPath = join(dir, group.file); + let groupText = readFileSync(groupPath, "utf8"); + if (!groupText.includes('from "./emitter-state.js"')) + groupText = addImport(groupText, `import { defineModuleState } from "./emitter-state.js";`); + groupText = groupText.trimEnd() + "\n" + stateText; + writeFileSync(groupPath, groupText); + touched.add(groupPath); + group.rename = group.present.length + ? new RegExp(`(? f.endsWith(".ts"))) { + const path = join(dir, file); + let text = path === modulePath ? moduleText : readFileSync(path, "utf8"); + let changed = false; + for (const group of config.groups) { + if (!group.rename) continue; + const next = text.replace( + group.rename, + (_, recv, field) => `${group.key}.of(${recv}).${field}`, + ); + if (next === text) continue; + text = next; + changed = true; + if (file === group.file) continue; + const spec = `./${group.file.replace(/\.ts$/, ".js")}`; + const existing = new RegExp( + `^import \\{([^}]*)\\} from "${spec.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}";$`, + "m", + ); + const m = existing.exec(text); + if (m) { + const names = new Set( + m[1] + .split(",") + .map((s) => s.trim()) + .filter(Boolean), + ); + names.add(group.key); + text = text.replace(existing, `import { ${[...names].sort().join(", ")} } from "${spec}";`); + } else { + text = addImport(text, `import { ${group.key} } from "${spec}";`); + } + } + if (changed) { + writeFileSync(path, text); + touched.add(path); + } +} +// The change's remaining edits, as literal replacements in order (the +// setup function call, imports, narrowing the rename loses). One whose text +// is not found as often as expected is reported and skipped: the branch +// changed that code, and the rehome's cherry-pick shows the difference. +let skipped = 0; +for (const r of config.replacements ?? []) { + const path = join(dir, r.file); + const text = existsSync(path) ? readFileSync(path, "utf8") : ""; + const count = text.split(r.find).length - 1; + if (count !== (r.count ?? 1)) { + console.warn( + `${r.file}: expected ${r.count ?? 1} of ${JSON.stringify(r.find.slice(0, 60))}, found ${count}; skipped`, + ); + skipped++; + continue; + } + writeFileSync(path, text.split(r.find).join(r.replace)); + touched.add(path); +} +const oxfmt = join(toolsDir, "node_modules/.bin/oxfmt"); +execFileSync(oxfmt, [...touched], { cwd: repo, stdio: "inherit" }); +if (skipped) console.warn(`${skipped} replacement(s) skipped`); +console.log( + `moved ${config.groups.map((g) => g.present.length).reduce((a, b) => a + b, 0)} fields; rewrote ${touched.size} files`, +); diff --git a/scripts/split/rehome-split.mjs b/scripts/split/rehome-split.mjs new file mode 100644 index 000000000..2884558fb --- /dev/null +++ b/scripts/split/rehome-split.mjs @@ -0,0 +1,311 @@ +#!/usr/bin/env node +/* Rehome a branch's changes across a module split. + * + * A split moves whole declarations out of one module into new files, so git + * cannot follow a branch's edits into them: merging the split reports every + * edit to a moved declaration as a conflict. This script carries the branch + * across instead. It applies the same split (split-module.mjs, + * split-methods.mjs or move-state.mjs with the split's config; each also + * formats with oxfmt) to the branch's merge base and to the branch tip, and + * cherry-picks the difference onto the target. That is a three-way merge + * whose base and branch side already use the new layout, so each hunk lands + * in the file its declaration moved to. Hunks in declarations that stay, and + * declarations the branch adds, stay in the module. + * + * node rehome-split.mjs --branch --onto --config + * [--out ] [--squash] [--scratch ] + * + * Run it from a checkout with node_modules installed (it supplies typescript + * and oxfmt). --onto is a commit that contains the split, usually main. + * The result is a new branch (default -rehomed). By default its tip + * is a merge commit with parents and : the branch's history + * is kept and the result reads as "merge main into the branch", so it can be + * pushed without rewriting the branch. --squash makes one ordinary commit on + * instead. When the cherry-pick conflicts, the script stops and + * leaves the work tree with the conflicts to resolve; see the message. */ +import { execFileSync, spawnSync } from "node:child_process"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { createRequire } from "node:module"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const args = process.argv.slice(2); +const opt = (name, fallback) => { + const i = args.indexOf(`--${name}`); + return i >= 0 ? args[i + 1] : fallback; +}; +const has = (name) => args.includes(`--${name}`); +const branch = opt("branch"); +const onto = opt("onto"); +const configPath = opt("config"); +if (!branch || !onto || !configPath) { + console.error( + "usage: rehome-split.mjs --branch --onto --config [--out ] [--squash]", + ); + process.exit(2); +} +const out = opt("out", `${branch.replace(/^.*\//, "")}-rehomed`); +const tools = process.cwd(); +const here = dirname(fileURLToPath(import.meta.url)); +const config = JSON.parse(readFileSync(configPath, "utf8")); +// A config with "groups" moves fields into feature state (move-state.mjs), +// one with "methods" moves methods (split-methods.mjs), and any other moves +// top-level declarations (split-module.mjs). +const splitTool = join( + here, + config.groups ? "move-state.mjs" : config.class ? "split-methods.mjs" : "split-module.mjs", +); + +const git = (...a) => + execFileSync("git", a, { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }).trim(); +const gitIn = (cwd, ...a) => + execFileSync("git", a, { cwd, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }).trim(); +// Every commit the script makes is authored and committed as the branch's +// last author. +const author = (format) => git("log", "-1", `--format=${format}`, branch); +const ident = { + GIT_AUTHOR_NAME: author("%an"), + GIT_AUTHOR_EMAIL: author("%ae"), +}; +ident.GIT_COMMITTER_NAME = ident.GIT_AUTHOR_NAME; +ident.GIT_COMMITTER_EMAIL = ident.GIT_AUTHOR_EMAIL; +Object.assign(process.env, ident); + +const tip = git("rev-parse", `${branch}^{commit}`); +const target = git("rev-parse", `${onto}^{commit}`); +// The split must not be in the branch already, or there is nothing to rehome. +const moduleFile = config.module; +// With several merge bases (criss-cross history, such as a branch that +// carries another branch's commits that main merged), merge them into one +// virtual base, as git's merge does, so the cherry-pick sees the same base. +let base = git("merge-base", tip, target); +{ + const bases = git("merge-base", "--all", tip, target).split("\n").filter(Boolean); + for (const next of bases.slice(1)) { + const merged = spawnSync("git", ["merge-tree", "--write-tree", base, next], { + encoding: "utf8", + }); + const tree = merged.stdout.split("\n")[0]; + base = git("commit-tree", tree, "-p", base, "-p", next, "-m", "virtual merge base"); + } +} + +// Scratch worktrees go next to the checkout: not inside it (they would show +// as untracked files) and not under node_modules (oxfmt skips those paths). +const scratch = mkdtempSync(opt("scratch", join(dirname(tools), ".rehome-"))); +const worktree = (name, commit) => { + const dir = join(scratch, name); + git("worktree", "add", "--detach", "-q", dir, commit); + return dir; +}; +const split = (dir) => { + execFileSync( + process.execPath, + [splitTool, "apply", configPath, "--repo", dir, "--tools", tools], + { + stdio: ["ignore", "pipe", "inherit"], + }, + ); + gitIn(dir, "add", "-A"); + return gitIn(dir, "write-tree"); +}; +const commitTree = (tree, parents, message) => + execFileSync("git", ["commit-tree", tree, ...parents.flatMap((p) => ["-p", p]), "-m", message], { + encoding: "utf8", + env: { + ...process.env, + ...ident, + }, + }).trim(); + +let baseDir, tipDir; +try { + baseDir = worktree("base", base); + tipDir = worktree("tip", tip); + const splitBase = commitTree(split(baseDir), [base], "split of the merge base"); + const splitTip = commitTree(split(tipDir), [splitBase], "split of the branch tip"); + if (git("rev-parse", `${splitBase}^{tree}`) === git("rev-parse", `${splitTip}^{tree}`)) { + console.log(`${branch} has no changes relative to ${base.slice(0, 9)}`); + process.exit(0); + } + const touched = git("diff", "--name-only", splitBase, splitTip).split("\n").filter(Boolean); + console.log(`${branch}: ${touched.length} files changed in the split layout`); + // Apply the branch's changes onto the target in a worktree of its own. + const resultDir = worktree("result", target); + const pick = spawnSync( + "git", + ["-c", "merge.conflictStyle=merge", "cherry-pick", "--no-commit", splitTip], + { + cwd: resultDir, + encoding: "utf8", + }, + ); + // Moved declarations bring short import blocks, where an import main added + // and one the branch added can sit on adjacent lines. Resolve conflicts + // made only of import declarations to the union of both sides. + if (pick.status !== 0) { + // Only the files the split writes: other conflicts are the branch's own. + // A method split names its files in the config; a module split, in its manifest. + const files = config.groups + ? [ + ...config.groups.map((g) => g.file), + ...Object.keys(config.create ?? {}), + ...(config.replacements ?? []).map((r) => r.file), + ] + : config.class + ? Object.keys(config.methods) + : Object.keys( + (({ files, ...rest }) => files ?? rest)( + JSON.parse( + readFileSync(resolve(dirname(resolve(configPath)), config.manifest), "utf8"), + ), + ), + ); + const moduleDir = config.dir ?? dirname(moduleFile); + const splitFiles = new Set([moduleFile, ...files.map((f) => join(moduleDir, f))]); + for (const file of gitIn(resultDir, "diff", "--name-only", "--diff-filter=U") + .split("\n") + .filter(Boolean)) { + if (splitFiles.has(file) && resolveImportConflicts(join(resultDir, file))) { + execFileSync(join(tools, "node_modules/.bin/oxfmt"), [file], { + cwd: resultDir, + stdio: ["ignore", "ignore", "inherit"], + }); + gitIn(resultDir, "add", file); + console.log(`${file}: import conflicts resolved to both sides' imports`); + } + } + } + if (pick.status !== 0 && gitIn(resultDir, "diff", "--name-only", "--diff-filter=U") !== "") { + const conflicts = gitIn(resultDir, "diff", "--name-only", "--diff-filter=U"); + console.error( + `conflicts in:\n${conflicts}\n\nResolve them in ${resultDir}, then run:\n git -C ${resultDir} add -A && git -C ${resultDir} commit -m "Merge ${onto} into ${branch}" # or rerun with the resolution\nThe scratch worktrees stay until you remove ${scratch}.`, + ); + process.exit(1); + } + const tree = gitIn(resultDir, "write-tree"); + const subjects = git("log", "--format=%s", `${base}..${tip}`).split("\n").filter(Boolean); + const result = has("squash") + ? commitTree( + tree, + [target], + subjects.length === 1 + ? subjects[0] + : `${subjects.at(-1)}\n\n${subjects.map((s) => `- ${s}`).join("\n")}`, + ) + : commitTree( + tree, + [tip, target], + `Merge ${onto} into ${branch.replace(/^.*\//, "")}\n\nCarries the branch's changes to ${moduleFile} into the files its declarations moved to.`, + ); + git("branch", "-f", out, result); + console.log( + `${out} -> ${result.slice(0, 9)} (${has("squash") ? "one commit on" : "merge of"} ${onto})`, + ); + for (const dir of [baseDir, tipDir, resultDir]) git("worktree", "remove", "--force", dir); + rmSync(scratch, { recursive: true, force: true }); +} catch (error) { + console.error(error.stderr?.toString() || error.message); + console.error(`scratch worktrees left in ${scratch}`); + process.exit(1); +} + +/* Rewrites every conflict in `path` whose two sides hold only import + * declarations as the union of both: one declaration per module and kind, + * with the named imports of both sides in first-seen order. Returns false, + * leaving the file unchanged, when any conflict holds other code. */ +function resolveImportConflicts(path) { + const ts = createRequire(join(tools, "package.json"))("typescript"); + const text = readFileSync(path, "utf8"); + const region = /^<<<<<<< [^\n]*\n([\s\S]*?)^=======\n([\s\S]*?)^>>>>>>> [^\n]*\n/gm; + const imports = (side) => { + const sf = ts.createSourceFile("side.ts", side, ts.ScriptTarget.Latest, true); + return sf.statements.every(ts.isImportDeclaration) ? sf.statements : null; + }; + let ok = true; + const imported = new Set(); + const resolved = text.replace(region, (whole, ours, theirs) => { + const a = imports(ours), + b = imports(theirs); + if (!a || !b) { + ok = false; + return whole; + } + const merged = new Map(); + for (const decl of [...a, ...b]) { + const clause = decl.importClause; + const named = + clause?.namedBindings && ts.isNamedImports(clause.namedBindings) + ? clause.namedBindings + : null; + if (!clause || !named || clause.name) { + // Side-effect, default and namespace imports are kept as written, once. + merged.set(decl.getText(), null); + continue; + } + const key = `${clause.isTypeOnly ? "type " : ""}${decl.moduleSpecifier.getText()}`; + const entry = merged.get(key) ?? { + typeOnly: clause.isTypeOnly, + from: decl.moduleSpecifier.getText(), + names: [], + }; + for (const element of named.elements) { + imported.add(element.name.text); + if (!entry.names.includes(element.getText())) entry.names.push(element.getText()); + } + merged.set(key, entry); + } + return ( + [...merged] + .map(([key, entry]) => + entry === null + ? key + : `import ${entry.typeOnly ? "type " : ""}{ ${entry.names.join(", ")} } from ${entry.from};`, + ) + .join("\n") + "\n" + ); + }); + if (!ok || resolved === text) return false; + writeFileSync(path, pruneUnusedImports(ts, resolved, imported)); + return true; +} + +/* Drops the named imports among `names` that nothing outside import + * declarations references: a side's import whose use the other side's code + * replaced. The split computes each file's imports from its references, so + * this keeps a rehomed file's imports the ones the split would write. */ +function pruneUnusedImports(ts, text, names) { + const sf = ts.createSourceFile("file.ts", text, ts.ScriptTarget.Latest, true); + const used = new Set(); + const visit = (node) => { + if (ts.isImportDeclaration(node)) return; + if (ts.isIdentifier(node)) used.add(node.text); + ts.forEachChild(node, visit); + }; + visit(sf); + const edits = []; + for (const decl of sf.statements) { + const named = decl.importClause?.namedBindings; + if ( + !ts.isImportDeclaration(decl) || + !named || + !ts.isNamedImports(named) || + decl.importClause.name + ) + continue; + const kept = named.elements.filter((e) => !names.has(e.name.text) || used.has(e.name.text)); + if (kept.length === named.elements.length) continue; + const from = decl.moduleSpecifier.getText(); + const prefix = decl.importClause.isTypeOnly ? "import type" : "import"; + // A dropped declaration takes its line break with it. + const end = kept.length || text[decl.getEnd()] !== "\n" ? decl.getEnd() : decl.getEnd() + 1; + edits.push([ + decl.getStart(), + end, + kept.length ? `${prefix} { ${kept.map((e) => e.getText()).join(", ")} } from ${from};` : "", + ]); + } + for (const [start, end, replacement] of edits.reverse()) + text = text.slice(0, start) + replacement + text.slice(end); + return text; +} diff --git a/scripts/split/split-methods.mjs b/scripts/split/split-methods.mjs new file mode 100644 index 000000000..66d6b36ad --- /dev/null +++ b/scripts/split/split-methods.mjs @@ -0,0 +1,414 @@ +#!/usr/bin/env node +/* Move methods of one class into other files as functions of a host, as a + * pure move. + * + * node split-methods.mjs apply [--repo ] [--tools ] + * + * : + * { "tsconfig": "packages/compiler/tsconfig.json", + * "module": "packages/compiler/src/backend/llvm/emitter.ts", + * "class": "LlEmitter", + * "methods": { "emit-statements.ts": ["emitStmtBody", ...], ... }, + * "headers": { "emit-statements.ts": "/* ... *\/" } } + * + * Each listed method becomes `export function (host: , ...)` + * in its file (relative to the module's directory), with its doc comment + * and `this` replaced by the host parameter (`emitter`, or "hostName"). The class keeps a stub with the method's + * signature that calls the function, so every caller is unchanged. Members + * the moved bodies reach through `host` lose a `private` modifier (a + * compile-time check only). Each new file imports what its functions + * reference from the module's scope, with paths rebased; the module imports + * the functions and drops imports nothing uses anymore. Names the config + * lists but the class lacks are skipped. */ +import { readFileSync, writeFileSync, existsSync } from "node:fs"; +import { dirname, join, relative, resolve, posix } from "node:path"; +import { createRequire } from "node:module"; +import { execFileSync } from "node:child_process"; + +const [command, configPath, ...flags] = process.argv.slice(2); +if (command !== "apply") throw new Error("usage: split-methods.mjs apply "); +const config = JSON.parse(readFileSync(configPath, "utf8")); +const flag = (name) => { + const i = flags.indexOf(name); + return i >= 0 ? flags[i + 1] : undefined; +}; +const repo = resolve(flag("--repo") ?? process.cwd()); +const toolsDir = resolve(flag("--tools") ?? process.cwd()); +const require = createRequire(join(toolsDir, "package.json")); +const ts = require("typescript"); +const modulePath = join(repo, config.module); +const moduleDir = dirname(modulePath); +const className = config.class; +// The host parameter's name; a moved body that already uses it is refused. +const hostName = config.hostName ?? "emitter"; + +const text = readFileSync(modulePath, "utf8"); +const host = ts.createCompilerHost({ noEmit: true }); +const sf = ts.createSourceFile(modulePath, text, ts.ScriptTarget.Latest, true); +const realGet = host.getSourceFile; +host.getSourceFile = (f, l, ...r) => + resolve(f) === modulePath ? sf : realGet.call(host, f, l, ...r); +const program = ts.createProgram([modulePath], { noEmit: true, noResolve: true }, host); +const checker = program.getTypeChecker(); +const source = program.getSourceFile(modulePath); + +const cls = source.statements.find((s) => ts.isClassDeclaration(s) && s.name?.text === className); +if (!cls) throw new Error(`no class ${className}`); +const memberByName = new Map(); +for (const m of cls.members) + if (m.name && ts.isIdentifier(m.name)) { + if (!memberByName.has(m.name.text)) memberByName.set(m.name.text, []); + memberByName.get(m.name.text).push(m); + } + +// Module scope: import bindings and top-level declarations. +const imports = new Map(); +const topLevel = new Map(); +for (const st of source.statements) { + if (ts.isImportDeclaration(st)) { + const c = st.importClause; + if (!c) continue; + if (c.name) imports.set(c.name.text, { st, kind: "default", typeOnly: c.isTypeOnly }); + const nb = c.namedBindings; + if (nb && ts.isNamespaceImport(nb)) + imports.set(nb.name.text, { st, kind: "namespace", typeOnly: c.isTypeOnly }); + if (nb && ts.isNamedImports(nb)) + for (const el of nb.elements) + imports.set(el.name.text, { + st, + kind: "named", + typeOnly: c.isTypeOnly || el.isTypeOnly, + imported: (el.propertyName ?? el.name).text, + }); + continue; + } + const names = []; + if ( + (ts.isFunctionDeclaration(st) || + ts.isClassDeclaration(st) || + ts.isInterfaceDeclaration(st) || + ts.isTypeAliasDeclaration(st) || + ts.isEnumDeclaration(st)) && + st.name + ) + names.push(st.name.text); + if (ts.isVariableStatement(st)) + for (const d of st.declarationList.declarations) + if (ts.isIdentifier(d.name)) names.push(d.name.text); + for (const n of names) topLevel.set(n, st); +} +const isExported = (st) => + st.modifiers?.some((m) => m.kind === ts.SyntaxKind.ExportKeyword) ?? false; +const isTypeDecl = (st) => ts.isInterfaceDeclaration(st) || ts.isTypeAliasDeclaration(st); + +/** Module-scope references in `node`: name -> { kind: "import"|"local" }, and `this.` names. */ +function references(node) { + const refs = new Map(); + const members = new Set(); + const visit = (n) => { + if ( + n.kind === ts.SyntaxKind.ThisKeyword && + n.parent && + ts.isPropertyAccessExpression(n.parent) && + n.parent.expression === n + ) + members.add(n.parent.name.text); + if (ts.isIdentifier(n)) { + const p = n.parent; + const propName = + p && + ((ts.isPropertyAccessExpression(p) && p.name === n) || + (ts.isQualifiedName(p) && p.right === n) || + (ts.isPropertyAssignment(p) && p.name === n)); + if (!propName) { + let sym = checker.getSymbolAtLocation(n); + if (p && ts.isShorthandPropertyAssignment(p)) + sym = checker.getShorthandAssignmentValueSymbol(p); + const decl = sym?.declarations?.[0]; + if (decl && decl.getSourceFile() === source) { + if (ts.isImportSpecifier(decl) || ts.isNamespaceImport(decl) || ts.isImportClause(decl)) + refs.set(decl.name.text, { kind: "import" }); + else if ( + topLevel.get(n.text) && + (decl === topLevel.get(n.text) || + decl.parent?.parent === topLevel.get(n.text) || + decl.parent === topLevel.get(n.text)) + ) + refs.set(n.text, { kind: "local" }); + } + } + } + ts.forEachChild(n, visit); + }; + visit(node); + return { refs, members }; +} + +const rebase = (spec, toDir) => { + if (!spec.startsWith(".")) return spec; + let rel = posix.normalize(relative(toDir, resolve(moduleDir, spec)).split("\\").join("/")); + return rel.startsWith(".") ? rel : "./" + rel; +}; +const specTo = (fromDir, file) => { + let rel = relative(fromDir, file).split("\\").join("/").replace(/\.ts$/, ".js"); + return rel.startsWith(".") ? rel : "./" + rel; +}; + +const edits = []; // [start, end, replacement] in the module +const stubs = []; +const files = new Map(); +const publicize = new Set(); +const moduleNeeds = new Map(); // spec -> names (functions the stubs call) +const exportLocals = new Set(); +let moved = 0; + +for (const [file, names] of Object.entries(config.methods)) { + const path = join(moduleDir, file); + const dir = dirname(path); + const entry = { decls: [], named: new Map(), other: [] }; + const addNamed = (spec, name, imported, typeOnly) => { + if (!entry.named.has(spec)) entry.named.set(spec, new Map()); + const m = entry.named.get(spec); + const prev = m.get(name); + m.set(name, { imported, typeOnly: prev ? prev.typeOnly && typeOnly : typeOnly }); + }; + addNamed(specTo(dir, modulePath), className, className, true); + for (const name of names) { + const decls = memberByName.get(name); + if (!decls) { + console.error(`skip: ${className}.${name} not found`); + continue; + } + if (decls.length !== 1 || !ts.isMethodDeclaration(decls[0]) || !decls[0].body) + throw new Error(`${name}: expected one method with a body`); + const m = decls[0]; + if (m.typeParameters?.length || m.asteriskToken) + throw new Error(`${name}: generic and generator methods are not handled`); + const isAsync = m.modifiers?.some((x) => x.kind === ts.SyntaxKind.AsyncKeyword) ?? false; + // The function: doc comment, signature (initializers kept, so a + // defaulted parameter after an optional one stays valid; the stub + // passes every argument), and the body with this -> host. + const lead = text.slice(m.getFullStart(), m.getStart()).replace(/^\s*\n/, ""); + const params = m.parameters.map((p) => { + const nameText = p.name.getText(); + // A parameter typed only by its default takes the default's widened type. + const type = p.type + ? `: ${p.type.getText()}` + : p.initializer + ? `: ${checker.typeToString(checker.getBaseTypeOfLiteralType(checker.getTypeAtLocation(p.initializer)))}` + : ""; + const optional = p.questionToken ? "?" : ""; + let init = ""; + if (p.initializer) { + const initText = p.initializer.getText(); + if (/\bthis\b/.test(initText)) + throw new Error(`${name}: a parameter default that reads this is not handled`); + init = ` = ${initText}`; + } + return `${p.dotDotDotToken ? "..." : ""}${nameText}${optional}${type}${init}`; + }); + const ret = m.type ? `: ${m.type.getText()}` : ""; + const body = m.body; + const thisEdits = []; + const findThis = (n) => { + // `this` in an expression, or in a type query (`typeof this.x`). + // (In a type query `this` parses as an identifier.) + if ( + n.kind === ts.SyntaxKind.ThisKeyword || + (ts.isIdentifier(n) && n.text === "this" && ts.findAncestor(n, ts.isTypeQueryNode)) + ) + thisEdits.push(n.getStart()); + if (ts.isFunctionExpression(n) || ts.isFunctionDeclaration(n) || ts.isClassLike(n)) + throw new Error(`${name}: a nested function or class rebinds this`); + ts.forEachChild(n, findThis); + }; + ts.forEachChild(body, findThis); + const clash = (n) => { + if (ts.isIdentifier(n) && n.text === hostName) + throw new Error(`${name}: already uses the name ${hostName}`); + ts.forEachChild(n, clash); + }; + clash(m); + let bodyText = ""; + let cursor = body.getStart(); + for (const at of thisEdits.sort((a, b) => a - b)) { + bodyText += text.slice(cursor, at) + hostName; + cursor = at + 4; + } + bodyText += text.slice(cursor, body.end); + const leadOut = lead + .split("\n") + .map((l) => (l.startsWith(" ") ? l.slice(2) : l)) + .join("\n"); + const fn = `${leadOut}export ${isAsync ? "async " : ""}function ${name}(${hostName}: ${className}${params.length ? ", " + params.join(", ") : ""})${ret} ${bodyText}`; + entry.decls.push(fn); + stubs.push(m); + if (!moduleNeeds.has(specTo(moduleDir, path))) + moduleNeeds.set(specTo(moduleDir, path), new Set()); + moduleNeeds.get(specTo(moduleDir, path)).add(name); + // What the function needs. + const { refs, members } = references(m); + for (const member of members) publicize.add(member); + for (const [ref, { kind }] of refs) { + if (kind === "import") { + const imp = imports.get(ref); + const spec = rebase(imp.st.moduleSpecifier.text, dir); + if (imp.kind === "named") addNamed(spec, ref, imp.imported, imp.typeOnly); + else entry.other.push({ spec, kind: imp.kind, name: ref, typeOnly: imp.typeOnly }); + } else { + const st = topLevel.get(ref); + if (!isExported(st)) exportLocals.add(st); + addNamed(specTo(dir, modulePath), ref, ref, isTypeDecl(st)); + } + } + moved++; + } + files.set(file, entry); +} + +const renderImports = (entry) => { + const lines = []; + const seen = new Set(); + for (const o of entry.other) { + const key = `${o.kind}:${o.name}:${o.spec}`; + if (seen.has(key)) continue; + seen.add(key); + lines.push( + o.kind === "namespace" + ? `import ${o.typeOnly ? "type " : ""}* as ${o.name} from "${o.spec}";` + : `import ${o.typeOnly ? "type " : ""}${o.name} from "${o.spec}";`, + ); + } + for (const [spec, names] of entry.named) { + const all = [...names.entries()].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)); + const allType = all.every(([, v]) => v.typeOnly); + const parts = all.map( + ([n, v]) => + `${!allType && v.typeOnly ? "type " : ""}${v.imported === n ? n : `${v.imported} as ${n}`}`, + ); + lines.push(`import ${allType ? "type " : ""}{ ${parts.join(", ")} } from "${spec}";`); + } + return lines; +}; + +for (const [file, entry] of files) { + if (!entry.decls.length) continue; + const path = join(moduleDir, file); + if (existsSync(path)) throw new Error(`${file} exists`); + const header = config.headers?.[file]; + writeFileSync( + path, + [...(header ? [header] : []), ...renderImports(entry), "", entry.decls.join("\n\n"), ""].join( + "\n", + ), + ); +} + +// Each moved method leaves a stub with its signature (defaults included) +// that calls the function; one the moved bodies call loses `private`. +const stubSet = new Set(stubs); +for (const m of stubs) { + let signature = m.modifiers + ? text.slice(m.getStart(), m.body.getStart()).trimEnd() + : text.slice(m.getStart(), m.body.getStart()).trimEnd(); + if (publicize.has(m.name.text)) signature = signature.replace(/^private /, ""); + const args = m.parameters.map((p) => `${p.dotDotDotToken ? "..." : ""}${p.name.getText()}`); + const call = `${m.name.text}(this${args.length ? ", " + args.join(", ") : ""})`; + edits.push([m.getFullStart(), m.end, `\n ${signature} {\n return ${call};\n }`]); +} +// Other members the moved bodies reach through the host lose `private`. +for (const name of publicize) + for (const m of memberByName.get(name) ?? []) + if (!stubSet.has(m)) + for (const mod of m.modifiers ?? []) + if (mod.kind === ts.SyntaxKind.PrivateKeyword) + edits.push([mod.getStart(), mod.end + 1, ""]); +// Top-level declarations the new files import get `export`. +for (const st of exportLocals) edits.push([st.getStart(), st.getStart(), "export "]); + +let out = ""; +let cursor = 0; +for (const [a, b, r] of edits.sort((x, y) => x[0] - y[0])) { + out += text.slice(cursor, a) + r; + cursor = b; +} +out += text.slice(cursor); + +// Re-analyze the rewritten module for unused imports, then add the new ones. +const sf2 = ts.createSourceFile(modulePath, out, ts.ScriptTarget.Latest, true); +const host2 = ts.createCompilerHost({ noEmit: true }); +const get2 = host2.getSourceFile; +host2.getSourceFile = (f, l, ...r) => + resolve(f) === modulePath ? sf2 : get2.call(host2, f, l, ...r); +const program2 = ts.createProgram([modulePath], { noEmit: true, noResolve: true }, host2); +const checker2 = program2.getTypeChecker(); +const src2 = program2.getSourceFile(modulePath); +const used = new Set(); +const walk = (n) => { + if ( + ts.isIdentifier(n) && + !(n.parent && ts.isImportSpecifier(n.parent)) && + !(n.parent && ts.isImportClause(n.parent)) && + !(n.parent && ts.isNamespaceImport(n.parent)) + ) { + const sym = checker2.getSymbolAtLocation(n); + const d = sym?.declarations?.[0]; + if (d && (ts.isImportSpecifier(d) || ts.isImportClause(d) || ts.isNamespaceImport(d))) + used.add(d.name.text); + } + ts.forEachChild(n, walk); +}; +for (const st of src2.statements) if (!ts.isImportDeclaration(st)) walk(st); +const importEdits = []; +for (const st of src2.statements) { + if (!ts.isImportDeclaration(st) || !st.importClause) continue; + const c = st.importClause; + const def = c.name && used.has(c.name.text) ? c.name.text : null; + const nb = c.namedBindings; + const ns = nb && ts.isNamespaceImport(nb) && used.has(nb.name.text) ? nb.name.text : null; + const named = + nb && ts.isNamedImports(nb) ? nb.elements.filter((el) => used.has(el.name.text)) : []; + const all = + (c.name ? def !== null : true) && + (!nb || (ts.isNamespaceImport(nb) ? ns !== null : named.length === nb.elements.length)); + if (all) continue; + let r = ""; + if (def || ns || named.length) { + const bits = []; + if (def) bits.push(def); + if (ns) bits.push(`* as ${ns}`); + if (named.length) bits.push(`{ ${named.map((el) => el.getText()).join(", ")} }`); + r = `import ${c.isTypeOnly ? "type " : ""}${bits.join(", ")} from ${st.moduleSpecifier.getText()};`; + } + const end = r === "" && out[st.end] === "\n" ? st.end + 1 : st.end; + importEdits.push([st.getStart(), end, r]); +} +let final = ""; +cursor = 0; +for (const [a, b, r] of importEdits.sort((x, y) => x[0] - y[0])) { + final += out.slice(cursor, a) + r; + cursor = b; +} +final += out.slice(cursor); +const lastImport = [ + ...ts.createSourceFile(modulePath, final, ts.ScriptTarget.Latest, true).statements, +] + .filter(ts.isImportDeclaration) + .at(-1); +const add = [...moduleNeeds].map( + ([spec, names]) => `import { ${[...names].sort().join(", ")} } from "${spec}";`, +); +const at = lastImport ? lastImport.end : 0; +final = final.slice(0, at) + (add.length ? "\n" + add.join("\n") : "") + final.slice(at); +writeFileSync(modulePath, final); +// Format what changed with the tools checkout's oxfmt, so the split and +// every rehome of it are formatted the same way. +const touched = [ + modulePath, + ...[...files].filter(([, e]) => e.decls.length).map(([f]) => join(moduleDir, f)), +]; +execFileSync(join(toolsDir, "node_modules/.bin/oxfmt"), touched, { + cwd: repo, + stdio: ["ignore", "ignore", "inherit"], +}); +console.log(`${moved} methods moved; ${publicize.size} members reached through host`); diff --git a/scripts/split/split-module.mjs b/scripts/split/split-module.mjs new file mode 100644 index 000000000..5f7cbf549 --- /dev/null +++ b/scripts/split/split-module.mjs @@ -0,0 +1,539 @@ +#!/usr/bin/env node +/* Split a large TypeScript module into files by a manifest of top-level + * declaration names, as a pure move. + * + * node split-module.mjs plan compute the manifest from seeds + * node split-module.mjs apply extract branches, move declarations + * + * : + * { "tsconfig": "packages/compiler/tsconfig.json", + * "module": "packages/compiler/src/frontend/lowering/lower-exprs.ts", + * "extract": [{ "function": "lowerExprInner", "condition": "ts.isIdentifier(expr)", + * "name": "lowerIdentifierExpr", "exprType": "ts.Identifier" }, ...], + * "seeds": { "expressions/binary.ts": ["lowerBinary", ...], ... }, + * "manifest": "" } + * + * It runs on the checkout in the current directory, or --repo , and + * loads typescript from that checkout or --tools . + * + * extract turns `if () { ...block... }` at the top level of the + * named function into `if () return (lowerer, expr, loc);` + * and adds `function (lowerer: Lowerer, expr: , loc: SrcLoc): + * IrExpr { ...block... }` after it. The block must end in a call that never + * returns, so no path falls through. + * + * plan assigns each seed to its file, then moves every other top-level + * declaration whose referrers all sit in one file (other than the module) + * into that file, to a fixpoint. Module-level `let` bindings stay put (an + * importer cannot assign them). The manifest lists file → names. + * + * apply moves the manifest's declarations with their leading comments, + * in source order, into the new files. Each new file imports what its + * declarations reference: names from the module or another new file, and + * the module's own imports with their paths rebased. The module imports the + * moved names it still uses, re-exports the moved names that were exported, + * and drops imports nothing uses anymore. Moved declarations that something + * outside their file references get `export`. Names a manifest lists but + * the module lacks are skipped, and new declarations stay in the module, + * so the same manifest applies to an edited copy of the module. */ +import { readFileSync, writeFileSync, mkdirSync, existsSync } from "node:fs"; +import { dirname, join, relative, resolve, posix } from "node:path"; +import { createRequire } from "node:module"; +import { execFileSync } from "node:child_process"; + +const [command, configPath, ...flags] = process.argv.slice(2); +const config = JSON.parse(readFileSync(configPath, "utf8")); +const flag = (name) => { + const i = flags.indexOf(name); + return i >= 0 ? flags[i + 1] : undefined; +}; +// --repo runs on another checkout (a temporary worktree); --tools names the +// checkout whose node_modules supply typescript. +const repo = resolve(flag("--repo") ?? config.repo ?? process.cwd()); +// The manifest path is relative to the config file. +config.manifest = resolve(dirname(resolve(configPath)), flag("--manifest") ?? config.manifest); +const toolsDir = resolve(flag("--tools") ?? config.repo ?? process.cwd()); +const require = createRequire(join(toolsDir, "package.json")); +const ts = require("typescript"); +const modulePath = join(repo, config.module); +const moduleDir = dirname(modulePath); + +function parse(text, path = modulePath) { + return ts.createSourceFile(path, text, ts.ScriptTarget.Latest, true); +} + +/* ── extract ─────────────────────────────────────────────────────────── */ + +function extractBranches(text) { + for (const spec of config.extract ?? []) { + const sf = parse(text); + const fn = sf.statements.find( + (s) => ts.isFunctionDeclaration(s) && s.name?.text === spec.function, + ); + if (!fn) throw new Error(`no function ${spec.function}`); + const branch = fn.body.statements.find( + (s) => + ts.isIfStatement(s) && + s.expression.getText() === spec.condition && + ts.isBlock(s.thenStatement) && + !s.elseStatement, + ); + if (!branch) { + console.error(`skip: no \`if (${spec.condition}) { ... }\` in ${spec.function}`); + continue; + } + const block = branch.thenStatement; + const last = block.statements.at(-1); + if ( + !last || + !(ts.isExpressionStatement(last) || ts.isReturnStatement(last) || ts.isThrowStatement(last)) + ) + throw new Error( + `${spec.condition}: the block must end in a never-returning call, return or throw`, + ); + // The block's inner text, dedented one level (two spaces). + const inner = text.slice(block.statements[0].getFullStart(), block.statements.at(-1).end); + const body = inner + .split("\n") + .map((line) => (line.startsWith(" ") ? line.slice(2) : line)) + .join("\n") + .replace(/^\n+/, ""); + const call = `if (${spec.condition}) return ${spec.name}(lowerer, expr, loc);`; + const fnText = `\n\nfunction ${spec.name}(lowerer: Lowerer, expr: ${spec.exprType}, loc: SrcLoc): IrExpr {\n ${body.trimStart()}\n}`; + text = + text.slice(0, branch.getStart()) + + call + + text.slice(branch.end, fn.end) + + fnText + + text.slice(fn.end); + } + return text; +} + +/* ── analysis ────────────────────────────────────────────────────────── */ + +function declNames(st) { + if ( + ts.isFunctionDeclaration(st) || + ts.isClassDeclaration(st) || + ts.isInterfaceDeclaration(st) || + ts.isTypeAliasDeclaration(st) || + ts.isEnumDeclaration(st) + ) + return st.name ? [st.name.text] : []; + if (ts.isVariableStatement(st)) + return st.declarationList.declarations.flatMap((d) => + ts.isIdentifier(d.name) ? [d.name.text] : [], + ); + return []; +} +const isExported = (st) => + st.modifiers?.some((m) => m.kind === ts.SyntaxKind.ExportKeyword) ?? false; +const isTypeDecl = (st) => ts.isInterfaceDeclaration(st) || ts.isTypeAliasDeclaration(st); +const isMutable = (st) => + ts.isVariableStatement(st) && !(st.declarationList.flags & ts.NodeFlags.Const); + +function inTypePosition(node) { + for (let n = node; n.parent; n = n.parent) { + if (ts.isTypeNode(n) || ts.isTypeQueryNode?.(n)) return true; + if (ts.isExpressionWithTypeArguments(n) && n.parent && ts.isHeritageClause(n.parent)) + return false; + if (ts.isStatement(n) || (ts.isExpression(n) && !ts.isIdentifier(n))) break; + } + return false; +} + +/** For the module text: its statements with names, its import bindings, and + * each declaration's references (local names and import bindings), each + * marked type-only when every reference sits in a type position. */ +function analyze(text) { + const sf = parse(text); + const host = ts.createCompilerHost({}); + const tsconfig = ts.getParsedCommandLineOfConfigFile( + join(repo, config.tsconfig), + {}, + { ...ts.sys, onUnRecoverableConfigFileDiagnostic: () => {} }, + ); + const options = { ...tsconfig.options, noEmit: true }; + const realGet = host.getSourceFile; + host.getSourceFile = (fileName, lang, ...rest) => + resolve(fileName) === modulePath ? sf : realGet.call(host, fileName, lang, ...rest); + const program = ts.createProgram([modulePath], options, host); + const checker = program.getTypeChecker(); + const source = program.getSourceFile(modulePath); + + const imports = new Map(); // binding name -> { decl, spec, kind, typeOnly } + const statements = []; + for (const st of source.statements) { + if (ts.isImportDeclaration(st)) { + const clause = st.importClause; + const typeOnlyDecl = clause?.isTypeOnly ?? false; + if (clause?.name) + imports.set(clause.name.text, { + decl: st, + kind: "default", + typeOnly: typeOnlyDecl, + node: clause, + }); + const nb = clause?.namedBindings; + if (nb && ts.isNamespaceImport(nb)) + imports.set(nb.name.text, { + decl: st, + kind: "namespace", + typeOnly: typeOnlyDecl, + node: nb, + }); + if (nb && ts.isNamedImports(nb)) + for (const el of nb.elements) + imports.set(el.name.text, { + decl: st, + kind: "named", + typeOnly: typeOnlyDecl || el.isTypeOnly, + node: el, + imported: (el.propertyName ?? el.name).text, + }); + continue; + } + statements.push({ node: st, names: declNames(st) }); + } + const byNode = new Map(); + for (const s of statements) byNode.set(s.node, s); + const ownerOf = (decl) => { + for (let n = decl; n; n = n.parent) if (byNode.has(n)) return byNode.get(n); + return undefined; + }; + for (const s of statements) { + const refs = new Map(); // name -> { kind: "local"|"import", typeOnly } + const visit = (node) => { + if (ts.isIdentifier(node)) { + const parent = node.parent; + const isDeclName = + parent && + parent.name === node && + (ts.isFunctionDeclaration(parent) || + ts.isVariableDeclaration(parent) || + ts.isInterfaceDeclaration(parent) || + ts.isTypeAliasDeclaration(parent) || + ts.isParameter(parent) || + ts.isPropertyAssignment(parent) || + ts.isPropertySignature(parent) || + ts.isPropertyDeclaration(parent) || + ts.isMethodDeclaration(parent) || + ts.isBindingElement(parent)); + const isPropertyName = + parent && + ((ts.isPropertyAccessExpression(parent) && parent.name === node) || + (ts.isQualifiedName(parent) && parent.right === node)); + if (!isPropertyName) { + let sym = checker.getSymbolAtLocation(node); + if (ts.isShorthandPropertyAssignment(parent)) + sym = checker.getShorthandAssignmentValueSymbol(parent); + const decl = sym?.declarations?.[0]; + if (decl && decl.getSourceFile() === source) { + const typePos = inTypePosition(node); + let target; + if ( + ts.isImportSpecifier(decl) || + ts.isNamespaceImport(decl) || + ts.isImportClause(decl) + ) { + const name = (ts.isImportClause(decl) ? decl.name : decl.name).text; + target = { name, kind: "import" }; + } else { + const owner = ownerOf(decl); + if (owner && owner !== s && !isDeclName) { + const name = owner.names.find((n) => n === node.text) ?? node.text; + if (owner.names.includes(name)) target = { name, kind: "local" }; + } + } + if (target) { + const prev = refs.get(target.name); + refs.set(target.name, { + kind: target.kind, + typeOnly: (prev ? prev.typeOnly : true) && typePos, + }); + } + } + } + } + ts.forEachChild(node, visit); + }; + visit(s.node); + s.refs = refs; + } + return { source, statements, imports }; +} + +/* ── plan ────────────────────────────────────────────────────────────── */ + +function plan() { + const text = extractBranches(readFileSync(modulePath, "utf8")); + const { statements } = analyze(text); + const fileOf = new Map(); + for (const [file, names] of Object.entries(config.seeds)) + for (const name of names) { + if (!statements.some((s) => s.names.includes(name))) + throw new Error(`seed ${name} not found`); + fileOf.set(name, file); + } + const keep = new Set(config.keep ?? []); + const referrers = new Map(); + for (const s of statements) + for (const [name, ref] of s.refs) + if (ref.kind === "local") { + if (!referrers.has(name)) referrers.set(name, new Set()); + for (const n of s.names) referrers.get(name).add(n); + } + for (let changed = true; changed;) { + changed = false; + for (const s of statements) { + if ( + s.names.length === 0 || + s.names.some((n) => fileOf.has(n) || keep.has(n)) || + isMutable(s.node) + ) + continue; + const users = new Set(); + for (const n of s.names) + for (const u of referrers.get(n) ?? []) if (!s.names.includes(u)) users.add(u); + const files = new Set([...users].map((u) => fileOf.get(u) ?? null)); + if (users.size > 0 && files.size === 1 && !files.has(null)) { + const [file] = files; + for (const n of s.names) fileOf.set(n, file); + changed = true; + } + } + } + const manifest = {}; + for (const s of statements) { + const file = s.names.map((n) => fileOf.get(n)).find(Boolean); + if (file) (manifest[file] ??= []).push(...s.names); + } + writeFileSync(config.manifest, JSON.stringify(manifest, null, 2) + "\n"); + for (const [file, names] of Object.entries(manifest)) console.log(`${file}: ${names.length}`); +} + +/* ── apply ───────────────────────────────────────────────────────────── */ + +function rebase(spec, fromDir, toDir) { + if (!spec.startsWith(".")) return spec; + let rel = posix.normalize(relative(toDir, resolve(fromDir, spec)).split("\\").join("/")); + if (!rel.startsWith(".")) rel = "./" + rel; + return rel; +} +const moduleSpecifierTo = (fromDir, file) => { + let rel = relative(fromDir, file).split("\\").join("/").replace(/\.ts$/, ".js"); + return rel.startsWith(".") ? rel : "./" + rel; +}; + +/** The statement's text with its leading comments, without leading blank lines. */ +function fullText(text, st) { + return text.slice(st.getFullStart(), st.end).replace(/^\s*\n/, ""); +} + +function apply() { + const manifest = JSON.parse(readFileSync(config.manifest, "utf8")); + let text = extractBranches(readFileSync(modulePath, "utf8")); + const { source, statements, imports } = analyze(text); + const fileOf = new Map(); + for (const [file, names] of Object.entries(manifest)) for (const n of names) fileOf.set(n, file); + const stFile = (s) => s.names.map((n) => fileOf.get(n)).find(Boolean) ?? null; + const nameFile = new Map(); + const nameStmt = new Map(); + for (const s of statements) + for (const n of s.names) { + nameFile.set(n, stFile(s)); + nameStmt.set(n, s); + } + + // Which names need `export`: referenced from another file, or exported already. + const needsExport = new Set(); + for (const s of statements) + for (const [name, ref] of s.refs) + if (ref.kind === "local" && nameFile.get(name) !== stFile(s)) needsExport.add(name); + + const files = new Map(); // file -> { decls: [], imports: Map } + const fileImports = (file) => { + if (!files.has(file)) files.set(file, { decls: [], named: new Map(), other: [] }); + return files.get(file); + }; + const addNamed = (entry, spec, name, imported, typeOnly) => { + if (!entry.named.has(spec)) entry.named.set(spec, new Map()); + const m = entry.named.get(spec); + const prev = m.get(name); + m.set(name, { imported, typeOnly: prev ? prev.typeOnly && typeOnly : typeOnly }); + }; + const moduleEntry = { decls: [], named: new Map(), other: [] }; + for (const s of statements) { + const file = stFile(s); + const entry = file ? fileImports(file) : moduleEntry; + const dir = file ? dirname(join(moduleDir, file)) : moduleDir; + if (file) { + let t = fullText(text, s.node); + if (!isExported(s.node) && s.names.some((n) => needsExport.has(n))) { + const start = s.node.getStart() - s.node.getFullStart(); + const lead = text.slice(s.node.getFullStart(), s.node.getStart()).replace(/^\s*\n/, ""); + t = lead + "export " + text.slice(s.node.getStart(), s.node.end); + void start; + } + entry.decls.push(t); + } + for (const [name, ref] of s.refs) { + if (ref.kind === "local") { + const target = nameFile.get(name) ?? null; + if (target === file) continue; + const targetPath = target ? join(moduleDir, target) : modulePath; + const typeOnly = ref.typeOnly || isTypeDecl(nameStmt.get(name).node); + addNamed(entry, moduleSpecifierTo(dir, targetPath), name, name, typeOnly); + } else if (file) { + const imp = imports.get(name); + const spec = rebase(imp.decl.moduleSpecifier.text, moduleDir, dir); + if (imp.kind === "named") addNamed(entry, spec, name, imp.imported, imp.typeOnly); + else entry.other.push({ spec, kind: imp.kind, name, typeOnly: imp.typeOnly }); + } + } + } + + const renderImports = (entry) => { + const lines = []; + const seen = new Set(); + for (const o of entry.other) { + const key = `${o.kind}:${o.name}:${o.spec}`; + if (seen.has(key)) continue; + seen.add(key); + lines.push( + o.kind === "namespace" + ? `import ${o.typeOnly ? "type " : ""}* as ${o.name} from "${o.spec}";` + : `import ${o.typeOnly ? "type " : ""}${o.name} from "${o.spec}";`, + ); + } + for (const [spec, names] of entry.named) { + const all = [...names.entries()]; + const allType = all.every(([, v]) => v.typeOnly); + const parts = all + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + .map( + ([name, v]) => + `${!allType && v.typeOnly ? "type " : ""}${v.imported === name ? name : `${v.imported} as ${name}`}`, + ); + lines.push(`import ${allType ? "type " : ""}{ ${parts.join(", ")} } from "${spec}";`); + } + return lines; + }; + + // New files. + for (const [file, entry] of files) { + const header = config.headers?.[file]; + const out = [ + ...(header ? [header] : []), + ...renderImports(entry), + "", + entry.decls.join("\n\n"), + "", + ].join("\n"); + const path = join(moduleDir, file); + mkdirSync(dirname(path), { recursive: true }); + if (existsSync(path)) throw new Error(`${file} exists`); + writeFileSync(path, out); + } + + // The module: drop moved declarations, rewrite its import block. + const keptRefs = new Set(); + for (const s of statements) + if (!stFile(s)) for (const [name, ref] of s.refs) if (ref.kind === "import") keptRefs.add(name); + const pieces = []; + let cursor = 0; + const removals = []; + for (const s of statements) if (stFile(s)) removals.push([s.node.getFullStart(), s.node.end]); + // Import declarations: rebuild each without bindings nothing uses. + const importEdits = []; + for (const st of source.statements) { + if (!ts.isImportDeclaration(st)) continue; + const clause = st.importClause; + if (!clause) continue; // side-effect import + const used = (n) => keptRefs.has(n); + const def = clause.name && used(clause.name.text) ? clause.name.text : null; + const nb = clause.namedBindings; + let ns = null; + let named = []; + if (nb && ts.isNamespaceImport(nb) && used(nb.name.text)) ns = nb.name.text; + if (nb && ts.isNamedImports(nb)) named = nb.elements.filter((el) => used(el.name.text)); + const keepAll = + def === (clause.name?.text ?? null) && + (!nb || (ts.isNamespaceImport(nb) ? ns !== null : named.length === nb.elements.length)); + if (keepAll) continue; + let replacement = ""; + if (def || ns || named.length) { + const bits = []; + if (def) bits.push(def); + if (ns) bits.push(`* as ${ns}`); + if (named.length) bits.push(`{ ${named.map((el) => el.getText()).join(", ")} }`); + replacement = `import ${clause.isTypeOnly ? "type " : ""}${bits.join(", ")} from ${st.moduleSpecifier.getText()};`; + } + // A removed declaration takes its line break with it. + const end = replacement === "" && text[st.end] === "\n" ? st.end + 1 : st.end; + importEdits.push([st.getStart(), end, replacement]); + } + // Declarations that stay but that a moved one references get `export`. + const exportEdits = []; + for (const s of statements) + if (!stFile(s) && !isExported(s.node) && s.names.some((n) => needsExport.has(n))) + exportEdits.push([s.node.getStart(), s.node.getStart(), "export "]); + const edits = [...removals.map(([a, b]) => [a, b, ""]), ...importEdits, ...exportEdits].sort( + (x, y) => x[0] - y[0], + ); + for (const [a, b, r] of edits) { + pieces.push(text.slice(cursor, a), r); + cursor = b; + } + pieces.push(text.slice(cursor)); + let moduleText = pieces.join(""); + // Imports of moved names the module still uses, and re-exports of moved + // names that were exported (other modules import them from here). + const addLines = renderImports(moduleEntry); + const reexports = new Map(); + for (const s of statements) { + const file = stFile(s); + if (!file || !isExported(s.node)) continue; + const spec = moduleSpecifierTo(moduleDir, join(moduleDir, file)); + if (!reexports.has(spec)) reexports.set(spec, []); + for (const n of s.names) reexports.get(spec).push({ name: n, type: isTypeDecl(s.node) }); + } + for (const [spec, names] of reexports) { + const values = names + .filter((n) => !n.type) + .map((n) => n.name) + .sort(); + const types = names + .filter((n) => n.type) + .map((n) => n.name) + .sort(); + if (values.length) addLines.push(`export { ${values.join(", ")} } from "${spec}";`); + if (types.length) addLines.push(`export type { ${types.join(", ")} } from "${spec}";`); + } + // Insert after the last import declaration. + const reparsed = parse(moduleText); + const lastImport = [...reparsed.statements].filter(ts.isImportDeclaration).at(-1); + const at = lastImport ? lastImport.end : 0; + moduleText = + moduleText.slice(0, at) + + (addLines.length ? "\n" + addLines.join("\n") : "") + + moduleText.slice(at); + moduleText = moduleText.replace(/\n{3,}/g, "\n\n"); + writeFileSync(modulePath, moduleText); + // Format what changed with the tools checkout's oxfmt, so the split and + // every rehome of it are formatted the same way. + const touched = [modulePath, ...[...files.keys()].map((f) => join(moduleDir, f))]; + execFileSync(join(toolsDir, "node_modules/.bin/oxfmt"), touched, { + cwd: repo, + stdio: ["ignore", "ignore", "inherit"], + }); + for (const [file, entry] of files) console.log(`${file}: ${entry.decls.length} declarations`); +} + +if (command === "plan") plan(); +else if (command === "apply") apply(); +else { + console.error("usage: split-module.mjs plan|apply "); + process.exit(2); +} From 71e3dc5b80fc1af370961605b636edf3b76c235f Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Wed, 21 Oct 2026 03:21:11 -0500 Subject: [PATCH 002/197] Keep shorthand-property imports in split methods and treat extends as a value use in split modules --- scripts/split/split-methods.mjs | 5 +- scripts/split/split-module.mjs | 6 +- tests/harness/split-tools.test.ts | 121 ++++++++++++++++++++++++++++++ 3 files changed, 129 insertions(+), 3 deletions(-) create mode 100644 tests/harness/split-tools.test.ts diff --git a/scripts/split/split-methods.mjs b/scripts/split/split-methods.mjs index 66d6b36ad..0b4f9a161 100644 --- a/scripts/split/split-methods.mjs +++ b/scripts/split/split-methods.mjs @@ -351,7 +351,10 @@ const walk = (n) => { !(n.parent && ts.isImportClause(n.parent)) && !(n.parent && ts.isNamespaceImport(n.parent)) ) { - const sym = checker2.getSymbolAtLocation(n); + const sym = + n.parent && ts.isShorthandPropertyAssignment(n.parent) + ? checker2.getShorthandAssignmentValueSymbol(n.parent) + : checker2.getSymbolAtLocation(n); const d = sym?.declarations?.[0]; if (d && (ts.isImportSpecifier(d) || ts.isImportClause(d) || ts.isNamespaceImport(d))) used.add(d.name.text); diff --git a/scripts/split/split-module.mjs b/scripts/split/split-module.mjs index 5f7cbf549..aa20baaf4 100644 --- a/scripts/split/split-module.mjs +++ b/scripts/split/split-module.mjs @@ -135,9 +135,11 @@ const isMutable = (st) => function inTypePosition(node) { for (let n = node; n.parent; n = n.parent) { + // A class's `extends` names a value; `implements` and an interface's + // `extends` name types. + if (ts.isExpressionWithTypeArguments(n) && ts.isHeritageClause(n.parent)) + return !(n.parent.token === ts.SyntaxKind.ExtendsKeyword && ts.isClassLike(n.parent.parent)); if (ts.isTypeNode(n) || ts.isTypeQueryNode?.(n)) return true; - if (ts.isExpressionWithTypeArguments(n) && n.parent && ts.isHeritageClause(n.parent)) - return false; if (ts.isStatement(n) || (ts.isExpression(n) && !ts.isIdentifier(n))) break; } return false; diff --git a/tests/harness/split-tools.test.ts b/tests/harness/split-tools.test.ts new file mode 100644 index 000000000..659015cc6 --- /dev/null +++ b/tests/harness/split-tools.test.ts @@ -0,0 +1,121 @@ +import { execFileSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import ts from "typescript"; +import { afterEach, expect, test } from "vitest"; + +// The repository root supplies typescript and oxfmt to the split tools. +const root = fileURLToPath(new URL("../..", import.meta.url)); +const tool = (name: string): string => join(root, "scripts/split", name); +const dirs: string[] = []; + +afterEach(() => { + for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); +}); + +const compilerOptions = { + strict: true, + noEmit: true, + target: "ES2022", + module: "ESNext", + moduleResolution: "Bundler", +}; + +/** A checkout holding `files` (paths relative to its root) and a tsconfig. */ +function checkout(files: Record): string { + const dir = mkdtempSync(join(tmpdir(), "split-tools-")); + dirs.push(dir); + const all = { ...files, "tsconfig.json": JSON.stringify({ compilerOptions }) }; + for (const [path, text] of Object.entries(all)) { + mkdirSync(dirname(join(dir, path)), { recursive: true }); + writeFileSync(join(dir, path), text); + } + return dir; +} + +function run(script: string, args: string[], repo: string): void { + execFileSync(process.execPath, [tool(script), ...args, "--repo", repo, "--tools", root], { + cwd: repo, + stdio: "pipe", + }); +} + +/** Type errors in the checkout's files. */ +function diagnostics(dir: string): string[] { + const parsed = ts.getParsedCommandLineOfConfigFile( + join(dir, "tsconfig.json"), + {}, + { + ...ts.sys, + onUnRecoverableConfigFileDiagnostic: () => {}, + }, + ); + const program = ts.createProgram(parsed!.fileNames, parsed!.options); + return ts + .getPreEmitDiagnostics(program) + .map((d) => ts.flattenDiagnosticMessageText(d.messageText, "\n")); +} + +test("split-methods keeps an import a staying method uses as a shorthand property", () => { + const dir = checkout({ + "src/helper.ts": "export const helper = 2;\n", + "src/host.ts": [ + 'import { helper } from "./helper.js";', + "", + "export class Host {", + " value = 1;", + " keep(): { helper: number } {", + " return { helper };", + " }", + " moved(): number {", + " return this.value + 1;", + " }", + "}", + "", + ].join("\n"), + "split.json": JSON.stringify({ + module: "src/host.ts", + class: "Host", + methods: { "moved.ts": ["moved"] }, + }), + }); + run("split-methods.mjs", ["apply", join(dir, "split.json")], dir); + expect(readFileSync(join(dir, "src/host.ts"), "utf8")).toContain( + 'import { helper } from "./helper.js";', + ); + expect(diagnostics(dir)).toEqual([]); +}); + +test("split-module imports a moved class's local base class as a value", () => { + const dir = checkout({ + "src/shapes.ts": [ + "export class Base {", + " x = 1;", + "}", + "", + "export class Derived extends Base {", + " y = 2;", + "}", + "", + "export function base(): Base {", + " return new Base();", + "}", + "", + ].join("\n"), + "split.json": JSON.stringify({ + tsconfig: "tsconfig.json", + module: "src/shapes.ts", + seeds: { "derived.ts": ["Derived"] }, + manifest: "manifest.json", + }), + }); + const config = join(dir, "split.json"); + run("split-module.mjs", ["plan", config], dir); + run("split-module.mjs", ["apply", config], dir); + expect(readFileSync(join(dir, "src/derived.ts"), "utf8")).toContain( + 'import { Base } from "./shapes.js";', + ); + expect(diagnostics(dir)).toEqual([]); +}); From d244b84931d4f706b31f81c90b6cac8fbb61ef02 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Wed, 21 Oct 2026 03:47:21 -0500 Subject: [PATCH 003/197] Keep the split scripts' reads and writes inside the worktree --- scripts/split/README.md | 2 +- scripts/split/confine.mjs | 89 +++++++++++++++++ scripts/split/move-state.mjs | 29 +++--- scripts/split/rehome-split.mjs | 21 ++-- scripts/split/split-methods.mjs | 17 ++-- scripts/split/split-module.mjs | 31 +++--- .../split-tools/symlink/outside/linked.ts | 2 + .../split-tools/symlink/repo/src/host.ts | 6 ++ .../split-tools/symlink/repo/src/linked.ts | 1 + tests/harness/split-tools.test.ts | 98 ++++++++++++++++++- 10 files changed, 255 insertions(+), 41 deletions(-) create mode 100644 scripts/split/confine.mjs create mode 100644 tests/fixtures/split-tools/symlink/outside/linked.ts create mode 100644 tests/fixtures/split-tools/symlink/repo/src/host.ts create mode 120000 tests/fixtures/split-tools/symlink/repo/src/linked.ts diff --git a/scripts/split/README.md b/scripts/split/README.md index 7d79b35c1..e3cdfd077 100644 --- a/scripts/split/README.md +++ b/scripts/split/README.md @@ -9,7 +9,7 @@ Run them from a checkout with `node_modules` installed. That checkout is the too - `split-methods.mjs apply ` moves class methods into exported functions that take the instance as a parameter (`emitter`). The class keeps a one-line stub with the same signature, so callers don't change. - `move-state.mjs apply ` moves a group of class fields into a feature's state object, defined in the feature's file with `defineModuleState` (`packages/compiler/src/backend/llvm/emitter-state.ts`). It rewrites `this.` and `emitter.` to `KEY.of(this).` and creates files from templates. It also applies the change's other edits as literal replacements, in order. A replacement whose text isn't found the expected number of times is reported and skipped. -Each transform takes `--repo ` (the tree to rewrite) and `--tools `, and formats what it writes with oxfmt. `configs/` holds the configs of splits that have landed. For example, `configs/loop-state/` moves the loop optimizations' state out of `LlEmitter`. +Each transform takes `--repo ` (the tree to rewrite) and `--tools `, and formats what it writes with oxfmt. The tools read and write the tree's files only at real paths inside `--repo`, and a config's templates and manifest only inside the config's directory. A path that resolves elsewhere, through a symlink or `..`, stops the tool with an error (`confine.mjs`). `configs/` holds the configs of splits that have landed. For example, `configs/loop-state/` moves the loop optimizations' state out of `LlEmitter`. ## Rehoming an open branch ```sh diff --git a/scripts/split/confine.mjs b/scripts/split/confine.mjs new file mode 100644 index 000000000..14af8b85a --- /dev/null +++ b/scripts/split/confine.mjs @@ -0,0 +1,89 @@ +/* Path confinement for the split tools. A branch they rewrite can commit + * symlinks, so every file a tool reads or writes goes through a guard for + * the directory it belongs to (the checkout being rewritten, or the + * config's directory). The guard resolves the path's real location, through + * every symlink on the way, and throws when that location is outside the + * directory's real path. A path that doesn't exist yet resolves through its + * nearest existing parent, so a write can't create a file through a + * symlinked directory or a dangling symlink either. */ +import { + existsSync, + lstatSync, + mkdirSync, + readdirSync, + readFileSync, + readlinkSync, + realpathSync, + writeFileSync, +} from "node:fs"; +import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; + +const isInside = (root, path) => { + const rel = relative(root, path); + return rel === "" || (rel !== ".." && !rel.startsWith(`..${sep}`) && !isAbsolute(rel)); +}; + +/** The real location of `path`, which need not exist: the real path of its + * nearest existing entry joined with the rest. A dangling symlink resolves + * to where its target would be. */ +function realLocation(path, hops = 0) { + const rest = []; + for (let at = resolve(path); ; at = dirname(at)) { + let stat = null; + try { + stat = lstatSync(at); + } catch (e) { + if (e.code !== "ENOENT" && e.code !== "ENOTDIR") throw e; + } + if (stat) { + try { + return join(realpathSync(at), ...rest); + } catch (e) { + if (e.code !== "ENOENT" || !stat.isSymbolicLink() || hops > 40) throw e; + return realLocation(join(resolve(dirname(at), readlinkSync(at)), ...rest), hops + 1); + } + } + if (dirname(at) === at) return resolve(path); + rest.unshift(basename(at)); + } +} + +/** A guard for the files under `root`. */ +export function confine(root) { + const lexical = resolve(root); + const real = realpathSync(lexical); + const check = (path) => { + const location = realLocation(path); + if (!isInside(real, location)) + throw new Error(`refusing ${path}: it resolves to ${location}, outside ${real}`); + return location; + }; + return { + /** The path's real location; throws when it is outside the root. */ + check, + /** True when `path` names a file under the root (before symlinks). */ + contains: (path) => isInside(lexical, resolve(path)) || isInside(real, resolve(path)), + read: (path) => readFileSync(check(path), "utf8"), + write: (path, text) => writeFileSync(check(path), text), + exists: (path) => existsSync(check(path)), + mkdir: (path) => mkdirSync(check(path), { recursive: true }), + list: (path) => readdirSync(check(path)), + }; +} + +/** Routes a TypeScript host's (or `ts.sys`-like object's) file reads under + * the guard's root through the guard; other files (the TypeScript libraries) + * read as before. */ +export function confineHost(host, guard) { + const checked = + (read) => + (fileName, ...rest) => { + if (guard.contains(fileName)) guard.check(fileName); + return read.call(host, fileName, ...rest); + }; + return { + ...host, + ...(host.getSourceFile ? { getSourceFile: checked(host.getSourceFile) } : {}), + readFile: checked(host.readFile), + }; +} diff --git a/scripts/split/move-state.mjs b/scripts/split/move-state.mjs index 5da787ac2..b70023c67 100644 --- a/scripts/split/move-state.mjs +++ b/scripts/split/move-state.mjs @@ -23,10 +23,11 @@ * rewritten file imports the key. * The group's file must already exist and hold everything else (header, * setup function); the tool appends the interface and key to it. */ -import { existsSync, readFileSync, writeFileSync, readdirSync } from "node:fs"; +import { readFileSync } from "node:fs"; import { dirname, join, resolve } from "node:path"; import { createRequire } from "node:module"; import { execFileSync } from "node:child_process"; +import { confine } from "./confine.mjs"; const [command, configPath, ...flags] = process.argv.slice(2); if (command !== "apply") throw new Error("usage: move-state.mjs apply "); @@ -41,6 +42,9 @@ const require = createRequire(join(toolsDir, "package.json")); const ts = require("typescript"); const modulePath = join(repo, config.module); const dir = join(repo, config.dir); +// Every read and write stays inside the checkout, or for the templates, the +// config's directory (confine.mjs). +const checkout = confine(repo); // Adds an import line after the file's first block of imports (up to a // blank line or other code). @@ -56,6 +60,7 @@ const addImport = (text, line) => { return text.slice(0, at) + line + "\n" + text.slice(at); }; const configDir = dirname(resolve(configPath)); +const configFiles = confine(configDir); // The names the class's instance goes by in the directory's sources. const receivers = config.receivers ?? ["this", "emitter"]; const touched = new Set(); @@ -63,11 +68,11 @@ const touched = new Set(); // branch already has the file). for (const [file, template] of Object.entries(config.create ?? {})) { const path = join(dir, file); - if (existsSync(path)) continue; - writeFileSync(path, readFileSync(join(configDir, template), "utf8")); + if (checkout.exists(path)) continue; + checkout.write(path, configFiles.read(join(configDir, template))); touched.add(path); } -let moduleText = readFileSync(modulePath, "utf8"); +let moduleText = checkout.read(modulePath); for (const group of config.groups) { const sf = ts.createSourceFile(modulePath, moduleText, ts.ScriptTarget.Latest, true); const cls = sf.statements.find((s) => ts.isClassDeclaration(s) && s.name?.text === config.class); @@ -124,22 +129,22 @@ for (const group of config.groups) { `export interface ${group.interfaceName ?? iName} {\n${iface.join("\n")}\n}\n\n` + `export const ${group.key} = defineModuleState<${group.interfaceName ?? iName}>(() => ({\n${factory.join("\n")}\n}));\n`; const groupPath = join(dir, group.file); - let groupText = readFileSync(groupPath, "utf8"); + let groupText = checkout.read(groupPath); if (!groupText.includes('from "./emitter-state.js"')) groupText = addImport(groupText, `import { defineModuleState } from "./emitter-state.js";`); groupText = groupText.trimEnd() + "\n" + stateText; - writeFileSync(groupPath, groupText); + checkout.write(groupPath, groupText); touched.add(groupPath); group.rename = group.present.length ? new RegExp(`(? f.endsWith(".ts"))) { +for (const file of checkout.list(dir).filter((f) => f.endsWith(".ts"))) { const path = join(dir, file); - let text = path === modulePath ? moduleText : readFileSync(path, "utf8"); + let text = path === modulePath ? moduleText : checkout.read(path); let changed = false; for (const group of config.groups) { if (!group.rename) continue; @@ -171,7 +176,7 @@ for (const file of readdirSync(dir).filter((f) => f.endsWith(".ts"))) { } } if (changed) { - writeFileSync(path, text); + checkout.write(path, text); touched.add(path); } } @@ -182,7 +187,7 @@ for (const file of readdirSync(dir).filter((f) => f.endsWith(".ts"))) { let skipped = 0; for (const r of config.replacements ?? []) { const path = join(dir, r.file); - const text = existsSync(path) ? readFileSync(path, "utf8") : ""; + const text = checkout.exists(path) ? checkout.read(path) : ""; const count = text.split(r.find).length - 1; if (count !== (r.count ?? 1)) { console.warn( @@ -191,7 +196,7 @@ for (const r of config.replacements ?? []) { skipped++; continue; } - writeFileSync(path, text.split(r.find).join(r.replace)); + checkout.write(path, text.split(r.find).join(r.replace)); touched.add(path); } const oxfmt = join(toolsDir, "node_modules/.bin/oxfmt"); diff --git a/scripts/split/rehome-split.mjs b/scripts/split/rehome-split.mjs index 2884558fb..89c32d6b7 100644 --- a/scripts/split/rehome-split.mjs +++ b/scripts/split/rehome-split.mjs @@ -24,10 +24,11 @@ * instead. When the cherry-pick conflicts, the script stops and * leaves the work tree with the conflicts to resolve; see the message. */ import { execFileSync, spawnSync } from "node:child_process"; -import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; import { createRequire } from "node:module"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import { confine } from "./confine.mjs"; const args = process.argv.slice(2); const opt = (name, fallback) => { @@ -157,7 +158,9 @@ try { : Object.keys( (({ files, ...rest }) => files ?? rest)( JSON.parse( - readFileSync(resolve(dirname(resolve(configPath)), config.manifest), "utf8"), + confine(dirname(resolve(configPath))).read( + resolve(dirname(resolve(configPath)), config.manifest), + ), ), ), ); @@ -166,7 +169,10 @@ try { for (const file of gitIn(resultDir, "diff", "--name-only", "--diff-filter=U") .split("\n") .filter(Boolean)) { - if (splitFiles.has(file) && resolveImportConflicts(join(resultDir, file))) { + if ( + splitFiles.has(file) && + resolveImportConflicts(confine(resultDir), join(resultDir, file)) + ) { execFileSync(join(tools, "node_modules/.bin/oxfmt"), [file], { cwd: resultDir, stdio: ["ignore", "ignore", "inherit"], @@ -212,11 +218,12 @@ try { /* Rewrites every conflict in `path` whose two sides hold only import * declarations as the union of both: one declaration per module and kind, - * with the named imports of both sides in first-seen order. Returns false, + * with the named imports of both sides in first-seen order, read and + * written through the result checkout's guard. Returns false, * leaving the file unchanged, when any conflict holds other code. */ -function resolveImportConflicts(path) { +function resolveImportConflicts(checkout, path) { const ts = createRequire(join(tools, "package.json"))("typescript"); - const text = readFileSync(path, "utf8"); + const text = checkout.read(path); const region = /^<<<<<<< [^\n]*\n([\s\S]*?)^=======\n([\s\S]*?)^>>>>>>> [^\n]*\n/gm; const imports = (side) => { const sf = ts.createSourceFile("side.ts", side, ts.ScriptTarget.Latest, true); @@ -266,7 +273,7 @@ function resolveImportConflicts(path) { ); }); if (!ok || resolved === text) return false; - writeFileSync(path, pruneUnusedImports(ts, resolved, imported)); + checkout.write(path, pruneUnusedImports(ts, resolved, imported)); return true; } diff --git a/scripts/split/split-methods.mjs b/scripts/split/split-methods.mjs index 0b4f9a161..05d72c4be 100644 --- a/scripts/split/split-methods.mjs +++ b/scripts/split/split-methods.mjs @@ -20,10 +20,11 @@ * reference from the module's scope, with paths rebased; the module imports * the functions and drops imports nothing uses anymore. Names the config * lists but the class lacks are skipped. */ -import { readFileSync, writeFileSync, existsSync } from "node:fs"; +import { readFileSync } from "node:fs"; import { dirname, join, relative, resolve, posix } from "node:path"; import { createRequire } from "node:module"; import { execFileSync } from "node:child_process"; +import { confine, confineHost } from "./confine.mjs"; const [command, configPath, ...flags] = process.argv.slice(2); if (command !== "apply") throw new Error("usage: split-methods.mjs apply "); @@ -33,6 +34,8 @@ const flag = (name) => { return i >= 0 ? flags[i + 1] : undefined; }; const repo = resolve(flag("--repo") ?? process.cwd()); +// Every read and write in the checkout stays inside it (confine.mjs). +const checkout = confine(repo); const toolsDir = resolve(flag("--tools") ?? process.cwd()); const require = createRequire(join(toolsDir, "package.json")); const ts = require("typescript"); @@ -42,8 +45,8 @@ const className = config.class; // The host parameter's name; a moved body that already uses it is refused. const hostName = config.hostName ?? "emitter"; -const text = readFileSync(modulePath, "utf8"); -const host = ts.createCompilerHost({ noEmit: true }); +const text = checkout.read(modulePath); +const host = confineHost(ts.createCompilerHost({ noEmit: true }), checkout); const sf = ts.createSourceFile(modulePath, text, ts.ScriptTarget.Latest, true); const realGet = host.getSourceFile; host.getSourceFile = (f, l, ...r) => @@ -294,9 +297,9 @@ const renderImports = (entry) => { for (const [file, entry] of files) { if (!entry.decls.length) continue; const path = join(moduleDir, file); - if (existsSync(path)) throw new Error(`${file} exists`); + if (checkout.exists(path)) throw new Error(`${file} exists`); const header = config.headers?.[file]; - writeFileSync( + checkout.write( path, [...(header ? [header] : []), ...renderImports(entry), "", entry.decls.join("\n\n"), ""].join( "\n", @@ -336,7 +339,7 @@ out += text.slice(cursor); // Re-analyze the rewritten module for unused imports, then add the new ones. const sf2 = ts.createSourceFile(modulePath, out, ts.ScriptTarget.Latest, true); -const host2 = ts.createCompilerHost({ noEmit: true }); +const host2 = confineHost(ts.createCompilerHost({ noEmit: true }), checkout); const get2 = host2.getSourceFile; host2.getSourceFile = (f, l, ...r) => resolve(f) === modulePath ? sf2 : get2.call(host2, f, l, ...r); @@ -403,7 +406,7 @@ const add = [...moduleNeeds].map( ); const at = lastImport ? lastImport.end : 0; final = final.slice(0, at) + (add.length ? "\n" + add.join("\n") : "") + final.slice(at); -writeFileSync(modulePath, final); +checkout.write(modulePath, final); // Format what changed with the tools checkout's oxfmt, so the split and // every rehome of it are formatted the same way. const touched = [ diff --git a/scripts/split/split-module.mjs b/scripts/split/split-module.mjs index aa20baaf4..ba340df40 100644 --- a/scripts/split/split-module.mjs +++ b/scripts/split/split-module.mjs @@ -36,10 +36,11 @@ * outside their file references get `export`. Names a manifest lists but * the module lacks are skipped, and new declarations stay in the module, * so the same manifest applies to an edited copy of the module. */ -import { readFileSync, writeFileSync, mkdirSync, existsSync } from "node:fs"; +import { readFileSync } from "node:fs"; import { dirname, join, relative, resolve, posix } from "node:path"; import { createRequire } from "node:module"; import { execFileSync } from "node:child_process"; +import { confine, confineHost } from "./confine.mjs"; const [command, configPath, ...flags] = process.argv.slice(2); const config = JSON.parse(readFileSync(configPath, "utf8")); @@ -50,6 +51,10 @@ const flag = (name) => { // --repo runs on another checkout (a temporary worktree); --tools names the // checkout whose node_modules supply typescript. const repo = resolve(flag("--repo") ?? config.repo ?? process.cwd()); +// Every read and write stays inside the checkout, or for the manifest, the +// config's directory (confine.mjs). +const checkout = confine(repo); +const configFiles = confine(dirname(resolve(configPath))); // The manifest path is relative to the config file. config.manifest = resolve(dirname(resolve(configPath)), flag("--manifest") ?? config.manifest); const toolsDir = resolve(flag("--tools") ?? config.repo ?? process.cwd()); @@ -150,11 +155,13 @@ function inTypePosition(node) { * marked type-only when every reference sits in a type position. */ function analyze(text) { const sf = parse(text); - const host = ts.createCompilerHost({}); + const host = confineHost(ts.createCompilerHost({}), checkout); + const tsconfigPath = join(repo, config.tsconfig); + checkout.check(tsconfigPath); const tsconfig = ts.getParsedCommandLineOfConfigFile( - join(repo, config.tsconfig), + tsconfigPath, {}, - { ...ts.sys, onUnRecoverableConfigFileDiagnostic: () => {} }, + { ...confineHost(ts.sys, checkout), onUnRecoverableConfigFileDiagnostic: () => {} }, ); const options = { ...tsconfig.options, noEmit: true }; const realGet = host.getSourceFile; @@ -269,7 +276,7 @@ function analyze(text) { /* ── plan ────────────────────────────────────────────────────────────── */ function plan() { - const text = extractBranches(readFileSync(modulePath, "utf8")); + const text = extractBranches(checkout.read(modulePath)); const { statements } = analyze(text); const fileOf = new Map(); for (const [file, names] of Object.entries(config.seeds)) @@ -311,7 +318,7 @@ function plan() { const file = s.names.map((n) => fileOf.get(n)).find(Boolean); if (file) (manifest[file] ??= []).push(...s.names); } - writeFileSync(config.manifest, JSON.stringify(manifest, null, 2) + "\n"); + configFiles.write(config.manifest, JSON.stringify(manifest, null, 2) + "\n"); for (const [file, names] of Object.entries(manifest)) console.log(`${file}: ${names.length}`); } @@ -334,8 +341,8 @@ function fullText(text, st) { } function apply() { - const manifest = JSON.parse(readFileSync(config.manifest, "utf8")); - let text = extractBranches(readFileSync(modulePath, "utf8")); + const manifest = JSON.parse(configFiles.read(config.manifest)); + let text = extractBranches(checkout.read(modulePath)); const { source, statements, imports } = analyze(text); const fileOf = new Map(); for (const [file, names] of Object.entries(manifest)) for (const n of names) fileOf.set(n, file); @@ -434,9 +441,9 @@ function apply() { "", ].join("\n"); const path = join(moduleDir, file); - mkdirSync(dirname(path), { recursive: true }); - if (existsSync(path)) throw new Error(`${file} exists`); - writeFileSync(path, out); + checkout.mkdir(dirname(path)); + if (checkout.exists(path)) throw new Error(`${file} exists`); + checkout.write(path, out); } // The module: drop moved declarations, rewrite its import block. @@ -522,7 +529,7 @@ function apply() { (addLines.length ? "\n" + addLines.join("\n") : "") + moduleText.slice(at); moduleText = moduleText.replace(/\n{3,}/g, "\n\n"); - writeFileSync(modulePath, moduleText); + checkout.write(modulePath, moduleText); // Format what changed with the tools checkout's oxfmt, so the split and // every rehome of it are formatted the same way. const touched = [modulePath, ...[...files.keys()].map((f) => join(moduleDir, f))]; diff --git a/tests/fixtures/split-tools/symlink/outside/linked.ts b/tests/fixtures/split-tools/symlink/outside/linked.ts new file mode 100644 index 000000000..7e551bf2e --- /dev/null +++ b/tests/fixtures/split-tools/symlink/outside/linked.ts @@ -0,0 +1,2 @@ +// A file outside the checkout that the split tools must not read or write. +export const outside = 1; diff --git a/tests/fixtures/split-tools/symlink/repo/src/host.ts b/tests/fixtures/split-tools/symlink/repo/src/host.ts new file mode 100644 index 000000000..4bf39c941 --- /dev/null +++ b/tests/fixtures/split-tools/symlink/repo/src/host.ts @@ -0,0 +1,6 @@ +export class Host { + count = 0; + moved(): number { + return this.count + 1; + } +} diff --git a/tests/fixtures/split-tools/symlink/repo/src/linked.ts b/tests/fixtures/split-tools/symlink/repo/src/linked.ts new file mode 120000 index 000000000..3a9e39fbe --- /dev/null +++ b/tests/fixtures/split-tools/symlink/repo/src/linked.ts @@ -0,0 +1 @@ +../../outside/linked.ts \ No newline at end of file diff --git a/tests/harness/split-tools.test.ts b/tests/harness/split-tools.test.ts index 659015cc6..778ef152b 100644 --- a/tests/harness/split-tools.test.ts +++ b/tests/harness/split-tools.test.ts @@ -1,5 +1,14 @@ -import { execFileSync } from "node:child_process"; -import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { execFileSync, spawnSync } from "node:child_process"; +import { + cpSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -119,3 +128,88 @@ test("split-module imports a moved class's local base class as a value", () => { ); expect(diagnostics(dir)).toEqual([]); }); + +/* A copy of tests/fixtures/split-tools/symlink: `repo/` is the checkout the + * tools rewrite, and its committed `src/linked.ts` links to `outside/linked.ts` + * next to it. */ +const symlinkFixture = fileURLToPath(new URL("../fixtures/split-tools/symlink", import.meta.url)); +const outsideText = readFileSync(join(symlinkFixture, "outside/linked.ts"), "utf8"); + +function symlinkCheckout(): { dir: string; repo: string } { + const dir = mkdtempSync(join(tmpdir(), "split-tools-")); + dirs.push(dir); + cpSync(symlinkFixture, dir, { recursive: true, verbatimSymlinks: true }); + return { dir, repo: join(dir, "repo") }; +} + +/** Runs a tool that must refuse; answers its stderr. */ +function refused(script: string, config: object, dir: string, repo: string): string { + const path = join(dir, "split.json"); + writeFileSync(path, JSON.stringify(config)); + const result = spawnSync( + process.execPath, + [tool(script), "apply", path, "--repo", repo, "--tools", root], + { cwd: repo, encoding: "utf8" }, + ); + expect(result.status).not.toBe(0); + return result.stderr; +} + +test("split-methods refuses to write a new file through a symlink that leaves the checkout", () => { + const { dir, repo } = symlinkCheckout(); + // A link to a file that doesn't exist yet: writing through it would create the file. + symlinkSync("../../outside/created.ts", join(repo, "src/moved.ts")); + const stderr = refused( + "split-methods.mjs", + { module: "src/host.ts", class: "Host", methods: { "moved.ts": ["moved"] } }, + dir, + repo, + ); + expect(stderr).toContain("refusing"); + expect(existsSync(join(dir, "outside/created.ts"))).toBe(false); +}); + +test("split-methods refuses a module that links outside the checkout", () => { + const { dir, repo } = symlinkCheckout(); + const stderr = refused( + "split-methods.mjs", + { module: "src/linked.ts", class: "Host", methods: { "moved.ts": ["moved"] } }, + dir, + repo, + ); + expect(stderr).toContain("refusing"); +}); + +test("split-module refuses a module that links outside the checkout", () => { + const { dir, repo } = symlinkCheckout(); + writeFileSync(join(repo, "tsconfig.json"), JSON.stringify({ compilerOptions })); + writeFileSync(join(dir, "manifest.json"), JSON.stringify({ "moved.ts": ["outside"] })); + const stderr = refused( + "split-module.mjs", + { tsconfig: "tsconfig.json", module: "src/linked.ts", manifest: "manifest.json" }, + dir, + repo, + ); + expect(stderr).toContain("refusing"); +}); + +test("move-state refuses to rewrite a file that links outside the checkout", () => { + const { dir, repo } = symlinkCheckout(); + writeFileSync(join(repo, "src/host-state.ts"), ""); + const stderr = refused( + "move-state.mjs", + { + module: "src/host.ts", + class: "Host", + dir: "src", + groups: [ + { file: "host-state.ts", key: "COUNTS", interfaceName: "Counts", fields: ["count"] }, + ], + replacements: [{ file: "linked.ts", find: "export const outside = 1;", replace: "" }], + }, + dir, + repo, + ); + expect(stderr).toContain("refusing"); + expect(readFileSync(join(dir, "outside/linked.ts"), "utf8")).toBe(outsideText); +}); From 054cb50ea4381566b4438a0e5b44448b426e4c04 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 02:31:20 -0500 Subject: [PATCH 004/197] Array keys and properties: non-index number keys, Object.keys, console.log --- .../compiler/src/backend/llvm/lib-shared.ts | 1 + .../src/backend/llvm/reference-writes.ts | 1 + .../compiler/src/contracts/int32-elements.ts | 12 ++- .../src/frontend/lowering/lower-calls.ts | 13 ++- .../src/frontend/lowering/lower-inspect.ts | 14 +++- .../compiler/src/ir/builtin-signatures.ts | 4 + packages/compiler/src/ir/validate.ts | 6 +- .../corpus/7060-array-numeric-keys.cjs.json | 6 ++ .../corpus/7061-array-object-keys.ts.json | 6 ++ .../7062-inspect-array-properties.ts.json | 6 ++ packages/runtime/src/scr_array.c | 46 +++++++---- packages/runtime/src/scr_array.h | 4 + packages/runtime/src/scr_dyn.h | 3 + packages/runtime/src/scr_inspect.c | 35 +++++++- packages/runtime/src/scr_json.c | 79 +++++++++++++++++++ .../6987-array-view-named-properties.ts | 10 --- tests/corpus/7060-array-numeric-keys.cjs | 50 ++++++++++++ tests/corpus/7061-array-object-keys.ts | 28 +++++++ tests/corpus/7062-inspect-array-properties.ts | 54 +++++++++++++ .../7060-array-numeric-keys.cjs.json | 1 + .../7061-array-object-keys.ts.json | 1 + .../7062-inspect-array-properties.ts.json | 1 + tests/harness/errors.test.ts | 22 ++++++ 23 files changed, 367 insertions(+), 36 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7060-array-numeric-keys.cjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7061-array-object-keys.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7062-inspect-array-properties.ts.json create mode 100644 tests/corpus/7060-array-numeric-keys.cjs create mode 100644 tests/corpus/7061-array-object-keys.ts create mode 100644 tests/corpus/7062-inspect-array-properties.ts create mode 100644 tests/harness/corpus-costs/7060-array-numeric-keys.cjs.json create mode 100644 tests/harness/corpus-costs/7061-array-object-keys.ts.json create mode 100644 tests/harness/corpus-costs/7062-inspect-array-properties.ts.json diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index 8bb8bc3aa..80e0b0044 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -1522,6 +1522,7 @@ export const LIB_FN_SYMS: Record = { "regex.resultProp": "scr_regex_result_prop", "array.namedKeys": "scr_arr_named_keys", "array.hasNamedKey": "scr_arr_has_named_key", + "array.hasProperties": "scr_arr_has_properties", // queueMicrotask's checked-dynamic form (borrowed dyn; a non-function // throws synchronously), the minted setImmediate value, and the // timers/promises immediate — all mark the loop live. diff --git a/packages/compiler/src/backend/llvm/reference-writes.ts b/packages/compiler/src/backend/llvm/reference-writes.ts index d31f14d12..d040e8c93 100644 --- a/packages/compiler/src/backend/llvm/reference-writes.ts +++ b/packages/compiler/src/backend/llvm/reference-writes.ts @@ -107,6 +107,7 @@ const INERT_LIB_CALLS = new Set([ "regex.resultProp", "array.namedKeys", "array.hasNamedKey", + "array.hasProperties", "process.cwd", "process.argv", "process.execArgv", diff --git a/packages/compiler/src/contracts/int32-elements.ts b/packages/compiler/src/contracts/int32-elements.ts index 8284d40ab..ca61a8e49 100644 --- a/packages/compiler/src/contracts/int32-elements.ts +++ b/packages/compiler/src/contracts/int32-elements.ts @@ -122,7 +122,8 @@ const COPIES = new Set([ const COPY_IN = new Set(["pushSpread", "unshiftSpread", "spliceInsert"]); /** Library calls that only read array arguments. */ -const READ_ONLY_LIB = /^(console\.|inspect\.|json\.stringify|math\.|insp\.)/; +const READ_ONLY_LIB = + /^(console\.|inspect\.|json\.stringify|math\.|insp\.|array\.(hasProperties|namedKeys|hasNamedKey)$)/; /** Library calls that always throw (the value they stand for never exists). */ const THROWING_LIB = /^error\.\w*Throw$/; @@ -872,6 +873,9 @@ export class Int32ElementContracts { }; const isNumberArray = (t: IrType): boolean => t.kind === "array" && t.elem.kind === "f64"; + // Live views handed straight to a read-only library call (util.inspect + // of an array with properties) are only read through. + const readOnlyViews = new Set(); const expr = (e: IrExpr): void => { switch (e.kind) { case "assignExpr": @@ -1000,6 +1004,7 @@ export class Int32ElementContracts { case "libCall": if (!READ_ONLY_LIB.test(e.fn)) for (const a of e.args) escape(a, a.loc, `the library call ${e.fn}`); + else for (const a of e.args) if (a.kind === "dynFrom") readOnlyViews.add(a); break; case "callValue": for (const a of e.args) escape(a, a.loc, "a call through a function value"); @@ -1008,7 +1013,10 @@ export class Int32ElementContracts { case "dynFrom": // Plain arrays and records cross into dynamic code as deep copies; // class instances and live references stay connected. - if (e.liveRef === true || this.containsObject(e.value.type, new Set())) + if ( + !readOnlyViews.has(e) && + (e.liveRef === true || this.containsObject(e.value.type, new Set())) + ) escape(e.value, e.loc, "dynamic (any or unknown) code"); break; default: diff --git a/packages/compiler/src/frontend/lowering/lower-calls.ts b/packages/compiler/src/frontend/lowering/lower-calls.ts index 7102f37ab..c6e3d0c1e 100644 --- a/packages/compiler/src/frontend/lowering/lower-calls.ts +++ b/packages/compiler/src/frontend/lowering/lower-calls.ts @@ -8354,11 +8354,15 @@ function lowerObjectStaticCall( probed?.type.kind === "union" && isJsSourceFile(ts.getSourceFileOfNode(call)) && lowerer.dynConvertible(probed.type); + // An array walks its live view: the present indices, then its numeric + // and named properties. + const isArray = probed?.type.kind === "array" && lowerer.dynConvertible(probed.type); if ( isDyn || isPrimitive || isFunction || isJsUnion || + isArray || (probed && isDynTypedRefType(probed.type)) ) { const fn = @@ -8368,7 +8372,14 @@ function lowerObjectStaticCall( ? "dyn.objValues" : "dyn.objEntries"; let v = lowerer.lowerExpr(argNode); - if (v.type.kind !== "dyn") v = { kind: "dynFrom", value: v, type: DYN, loc: locOf(call) }; + if (v.type.kind !== "dyn") + v = { + kind: "dynFrom", + value: v, + ...(isArray ? { liveRef: true as const } : {}), + type: DYN, + loc: locOf(call), + }; // In TypeScript the keys come back as a native string[] and the // entries as a native [string, any][]: the checker keeps every later // write to the arrays well typed. JavaScript may store anything into diff --git a/packages/compiler/src/frontend/lowering/lower-inspect.ts b/packages/compiler/src/frontend/lowering/lower-inspect.ts index 4be5d0b7e..14c67f70a 100644 --- a/packages/compiler/src/frontend/lowering/lower-inspect.ts +++ b/packages/compiler/src/frontend/lowering/lower-inspect.ts @@ -1912,13 +1912,21 @@ function inspectHelper(lowerer: Lowerer, t: IrType, loc: SrcLoc): string { // A closed record that kept the undeclared properties of its checked // source renders through its live checked view, which holds them in the - // source's own-key order beside the current field values. + // source's own-key order beside the current field values. An array with + // properties besides its elements does too: the view lists them after + // the elements. let prependSourceGate: (() => void) | null = null; - if (t.kind === "record" && recordMayKeepSource(lowerer, t)) { + const sourceTest: IrExpr | null = + t.kind === "record" && recordMayKeepSource(lowerer, t) + ? recordSourceTest(v(), loc) + : t.kind === "array" && lowerer.dynConvertible(t) + ? { kind: "libCall", fn: "array.hasProperties", args: [v()], type: BOOL, loc } + : null; + if (sourceTest !== null) { prependSourceGate = (): void => { body.unshift({ kind: "if", - cond: recordSourceTest(v(), loc), + cond: sourceTest, then: [ ret({ kind: "libCall", diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index fa7876579..054b16201 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -3738,6 +3738,10 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ // Whether the array (args[0]) still has the key (args[1]) from its // namedKeys list. Borrowed; never throws. "array.hasNamedKey": { argTypes: [null, STRING], result: BOOL }, + // Whether an array (args[0], any element type) has properties besides its + // elements: noncanonical numeric keys, match metadata or named + // properties. Borrowed; never throws. + "array.hasProperties": { argTypes: [null], result: BOOL }, // node:events EventEmitter: receivers are emitter-hierarchy objects and // the chaining forms (on/off/removeAll/setMax) return the receiver's // own class — program-dependent object types, checked in the libCall diff --git a/packages/compiler/src/ir/validate.ts b/packages/compiler/src/ir/validate.ts index 88ccbbc33..bdfe75f2e 100644 --- a/packages/compiler/src/ir/validate.ts +++ b/packages/compiler/src/ir/validate.ts @@ -4262,7 +4262,11 @@ function validateFunction( if (!ok) err(`libCall dyn.objEntryPairs must return a [string, any] tuple array`, e.loc); return true; } - if (e.fn === "array.namedKeys" || e.fn === "array.hasNamedKey") { + if ( + e.fn === "array.namedKeys" || + e.fn === "array.hasNamedKey" || + e.fn === "array.hasProperties" + ) { if (e.args[0]?.type.kind !== "array") { err(`libCall ${e.fn} arg 0: expected an array, got ${e.args[0]?.type.kind}`, e.loc); } diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7060-array-numeric-keys.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7060-array-numeric-keys.cjs.json new file mode 100644 index 000000000..5fff9651a --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7060-array-numeric-keys.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7060-array-numeric-keys.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7061-array-object-keys.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7061-array-object-keys.ts.json new file mode 100644 index 000000000..b2830e983 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7061-array-object-keys.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7061-array-object-keys.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7062-inspect-array-properties.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7062-inspect-array-properties.ts.json new file mode 100644 index 000000000..891c4ffe0 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7062-inspect-array-properties.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7062-inspect-array-properties.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_array.c b/packages/runtime/src/scr_array.c index c00d1b4f6..3d7da0c37 100644 --- a/packages/runtime/src/scr_array.c +++ b/packages/runtime/src/scr_array.c @@ -4825,6 +4825,10 @@ ScrStr *scr_str_raw(ScrArr *raw, ScrArr *subs) { bool scr_arr_numeric_key(const char *key, size_t len, double *out) { char text[64]; if (len == 0 || len >= sizeof text) return false; + /* A number's spelling starts with a digit, '-', "NaN" or "Infinity": + * other names answer without parsing. */ + char first = key[0]; + if (!(first >= '0' && first <= '9') && first != '-' && first != 'N' && first != 'I') return false; memcpy(text, key, len); text[len] = '\0'; char *end; @@ -4838,6 +4842,11 @@ bool scr_arr_numeric_key(const char *key, size_t len, double *out) { return true; } +bool scr_arr_has_properties(const ScrArr *a) { + const ScrArrExt *x = scr_arr_ext_of(a); + return x && (x->prop_len || x->metadata); +} + void scr_arr_copy_metadata(ScrArr *a, ScrDyn *target) { const ScrDyn *metadata = SCR_AX(a)->metadata; if (metadata && metadata->kind == SCR_DYN_HANDLE && metadata->v.handle.tag == SCR_DYNH_MATCH_SOURCE && @@ -4845,21 +4854,7 @@ void scr_arr_copy_metadata(ScrArr *a, ScrDyn *target) { return; metadata = SCR_AX(a)->metadata; if (!metadata) return; - for (size_t i = 0; i < metadata->v.obj.len; i++) { - const ScrDynEntry *entry = &metadata->v.obj.entries[i]; - ScrStr *key = scr_str_new(entry->key, entry->key_len); - if (!entry->accessor && entry->writable && entry->enumerable && entry->configurable) { - scr_dyn_key_set(target, key, entry->value); - } else { - /* Properties defined through a view keep their attributes. */ - ScrDyn *name = scr_dyn_new_str(key); - ScrDyn *descriptor = scr_dyn_own_descriptor(metadata, key); - if (descriptor) scr_dyn_release(scr_dyn_define_property(target, name, descriptor)); - scr_dyn_release(descriptor); - scr_dyn_release(name); - } - scr_str_release(key); - } + for (size_t i = 0; i < metadata->v.obj.len; i++) scr_dyn_copy_entry(target, metadata, &metadata->v.obj.entries[i]); } ScrArr *scr_arr_named_keys(ScrArr *a) { @@ -4917,8 +4912,27 @@ void scr_arr_adopt_view_properties(ScrArr *a, ScrDyn *view, ScrArr *target) { } ScrDyn *named = view && view->kind == SCR_DYN_ARR ? view->v.arr.properties : NULL; if (!named || named->v.obj.len == 0) return; + /* The view's numeric properties are copies of the native ones (every + * write to them goes to the native array), so only the others move. */ + bool numeric = false; + for (size_t i = 0; i < named->v.obj.len && !numeric; i++) { + double number; + numeric = scr_arr_numeric_key(named->v.obj.entries[i].key, named->v.obj.entries[i].key_len, &number); + } + if (numeric) { + ScrDyn *rest = scr_dyn_new_obj(); + for (size_t i = 0; i < named->v.obj.len; i++) { + const ScrDynEntry *entry = &named->v.obj.entries[i]; + double number; + if (!scr_arr_numeric_key(entry->key, entry->key_len, &number)) scr_dyn_copy_entry(rest, named, entry); + } + named = rest; + } else { + scr_dyn_retain(named); + } ScrArrExt *x = scr_arr_ext(a); ScrDyn *old = x->metadata; - x->metadata = scr_dyn_retain(named); + x->metadata = named->v.obj.len ? named : NULL; + if (!x->metadata) scr_dyn_release(named); scr_dyn_release(old); } diff --git a/packages/runtime/src/scr_array.h b/packages/runtime/src/scr_array.h index 6c41b3dbd..d08b94c14 100644 --- a/packages/runtime/src/scr_array.h +++ b/packages/runtime/src/scr_array.h @@ -333,6 +333,10 @@ bool scr_arr_has_named_key(ScrArr *a, ScrStr *key); * noncanonical numeric properties, then the named properties in insertion * order. +1. */ ScrArr *scr_arr_named_keys(ScrArr *a); +/* Whether the array has properties besides its elements (noncanonical + * numeric keys, match metadata, named properties): util.inspect lists + * them after the elements. */ +bool scr_arr_has_properties(const ScrArr *a); /* Installed by scr_regex.c when it records a pending match source; this * unit links without the regex runtime. Writes the pending metadata of * `a` into its dynamic copy or materializes it, answering whether it diff --git a/packages/runtime/src/scr_dyn.h b/packages/runtime/src/scr_dyn.h index 02c4d828f..8974fc7ce 100644 --- a/packages/runtime/src/scr_dyn.h +++ b/packages/runtime/src/scr_dyn.h @@ -561,6 +561,9 @@ static inline const ScrDynTypedArrayOps *scr_dyn_typed_ref_array_ops(const ScrDy ScrDyn *scr_dyn_obj_get(const ScrDyn *d, const char *key, size_t key_len); /* Own-enumerable check on a SCR_DYN_OBJ. */ bool scr_dyn_obj_enumerable(const ScrDyn *d, const char *key, size_t key_len); +/* Stores own property `entry` of `source` on `target` with its attributes: + * plain data properties by an ordinary write, others by their descriptor. */ +void scr_dyn_copy_entry(ScrDyn *target, const ScrDyn *source, const ScrDynEntry *entry); /* Literal discriminator tests used before selecting a typed record layout. * Missing fields and different primitive kinds never match. Borrowed args. */ bool scr_dyn_field_eq_str(const ScrDyn *d, const ScrStr *key, const ScrStr *value); diff --git a/packages/runtime/src/scr_inspect.c b/packages/runtime/src/scr_inspect.c index 223517e92..7a1685a02 100644 --- a/packages/runtime/src/scr_inspect.c +++ b/packages/runtime/src/scr_inspect.c @@ -1499,14 +1499,20 @@ ScrStr *scr_insp_dyn(ScrDyn *d, double recurse, double depth) { return scr_insp_str(d->v.str); case SCR_DYN_ARR: { size_t len = d->v.arr.len; - if (len == 0) return scr_str_new("[]", 2); + /* Named and noncanonical numeric properties follow the elements as + * `key: value` entries (Node's array extras). */ + const ScrDyn *props = d->v.arr.properties; + size_t nprops = 0; + for (size_t i = 0; props && i < props->v.obj.len; i++) nprops += props->v.obj.entries[i].enumerable; + if (len == 0 && nprops == 0) return scr_str_new("[]", 2); if (recurse > depth) return scr_str_new("[Array]", 7); scr_insp_begin(recurse + 1); /* Node's maxArrayLength (100) counts output entries; a run of holes * is one `` entry (formatSpecialArray). */ const unsigned char *presence = d->v.arr.presence; size_t i = 0; - for (size_t shown = 0; i < len && shown < 100; shown++) { + size_t shown = 0; + for (; i < len && shown < 100; shown++) { if (presence && !presence[i]) { size_t next = i + 1; while (next < len && !presence[next]) next++; @@ -1524,15 +1530,38 @@ ScrStr *scr_insp_dyn(ScrDyn *d, double recurse, double depth) { scr_str_release(s); } bool more = i < len; + size_t entries = shown + (more ? 1 : 0); if (more) { ScrStr *m = scr_insp_more_items((double)(len - i)); scr_insp_entry(m, d->v.arr.items[i]->kind == SCR_DYN_NUM); scr_str_release(m); } + for (size_t k = 0; props && k < props->v.obj.len; k++) { + const ScrDynEntry *ent = &props->v.obj.entries[k]; + if (!ent->enumerable) continue; + ScrStr *val = scr_insp_dyn(ent->value, recurse + 1, depth); + InspBuf eb = {0}; + insp_key_into(&eb, ent->key, ent->key_len); + ib_cstr(&eb, ": "); + ib_bytes(&eb, val->data, val->len); + scr_str_release(val); + ScrStr *entry = ib_take(&eb); + /* Node's grouping pads by the array element at the entry's output + * position (value[i] in groupArrayElements), not by the entry. */ + bool number = entries < len && (!presence || presence[entries]) && + d->v.arr.items[entries]->kind == SCR_DYN_NUM; + scr_insp_entry(entry, number); + scr_str_release(entry); + entries++; + } + /* Node's grouping leaves out the last entry when there are more + * entries than maxArrayLength (the `... more items` entry when the + * array has no properties). */ + bool last_out = entries > 100; ScrStr *base = scr_str_new("", 0); ScrStr *b0 = scr_str_new("[", 1); ScrStr *b1 = scr_str_new("]", 1); - ScrStr *out = scr_insp_end(base, b0, b1, recurse + 1, true, more); + ScrStr *out = scr_insp_end(base, b0, b1, recurse + 1, true, last_out); scr_str_release(base); scr_str_release(b0); scr_str_release(b1); diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index b8f16630b..e8b75e5fa 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -3241,6 +3241,43 @@ ScrDyn *scr_dyn_class_view_unavailable(void *ptr) { return scr_dyn_retain(scr_dyn_undefined()); } +void scr_dyn_copy_entry(ScrDyn *target, const ScrDyn *source, const ScrDynEntry *entry) { + ScrStr *key = scr_str_new(entry->key, entry->key_len); + if (!entry->accessor && entry->writable && entry->enumerable && entry->configurable) { + scr_dyn_key_set(target, key, entry->value); + } else { + /* Properties defined through a view keep their attributes. */ + ScrDyn *name = scr_dyn_new_str(key); + ScrDyn *descriptor = scr_dyn_own_descriptor(source, key); + if (descriptor) scr_dyn_release(scr_dyn_define_property(target, name, descriptor)); + scr_dyn_release(descriptor); + scr_dyn_release(name); + } + scr_str_release(key); +} + +static ScrDyn *scr_typed_ref_numeric_property(const ScrDyn *d, double key); + +/* A live array's view lists the native numeric properties first, then the + * named properties the snapshot copied from the array's metadata. */ +static void scr_typed_ref_view_numeric_properties(const ScrDyn *capsule, ScrDyn *view) { + const ScrDynTypedArrayOps *ops = scr_dyn_typed_ref_array_ops(capsule); + ScrArr *arr = capsule->v.typed_ref.ptr; + const ScrArrExt *x = ops ? scr_arr_ext_of(arr) : NULL; + if (!x || x->prop_len == 0 || view->kind != SCR_DYN_ARR) return; + ScrDyn *named = view->v.arr.properties; + view->v.arr.properties = NULL; + for (size_t i = 0; i < x->prop_len; i++) { + double number; + if (!scr_arr_numeric_key(x->props[i].key, x->props[i].key_len, &number)) continue; + ScrDyn *value = scr_typed_ref_numeric_property(capsule, number); + if (value) scr_dyn_arr_named_put(view, x->props[i].key, x->props[i].key_len, value); + } + if (!named) return; + for (size_t i = 0; i < named->v.obj.len; i++) scr_dyn_copy_entry(view, named, &named->v.obj.entries[i]); + scr_dyn_release(named); +} + ScrDyn *scr_dyn_typed_ref_materialize(const ScrDyn *d) { ScrDyn *capsule = (ScrDyn *)d; ScrDyn *fresh = capsule->v.typed_ref.materialize(capsule->v.typed_ref.ptr); @@ -3248,6 +3285,7 @@ ScrDyn *scr_dyn_typed_ref_materialize(const ScrDyn *d) { scr_dyn_release(fresh); return scr_dyn_retain(scr_dyn_undefined()); } + scr_typed_ref_view_numeric_properties(capsule, fresh); if (!capsule->v.typed_ref.materialized) { capsule->v.typed_ref.materialized = fresh; } else { @@ -3467,6 +3505,16 @@ ScrDyn *scr_dyn_typed_ref_string_view(const ScrDyn *d) { return out; } +/* A numeric property of a live array ("-1", "1.5"): +1, or NULL when the + * native array has none. */ +static ScrDyn *scr_typed_ref_numeric_property(const ScrDyn *d, double key) { + ScrArr *arr = d->v.typed_ref.ptr; + double state = scr_arr_state(arr, key); + if (state == (double)SCR_ARR_VALUE) return scr_dyn_typed_ref_array_ops(d)->get(arr, key); + if (state == (double)SCR_ARR_UNDEFINED) return scr_dyn_retain(scr_dyn_undefined()); + return NULL; +} + ScrDyn *scr_dyn_typed_ref_key_get(const ScrDyn *d, const ScrStr *key) { if (d->v.typed_ref.record) return scr_typed_ref_record_get(d, key); if (scr_event_ref_hooks) { @@ -3475,6 +3523,8 @@ ScrDyn *scr_dyn_typed_ref_key_get(const ScrDyn *d, const ScrStr *key) { } if (!scr_dyn_typed_ref_array_ops(d)) return NULL; if (scr_typed_ref_is_length(key)) return scr_dyn_new_num((double)((ScrArr *)d->v.typed_ref.ptr)->len); + double number; + if (scr_arr_numeric_key(key->data, key->len, &number)) return scr_typed_ref_numeric_property(d, number); size_t index; if (!scr_typed_ref_canonical_index(key->data, key->len, &index)) return NULL; return scr_dyn_typed_ref_element(d, index); @@ -3484,6 +3534,9 @@ ScrDyn *scr_dyn_typed_ref_key_get(const ScrDyn *d, const ScrStr *key) { static bool scr_typed_ref_array_has_own(const ScrDyn *d, const ScrStr *key) { if (!scr_dyn_typed_ref_array_ops(d)) return false; if (scr_typed_ref_is_length(key)) return true; + double number; + if (scr_arr_numeric_key(key->data, key->len, &number)) + return scr_arr_state(d->v.typed_ref.ptr, number) != (double)SCR_ARR_HOLE; size_t index; if (!scr_typed_ref_canonical_index(key->data, key->len, &index)) return false; ScrArr *arr = d->v.typed_ref.ptr; @@ -3496,6 +3549,22 @@ static bool scr_typed_ref_array_has_own(const ScrDyn *d, const ScrStr *key) { static bool scr_typed_ref_array_set(ScrDyn *d, const ScrStr *key, const ScrDyn *value) { const ScrDynTypedArrayOps *ops = scr_dyn_typed_ref_array_ops(d); if (!ops || !ops->set) return false; + /* Numeric properties live in the native array's property table, which + * holds values of the element type. */ + double number; + if (scr_arr_numeric_key(key->data, key->len, &number)) { + if (ops->set(d->v.typed_ref.ptr, number, value)) return true; + ScrStr *kind = scr_dyn_typeof(value); + ScrJsonBuf message; + scr_jb_init(&message); + scr_jb_puts(&message, "expected the array's element type at $["); + scr_jb_write(&message, key->data, key->len); + scr_jb_puts(&message, "], got "); + scr_jb_write(&message, kind->data, kind->len); + scr_str_release(kind); + scr_throw_error(SCR_ERR_TYPE, scr_jb_finish(&message)); + return true; + } size_t index; if (!scr_typed_ref_canonical_index(key->data, key->len, &index)) return false; ScrArr *arr = d->v.typed_ref.ptr; @@ -6341,6 +6410,16 @@ static bool scr_dyn_fn_property_fence(const ScrStr *key) { } ScrDyn *scr_dyn_define_property(ScrDyn *target, ScrDyn *key, ScrDyn *descriptor) { + if (target->kind == SCR_DYN_TYPED_REF && scr_dyn_typed_ref_array_ops(target) && key->kind == SCR_DYN_STR) { + /* A native array's numeric properties ("-1", "1.5") live in its + * property table, which holds values without attributes. */ + double number; + if (scr_arr_numeric_key(key->v.str->data, key->v.str->len, &number)) { + static const char message[] = "Object.defineProperty on a numeric property of a native array is not supported yet"; + scr_throw_error_msg(SCR_ERR_ERROR, message, sizeof message - 1); + return NULL; + } + } if (target->kind == SCR_DYN_TYPED_REF) { ScrDyn *view = scr_dyn_typed_ref_materialize(target); ScrDyn *defined = scr_exc_pending() ? NULL : scr_dyn_define_property(view, key, descriptor); diff --git a/tests/corpus/6987-array-view-named-properties.ts b/tests/corpus/6987-array-view-named-properties.ts index ee357d072..faba7dc90 100644 --- a/tests/corpus/6987-array-view-named-properties.ts +++ b/tests/corpus/6987-array-view-named-properties.ts @@ -102,13 +102,3 @@ for (const key in visited) { console.log("visited key", key); if (key === "first") delete (visited as unknown as Bag).second; } - -// A non-configurable numeric-looking property refuses the delete and stays. -const fixed: string[] = ["a"]; -Object.defineProperty(fixed as unknown as Bag, "-3", { value: "kept", enumerable: true }); -try { - delete (fixed as unknown as Bag)["-3"]; -} catch (e) { - console.log((e as Error).name); -} -console.log((fixed as unknown as Bag)["-3"], "-3" in (fixed as unknown as Bag)); diff --git a/tests/corpus/7060-array-numeric-keys.cjs b/tests/corpus/7060-array-numeric-keys.cjs new file mode 100644 index 000000000..5a1978cd9 --- /dev/null +++ b/tests/corpus/7060-array-numeric-keys.cjs @@ -0,0 +1,50 @@ +// Number keys that are not array indices ("-1", "1.5", "NaN", "Infinity") +// are ordinary properties of an array. Writes and reads through an untyped +// function, through a string key, and through the array itself all reach +// the same property, and key walks list them after the indices. + +function put(target, key, value) { + target[key] = value; +} +function get(target, key) { + return target[key]; +} +function has(target, key) { + return key in target; +} +function drop(target, key) { + delete target[key]; +} + +const empty = []; +empty[-1] = "x"; +empty[1.5] = "y"; +empty[NaN] = "z"; +console.log(empty[-1], empty[1.5], empty[NaN], empty.length); +console.log(empty["-1"], empty["1.5"], empty["NaN"]); + +const words = ["p", "q"]; +put(words, -1, "x"); +put(words, 1.5, "y"); +put(words, NaN, "z"); +put(words, Infinity, "inf"); +console.log(get(words, -1), get(words, 1.5), get(words, NaN), get(words, Infinity)); +console.log(words[-1], words[1.5], words[NaN], words[Infinity], words["-1"], words.length); +console.log(get(words, "-1"), get(words, "1.50"), get(words, "-0"), has(words, "1.5")); +console.log(Object.keys(words), JSON.stringify(words)); +for (const key in words) console.log("key", key); + +words[-1] = "again"; +console.log(get(words, -1), words["-1"]); +put(words, "label", "named"); +console.log(get(words, "label"), get(words, -1)); +drop(words, -1); +console.log(words[-1], has(words, -1), get(words, 1.5)); +console.log(Object.keys(words)); + +const numbers = [1, 2]; +put(numbers, -1, 10); +put(numbers, 2.5, 20); +console.log(numbers[-1], get(numbers, -1), numbers[2.5], numbers.length); +numbers.push(3); +console.log(get(numbers, -1), numbers.join(",")); diff --git a/tests/corpus/7061-array-object-keys.ts b/tests/corpus/7061-array-object-keys.ts new file mode 100644 index 000000000..93691a78f --- /dev/null +++ b/tests/corpus/7061-array-object-keys.ts @@ -0,0 +1,28 @@ +// Object.keys, Object.values and Object.entries of a typed array list the +// present indices, then its numeric properties ("-1", "1.5"), then its +// named properties, as in Node. Holes are skipped. + +type Bag = Record; + +const words: string[] = ["p", "q"]; +words[-1] = "x"; +(words as unknown as Bag)["1.5"] = "y"; +(words as unknown as Bag).label = "named"; +console.log(Object.keys(words), Object.values(words), Object.entries(words)); +console.log((words as unknown as Bag)["-1"], words[1.5], "1.5" in (words as unknown as Bag)); +for (const key of Object.keys(words)) console.log(key, key.length); + +const holes: (number | undefined)[] = [1, , 3, undefined]; +console.log(Object.keys(holes), Object.values(holes), Object.entries(holes)); + +const empty: number[] = []; +console.log(Object.keys(empty).length, Object.entries(empty)); + +interface Point { + x: number; +} +const points: Point[] = [{ x: 1 }, { x: 2 }]; +console.log(Object.keys(points), Object.values(points)); +const keys: string[] = Object.keys(points); +keys.push("extra"); +console.log(keys.join("|"), Object.keys(points).length); diff --git a/tests/corpus/7062-inspect-array-properties.ts b/tests/corpus/7062-inspect-array-properties.ts new file mode 100644 index 000000000..aa18cfca9 --- /dev/null +++ b/tests/corpus/7062-inspect-array-properties.ts @@ -0,0 +1,54 @@ +// console.log of an array lists its named and numeric properties after the +// elements, as Node's util.inspect does: typed arrays written through an +// unknown view, regex match results, arrays from JSON.parse, holes, Node's +// column grouping and the 100-entry limit. + +type Bag = Record; + +const named: number[] = [1, 2, 3]; +(named as unknown as Bag).extra = "named"; +console.log(named); + +const words: string[] = ["a"]; +(words as unknown as Bag)["my-key"] = { deep: [1, { x: 1 }] }; +console.log(words, [words]); + +const empty: number[] = []; +(empty as unknown as Bag).x = 1; +console.log(empty); + +const numeric: number[] = [1, 2]; +numeric[-1] = 5; +numeric[1.5] = 6; +console.log(numeric); + +const holes: (number | undefined)[] = [1, , 3]; +(holes as unknown as Bag).tail = true; +console.log(holes); + +const grid: number[] = []; +for (let i = 0; i < 30; i++) grid.push(i * 7); +(grid as unknown as Bag).label = "grid"; +console.log(grid); + +const long: number[] = []; +for (let i = 0; i < 120; i++) long.push(i); +(long as unknown as Bag).label = "long"; +console.log(long); + +const full: number[] = []; +for (let i = 0; i < 100; i++) full.push(i); +(full as unknown as Bag).label = "full"; +console.log(full); + +const match = /b(?c)?/.exec("abc"); +console.log(match); +const plainMatch = /x/.exec("axb"); +console.log(plainMatch); + +const parsed = JSON.parse("[1,2,3,4,5,6,7,8]") as unknown as Bag; +parsed.extra = "named"; +console.log(parsed); + +const plain: number[] = [1, 2, 3]; +console.log(plain, [] as number[]); diff --git a/tests/harness/corpus-costs/7060-array-numeric-keys.cjs.json b/tests/harness/corpus-costs/7060-array-numeric-keys.cjs.json new file mode 100644 index 000000000..184d2ea9f --- /dev/null +++ b/tests/harness/corpus-costs/7060-array-numeric-keys.cjs.json @@ -0,0 +1 @@ +{ "plain": 4400, "san": 6000 } diff --git a/tests/harness/corpus-costs/7061-array-object-keys.ts.json b/tests/harness/corpus-costs/7061-array-object-keys.ts.json new file mode 100644 index 000000000..b39c9d973 --- /dev/null +++ b/tests/harness/corpus-costs/7061-array-object-keys.ts.json @@ -0,0 +1 @@ +{ "plain": 4000, "san": 5400 } diff --git a/tests/harness/corpus-costs/7062-inspect-array-properties.ts.json b/tests/harness/corpus-costs/7062-inspect-array-properties.ts.json new file mode 100644 index 000000000..dcd73aea7 --- /dev/null +++ b/tests/harness/corpus-costs/7062-inspect-array-properties.ts.json @@ -0,0 +1 @@ +{ "plain": 4300, "san": 6000 } diff --git a/tests/harness/errors.test.ts b/tests/harness/errors.test.ts index 8af6998ca..38749c168 100644 --- a/tests/harness/errors.test.ts +++ b/tests/harness/errors.test.ts @@ -1777,3 +1777,25 @@ console.log("unreachable"); ); }); }); + +describe("numeric properties of native arrays", () => { + // A native array keeps its numeric properties ("-1", "1.5") in a property + // table that holds values without attributes, so defining one with a + // descriptor refuses instead of dropping the attributes. + test("Object.defineProperty on a numeric key is a named fence", async () => { + const r = await compileAndRun( + "array-numeric-define-property", + `const words: string[] = ["a"]; +words[-1] = "x"; +console.log(words[-1]); +Object.defineProperty(words as unknown as Record, "-3", { value: "kept" }); +console.log("unreachable"); +`, + ); + expect(r.exitCode).toBe(1); + expect(r.stdout).toBe("x\n"); + expect(r.stderr).toMatch( + /^Uncaught Error: Object\.defineProperty on a numeric property of a native array is not supported yet\n$/, + ); + }); +}); From bde0b64004a692c8254138475f7168439db3b3c4 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 05:36:03 -0500 Subject: [PATCH 005/197] Test that move-state refuses a state file committed as a link outside the checkout A branch can commit the file a state group writes (or that the config creates from a template) as a symlink. The split tools resolve every path they read or write through confine.mjs, so move-state refuses such a file instead of appending the state through the link. The test covers that path: the group file is listed under create, exists as a link to a file outside the checkout, and the tool exits with an error and leaves the outside file unchanged. --- tests/harness/split-tools.test.ts | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/harness/split-tools.test.ts b/tests/harness/split-tools.test.ts index 778ef152b..08d89e850 100644 --- a/tests/harness/split-tools.test.ts +++ b/tests/harness/split-tools.test.ts @@ -193,6 +193,30 @@ test("split-module refuses a module that links outside the checkout", () => { expect(stderr).toContain("refusing"); }); +test("move-state refuses a state file that a branch commits as a link outside the checkout", () => { + const { dir, repo } = symlinkCheckout(); + // The group file is listed under `create` and already exists as a link, so + // creation skips it and the move would append the state through the link. + symlinkSync("../../outside/linked.ts", join(repo, "src/host-state.ts")); + writeFileSync(join(dir, "state.template"), ""); + const stderr = refused( + "move-state.mjs", + { + module: "src/host.ts", + class: "Host", + dir: "src", + create: { "host-state.ts": "state.template" }, + groups: [ + { file: "host-state.ts", key: "COUNTS", interfaceName: "Counts", fields: ["count"] }, + ], + }, + dir, + repo, + ); + expect(stderr).toContain("refusing"); + expect(readFileSync(join(dir, "outside/linked.ts"), "utf8")).toBe(outsideText); +}); + test("move-state refuses to rewrite a file that links outside the checkout", () => { const { dir, repo } = symlinkCheckout(); writeFileSync(join(repo, "src/host-state.ts"), ""); From aeb429c066fec555512e23abb04d5cfd962863bc Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 05:45:38 -0500 Subject: [PATCH 006/197] Type empty array literals by their slots, and report package diagnostics accurately - An empty `[]` whose own type is `never[]` (a `return []` in an async function, whose contextual type is `T[] | PromiseLike`, or an object literal's `files: []` field) builds as the unit-only element array, like the empty tuple, so typed array slots receive it through the checked empty-array lift. It was `number[]`, so `string[]` slots refused it with SC1090 or SC2002. - The refusal "instanceof against this native constructor" names its site, and coverage lists the sites whose right-hand side is `Object` or `Function`, which always reach it. An arrow function or method on the right of an untyped `instanceof` throws Node's "Function has non-object prototype" TypeError for an object. - Coverage lists the diagnostics of a function whose signature did not compile. A use of a parameter whose type did not compile reports SC2004. - Coverage lists deferred fences in a module's top-level code under "runs at startup", and the diagnostics envelope marks them `startup: true`. --- docs/content/docs/coverage.mdx | 8 +- docs/content/docs/dependencies.mdx | 2 +- .../compiler/src/backend/llvm/lib-shared.ts | 2 +- packages/compiler/src/coverage/report.ts | 17 ++++- .../compiler/src/diagnostics/diagnostic.ts | 4 + packages/compiler/src/diagnostics/envelope.ts | 4 + .../frontend/lowering/class-construction.ts | 19 ++++- .../lowering/expressions/array-literals.ts | 21 ++++++ .../lowering/expressions/relational.ts | 31 +++++++- .../src/frontend/lowering/lower-functions.ts | 75 +++++++++++++++++++ .../src/frontend/lowering/lower-modules.ts | 34 +++++++++ .../compiler/src/frontend/lowering/lowerer.ts | 20 ++++- packages/compiler/src/frontend/type-mapper.ts | 21 ++++++ .../compiler/src/ir/builtin-signatures.ts | 2 +- .../main.ts.json | 6 ++ .../main.ts.json | 6 ++ packages/runtime/src/scr_dyn.h | 1 + packages/runtime/src/scr_json.c | 22 +++++- .../7002-empty-array-contextual-slots/main.ts | 58 ++++++++++++++ .../7005-dyn-constructor-instanceof/main.ts | 11 +++ .../node_modules/tag-types/index.d.ts | 4 + .../node_modules/tag-types/index.js | 61 +++++++++++++++ .../node_modules/tag-types/package.json | 1 + .../blocked-signature-body.ts | 15 ++++ tests/coverage-fixtures/startup-fences.js | 9 +++ .../coverage-js-builtin-callable-alias.txt | 2 +- .../main.ts.json | 1 + .../main.ts.json | 1 + tests/harness/coverage.test.ts | 47 ++++++++++++ 29 files changed, 492 insertions(+), 13 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7002-empty-array-contextual-slots/main.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7005-dyn-constructor-instanceof/main.ts.json create mode 100644 tests/corpus/7002-empty-array-contextual-slots/main.ts create mode 100644 tests/corpus/7005-dyn-constructor-instanceof/main.ts create mode 100644 tests/corpus/7005-dyn-constructor-instanceof/node_modules/tag-types/index.d.ts create mode 100644 tests/corpus/7005-dyn-constructor-instanceof/node_modules/tag-types/index.js create mode 100644 tests/corpus/7005-dyn-constructor-instanceof/node_modules/tag-types/package.json create mode 100644 tests/coverage-fixtures/blocked-signature-body.ts create mode 100644 tests/coverage-fixtures/startup-fences.js create mode 100644 tests/harness/corpus-costs/7002-empty-array-contextual-slots/main.ts.json create mode 100644 tests/harness/corpus-costs/7005-dyn-constructor-instanceof/main.ts.json diff --git a/docs/content/docs/coverage.mdx b/docs/content/docs/coverage.mdx index ae36f736d..954b040db 100644 --- a/docs/content/docs/coverage.mdx +++ b/docs/content/docs/coverage.mdx @@ -38,6 +38,8 @@ When a program has unsupported operations, the report adds a group for each root compile staticallyStatements that compile to native instructions. in dynamic tierOf those, the statements in functions that operate on values typed only at runtime, such as any values and untyped JavaScript. Shown when there are any. See Dynamic tier. blockersUnsupported operations with counts, diagnostic codes, reasons, locations, and hints. + runs at startupDeferred operations in a JavaScript module's top-level code, which runs when the module loads: at startup for a module the program imports. + deferred to runtimeOther deferred operations in JavaScript code. Each throws only if it runs. in unreached codeBlockers in code the entry never reaches. A build never compiles that code, so these cannot fail it. differs from NodeSites that compile but can behave differently from Node.js. See Node divergences. npm package compilationNative package compilation outcomes and refusal reasons. @@ -46,7 +48,7 @@ When a program has unsupported operations, the report adds a group for each root Blockers in unreached code and deferred operations in JavaScript packages are listed separately because they don't fail a build. A deferred operation throws a runtime error if it runs, so a successful build doesn't mean every API of a package works; see [unsupported code inside a package](/docs/dependencies#unsupported-code-inside-a-package). -If a function's signature can't compile, the statements in its body count as not compiling, so the percentage reflects how much of the program compiles. +If a function's signature can't compile, the statements in its body count as not compiling, so the percentage reflects how much of the program compiles. The report still lists the diagnostics of those statements; a use of a parameter whose type can't compile reports `SC2004` back to the signature. @@ -211,6 +213,10 @@ For editors, CI annotations, and other tools, `scriptc coverage --print=diagnost group The id of the root-cause group the diagnostic belongs to. A group's primary site is its first in source order. + + startup + true on a deferred diagnostic in a module's top-level code, which runs when the module loads. Absent otherwise. + diff --git a/docs/content/docs/dependencies.mdx b/docs/content/docs/dependencies.mdx index 85a232bb2..409b9cb30 100644 --- a/docs/content/docs/dependencies.mdx +++ b/docs/content/docs/dependencies.mdx @@ -132,7 +132,7 @@ CommonJS packages compile with Node.js's module semantics. Each `require()` eval ## Unsupported code inside a package -A JavaScript package can contain operations scriptc can't compile, often in functions your program never calls. These don't fail the build: each one compiles to code that throws a runtime error if it ever runs. Coverage reports list them as deferred sites, so review the list and test the parts of the package your program uses. +A JavaScript package can contain operations scriptc can't compile, often in functions your program never calls. These don't fail the build: each one compiles to code that throws a runtime error if it ever runs. Coverage reports list them as deferred sites, so review the list and test the parts of the package your program uses. Sites in a module's top-level code are listed under `runs at startup`: that code runs when the module loads, at startup for a module the program imports. ## Provenance sources (experimental) diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index c08d7b720..453392243 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -47,7 +47,7 @@ export const LIB_FN_SYMS: Record = { "dyn.fromEntries": "scr_dyn_from_entries", "bytes.constructor": "scr_bytes_constructor", "bytes.construct": "scr_bytes_construct", - "bytes.instanceOf": "scr_bytes_instanceof", + "bytes.instanceOf": "scr_bytes_instanceof_at", "arrayBuffer.constructor": "scr_array_buffer_constructor", "sharedArrayBuffer.new": "scr_shared_array_buffer_new", "sharedArrayBuffer.is": "scr_shared_array_buffer_is", diff --git a/packages/compiler/src/coverage/report.ts b/packages/compiler/src/coverage/report.ts index 37f77f4c6..79017c97a 100644 --- a/packages/compiler/src/coverage/report.ts +++ b/packages/compiler/src/coverage/report.ts @@ -318,12 +318,23 @@ export function renderCoverage(input: CoverageInput, opts: CoverageRenderOptions // statements the module carries): they compile, but executing one // throws its SC-coded error — the report lists them so a "compiling" // JS program never hides its dynamic gaps. + // Fences in a module's top-level code run when the module initializes: + // they are listed first, as sites that run at startup. const fences = input.runtimeFences ?? []; - if (fences.length > 0) { + const startup = fences.filter((d) => d.startup === true); + const later = fences.filter((d) => d.startup !== true); + if (startup.length > 0) { out.push( - ` ${c(YELLOW, "deferred to runtime")} ${fences.length} site${fences.length === 1 ? "" : "s"} ${c(DIM, "(JS statements that throw their fence if executed)")}`, + ` ${c(YELLOW, "runs at startup")} ${startup.length} site${startup.length === 1 ? "" : "s"} ${c(DIM, "(JS statements in module top-level code, which runs when the module loads)")}`, ); - renderRows(groupBlockers(fences, "deferred"), YELLOW); + renderRows(groupBlockers(startup, "deferred"), YELLOW); + out.push(""); + } + if (later.length > 0) { + out.push( + ` ${c(YELLOW, "deferred to runtime")} ${later.length} site${later.length === 1 ? "" : "s"} ${c(DIM, "(JS statements that throw their fence if executed)")}`, + ); + renderRows(groupBlockers(later, "deferred"), YELLOW); out.push(""); } diff --git a/packages/compiler/src/diagnostics/diagnostic.ts b/packages/compiler/src/diagnostics/diagnostic.ts index 17cb2efc0..625ee93af 100644 --- a/packages/compiler/src/diagnostics/diagnostic.ts +++ b/packages/compiler/src/diagnostics/diagnostic.ts @@ -92,6 +92,10 @@ export interface ScrDiagnostic { * the error may not reproduce under the project's tsc. Machine-readable * output reports these under the "environment" category. */ typeWorld?: boolean; + /** A runtime fence in a module's top-level code, outside every function + * and class member body: it throws when the module initializes, which is + * program startup for a module the entry imports. */ + startup?: true; } /** SC3002–SC3004 — native-helper target, installation, and execution diff --git a/packages/compiler/src/diagnostics/envelope.ts b/packages/compiler/src/diagnostics/envelope.ts index 59c285ead..b4de8db77 100644 --- a/packages/compiler/src/diagnostics/envelope.ts +++ b/packages/compiler/src/diagnostics/envelope.ts @@ -67,6 +67,9 @@ export interface LocatedDiagnostic extends SourcePosition { hint: string; /** The id of the root-cause group this site belongs to. */ group: string; + /** Deferred scope only: the statement is module top-level code, so it + * runs when its module loads. */ + startup?: true; } export interface DiagnosticGroup { @@ -290,6 +293,7 @@ export function locateDiagnostic( message: diag.message, hint: hintOf(diag), group: groupId(diag, scope), + ...(scope === "deferred" && diag.startup === true ? { startup: true as const } : {}), }; } diff --git a/packages/compiler/src/frontend/lowering/class-construction.ts b/packages/compiler/src/frontend/lowering/class-construction.ts index 30f504b4a..398029474 100644 --- a/packages/compiler/src/frontend/lowering/class-construction.ts +++ b/packages/compiler/src/frontend/lowering/class-construction.ts @@ -24,16 +24,21 @@ const PREFIX = "%class.construct:"; const PACKED = `${PREFIX}packed`; const INSTANCEOF = "%class.instanceof:dynamic"; +/** `value instanceof callee` for an untyped constructor value, through one + * shared helper. `refusal` is the site's message for a runtime + * constructor with no instance test. */ export function checkedClassInstanceOf( lowerer: Lowerer, value: IrExpr, callee: IrExpr, + refusal: string, loc: SrcLoc, ): IrExpr { if (!lowerer.liftedFns.some((fn) => fn.name === INSTANCEOF)) { const params = [ { localId: "value", name: "value", type: DYN }, { localId: "callee", name: "callee", type: DYN }, + { localId: "refusal", name: "refusal", type: STRING }, ]; lowerer.liftedFns.push({ name: INSTANCEOF, @@ -46,7 +51,11 @@ export function checkedClassInstanceOf( value: { kind: "libCall", fn: "bytes.instanceOf", - args: [varRef("value", DYN, loc), varRef("callee", DYN, loc)], + args: [ + varRef("value", DYN, loc), + varRef("callee", DYN, loc), + varRef("refusal", STRING, loc), + ], type: BOOL, loc, }, @@ -56,7 +65,13 @@ export function checkedClassInstanceOf( loc, }); } - return { kind: "call", callee: INSTANCEOF, args: [value, callee], type: BOOL, loc }; + return { + kind: "call", + callee: INSTANCEOF, + args: [value, callee, { kind: "strLit", value: refusal, type: STRING, loc }], + type: BOOL, + loc, + }; } /** Checked construction dispatches to emitted native constructor thunks. */ diff --git a/packages/compiler/src/frontend/lowering/expressions/array-literals.ts b/packages/compiler/src/frontend/lowering/expressions/array-literals.ts index 8f119504c..cb2d86804 100644 --- a/packages/compiler/src/frontend/lowering/expressions/array-literals.ts +++ b/packages/compiler/src/frontend/lowering/expressions/array-literals.ts @@ -58,6 +58,22 @@ export function jsHoldsDynObject(lowerer: Lowerer, expr: ts.Expression): boolean return false; } +/** An empty TypeScript literal whose own type is `never[]`: tsc's type for + * `[]` with no contextual array type (`return []` in an async function, + * whose contextual type is `string[] | PromiseLike`; `{ files: [] }`). + * It holds no element, as the empty tuple does, so it builds as the + * unit-only element array, and an array slot receives it through the + * checked empty-array lift. never's f64 element mapping would make it a + * `number[]`, which no `string[]` slot accepts. */ +function emptyNeverLiteral(lowerer: Lowerer, expr: ts.ArrayLiteralExpression): boolean { + if (expr.elements.nodes.length > 0 || isJsSourceFile(ts.getSourceFileOfNode(expr))) return false; + const own = lowerer.typeOf(expr); + return ( + lowerer.checker.isArrayType(own) && + (lowerer.checker.getTypeArguments(own as ts.TypeReference)[0]!.flags & ts.TypeFlags.Never) !== 0 + ); +} + /** `[a, b, c]`. The element type comes from the contextual type when tsc * has one (`const a: number[] = []`, arguments, nested literals) and from * the literal's own inferred type otherwise. A bare `[]` with no context @@ -80,6 +96,8 @@ export function lowerArrayLiteral( ? lowerer.typeOf(expr) : (ctxType ?? lowerer.typeOf(expr)); let mapped = expected ?? lowerer.mapTypeOf(tsType); + if (expected === undefined && ctxType === undefined && emptyNeverLiteral(lowerer, expr)) + mapped = arrayOf(unitOnlyUnion(lowerer.unions)); // Inferred JavaScript arrays can contain unrelated constructors with // structurally equal checker types. Their native class ABIs still differ. if ( @@ -131,11 +149,14 @@ export function lowerArrayLiteral( // the literal's own inferred type is the real tuple. if (!mapped || mapped.kind === "union" || mapped.kind === "dyn") { const ctxUnion = mapped?.kind === "union" ? mapped : null; + const ctxUnmapped = mapped === null; mapped = lowerer.mapTypeOf(lowerer.typeOf(expr)); // The never-tainted own type carries no element information here // either (the JS story above) — only a contextual union's single // array arm below may still supply a static home. if (neverTaintedOwnJs) mapped = null; + if (ctxUnmapped && emptyNeverLiteral(lowerer, expr)) + mapped = arrayOf(unitOnlyUnion(lowerer.unions)); // An EMPTY literal in a union slot (`const t: string[] | undefined = // []`, the `[]` default argument against `string[] | undefined`) has // no useful own type — tsc infers `never[]`, whose f64-element diff --git a/packages/compiler/src/frontend/lowering/expressions/relational.ts b/packages/compiler/src/frontend/lowering/expressions/relational.ts index 61558f008..759687ff0 100644 --- a/packages/compiler/src/frontend/lowering/expressions/relational.ts +++ b/packages/compiler/src/frontend/lowering/expressions/relational.ts @@ -49,6 +49,7 @@ import { InternalCompilerError } from "../../../errors.js"; import { tryLowerExpression } from "./try-lower-expression.js"; import { isJsSourceFile, locOf } from "../../program.js"; import { checkedClassInstanceOf } from "../class-construction.js"; +import { unsupportedDiag } from "../../../diagnostics/diagnostic.js"; import { globalSymbolKey } from "./global-symbols.js"; import { foldedStringKeyOf } from "./object-literals.js"; import { NARROW_FIRST, sideEffectFreeOptionValue } from "../surfaces.js"; @@ -490,7 +491,13 @@ export function lowerInstanceOf(lowerer: Lowerer, expr: ts.BinaryExpression, loc const constructor = tryLowerExpression(lowerer, expr.right); if (constructor?.type.kind === "dyn" && isJsSourceFile(ts.getSourceFileOfNode(expr))) { const value = lowerer.coerceInto(expr.left, lowerer.lowerExpr(expr.left), DYN); - return checkedClassInstanceOf(lowerer, value, constructor, loc); + return checkedClassInstanceOf( + lowerer, + value, + constructor, + dynInstanceOfRefusal(lowerer, expr), + loc, + ); } // `this instanceof Glob` in a JavaScript constructor function: the // function's prototype against the value's prototype chain. @@ -2000,3 +2007,25 @@ function lowerRuntimeKeyIn( } return { kind: "call", callee: helper, args: [keyIr, recv], type: BOOL, loc }; } + +/** Standard-library constructors the untyped instanceof helper has no + * instance test for: an untyped `x instanceof Object` always refuses. */ +const DYN_INSTANCEOF_REFUSED: ReadonlySet = new Set(["Object", "Function"]); + +/** The SC1090 message `x instanceof C` throws, with the site's position, + * when `C` is a runtime constructor with no instance test. A right-hand + * side that names one of DYN_INSTANCEOF_REFUSED always reaches that + * refusal, so the site is listed with the runtime fences. */ +function dynInstanceOfRefusal(lowerer: Lowerer, expr: ts.BinaryExpression): string { + const diag = unsupportedDiag("SC1090", locOf(expr), "instanceof against this native constructor"); + const symbol = lowerer.checker.getSymbolAtLocation(expr.right); + if ( + symbol !== undefined && + lowerer.isStdlibSymbol(symbol) && + DYN_INSTANCEOF_REFUSED.has(ts.symbolName(symbol)) + ) + lowerer.runtimeFences.push(diag); + const sf = ts.getSourceFileOfNode(expr); + const line = ts.getLineAndCharacterOfPosition(sf, diag.loc.start).line + 1; + return `${diag.message} [${diag.code} at ${diag.loc.file}:${line}]`; +} diff --git a/packages/compiler/src/frontend/lowering/lower-functions.ts b/packages/compiler/src/frontend/lowering/lower-functions.ts index 653c0d837..ea7d7957b 100644 --- a/packages/compiler/src/frontend/lowering/lower-functions.ts +++ b/packages/compiler/src/frontend/lowering/lower-functions.ts @@ -1,3 +1,4 @@ +import type { ScrDiagnostic } from "../../diagnostics/diagnostic.js"; import { declareTypedArguments } from "./typed-arguments.js"; import { dynUndefinedExpr, varRef } from "../../ir/build.js"; import { everyStmtList, transformStmtList } from "../../ir/traverse.js"; @@ -17,6 +18,7 @@ import { type IrLocal, type IrParam, type IrStmt, + type IrType, type SrcLoc, VOID, canBoxFuncIntoDyn, @@ -858,3 +860,76 @@ function isBufferCheckOf(e: IrExpr, slotId: string): boolean { e.type.elem === "u8" ); } + +/** The diagnostics of a TypeScript function whose signature did not + * compile, for the coverage remainder: its statements lower for their + * diagnostics only (the remainder's output is discarded), so the report + * lists them with the signature's blocker instead of hiding them behind it. + * Each parameter whose type maps is declared, and a use of any other + * parameter reports SC2004 back to the declaration. The statements already + * count as failed (countSkippedBody), so the pass leaves the stats alone. + * The pass adds diagnostics only: an internal error while it lowers drops + * what it found, and the signature's blocker stands alone as before. */ +export function blockedBodyDiagnostics( + lowerer: Lowerer, + decl: ts.FunctionDeclaration, +): ScrDiagnostic[] { + const body = decl.body; + if (!body || decl.typeParameters || decl.asteriskToken) return []; + if (isJsSourceFile(ts.getSourceFileOfNode(decl))) return []; + const symbol = declSymbolOf(lowerer, decl); + if (!symbol || lowerer.fnSigsBySymbol.has(symbol)) return []; + const diagsBefore = lowerer.diags.length; + const fencesBefore = lowerer.runtimeFences.length; + const suppressStats = lowerer.suppressStats; + lowerer.suppressStats = true; + let declared: IrType = DYN; + lowersCleanly(lowerer, () => { + declared = lowerer.declaredReturnType(decl, decl); + }); + const isAsync = decl.modifiers?.nodes.some((m) => m.kind === ts.SyntaxKind.AsyncKeyword) ?? false; + const ctx = newFnCtx(false, null, null, lowerer.bodyReturnType(isAsync, declared)); + ctx.isAsync = isAsync; + lowerer.fnStack.push(ctx); + let internal = false; + try { + for (const param of decl.parameters.nodes) { + if (isThisParameter(param)) continue; + const declaredParam = lowersCleanly(lowerer, () => { + lowerer.declareParams([param], lowerer.paramShapes([param])); + }); + if (declaredParam) continue; + const symbol = ts.isIdentifier(param.name) + ? lowerer.checker.getSymbolAtLocation(param.name) + : undefined; + if (symbol) lowerer.blockedBindings.add(symbol); + } + lowersCleanly(lowerer, () => { + lowerer.lowerStmts(body.statements.nodes); + }); + } catch (error) { + if (!(error instanceof InternalCompilerError)) throw error; + internal = true; + } finally { + lowerer.fnStack.pop(); + lowerer.suppressStats = suppressStats; + } + lowerer.runtimeFences.splice(fencesBefore); + const diags = lowerer.diags.splice(diagsBefore); + return internal || diags.some((d) => d.code === "SC9001") ? [] : diags; +} + +/** Runs `lower`, for blockedBodyDiagnostics: false when it poisons, with the + * diagnostics it recorded dropped (a failed mapping reports what the + * signature's blocker already reports). */ +function lowersCleanly(lowerer: Lowerer, lower: () => void): boolean { + const before = lowerer.diags.length; + try { + lower(); + return true; + } catch (error) { + if (!(error instanceof PoisonError)) throw error; + lowerer.diags.splice(before); + return false; + } +} diff --git a/packages/compiler/src/frontend/lowering/lower-modules.ts b/packages/compiler/src/frontend/lowering/lower-modules.ts index 5b78c7278..5238333ad 100644 --- a/packages/compiler/src/frontend/lowering/lower-modules.ts +++ b/packages/compiler/src/frontend/lowering/lower-modules.ts @@ -2929,6 +2929,38 @@ export function lowerDefaultExport(lowerer: Lowerer, stmt: ts.ExportAssignment): return { kind: "assign", localId: g.id, value, loc: locOf(stmt) }; } +/** Marks the runtime fences recorded since `fencesBefore` that lie in + * `sf`'s top-level code: they run when the module initializes. A fence + * inside a function, or inside a class member that runs later (a method, + * accessor, constructor or instance field initializer), runs when that + * code is called. */ +function markModuleInitFences(lowerer: Lowerer, sf: ts.SourceFile, fencesBefore: number): void { + for (let i = fencesBefore; i < lowerer.runtimeFences.length; i++) { + const fence = lowerer.runtimeFences[i]!; + if (fence.loc.file !== sf.fileName || runsAfterModuleInit(sf, fence.loc.start)) continue; + lowerer.runtimeFences[i] = { ...fence, startup: true }; + } +} + +/** Whether the code at `pos` sits in a function or a later-running class + * member body. */ +function runsAfterModuleInit(sf: ts.SourceFile, pos: number): boolean { + let later = false; + ts.walkPreorder(sf, (node) => { + if (node.pos > pos || node.end <= pos) return "skip"; + if ( + ts.isFunctionLike(node) || + (ts.isPropertyDeclaration(node) && + !node.modifiers?.nodes.some((m) => m.kind === ts.SyntaxKind.StaticKeyword)) + ) { + later = true; + return "stop"; + } + return undefined; + }); + return later; +} + export function lowerFileInit( lowerer: Lowerer, sf: ts.SourceFile, @@ -2938,6 +2970,7 @@ export function lowerFileInit( const isAsync = lowerer.asyncInitFiles.has(sf); const ctx = newFnCtx(false, null, null, VOID); ctx.isAsync = isAsync; + const fencesBefore = lowerer.runtimeFences.length; lowerer.fnStack.push(ctx); try { const loc0: SrcLoc = { file: sf.fileName, start: 0, end: 0 }; @@ -3288,6 +3321,7 @@ export function lowerFileInit( }; } finally { lowerer.fnStack.pop(); + markModuleInitFences(lowerer, sf, fencesBefore); } } diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index edf0a694b..7838213b2 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -444,6 +444,7 @@ import { lowerNestedFunctionDecl, lowerLambda, lowerFunction, + blockedBodyDiagnostics, } from "./lower-functions.js"; import { lowerArrayMethodCall } from "./containers/array-methods.js"; import { type ArrayHofInfo, fuseArrayPipelines } from "./containers/array-pipelines.js"; @@ -1346,6 +1347,7 @@ export function lowerToIr( // travel with them: a failed statement is never left unexplained. const covered: LowerResult = { ...result, + diagnostics: [...result.diagnostics, ...remainder.reachedBlockedBodyDiagnostics], stats: { ...result.stats, statementsDynamic }, unreached: { diagnostics: rem.diagnostics, @@ -7437,7 +7439,10 @@ export class Lowerer { if (sig && !this.wantBody(sig.name)) continue; const fn = this.lowerFunction(decl); if (fn) functions.push(fn); - else if (this.countsSkips()) this.countSkippedBody(decl); + else if (this.countsSkips()) { + this.countSkippedBody(decl); + if (this.remainder) this.reportBlockedBody(decl); + } } for (const decl of fp.classDecls) { const info = this.classes.get(this.classNamer(decl)); @@ -7768,6 +7773,19 @@ export class Lowerer { } } + /** Coverage remainder: the body diagnostics of a function whose + * signature blocked (blockedBodyDiagnostics). A function reachable emit + * reached (its signature's diagnostics flushed there) reports them with + * the reached diagnostics; any other reports them as unreached. */ + readonly reachedBlockedBodyDiagnostics: ScrDiagnostic[] = []; + private reportBlockedBody(decl: ts.FunctionDeclaration): void { + const diags = blockedBodyDiagnostics(this, decl); + const symbol = declSymbolOf(this, decl); + if (symbol && this.alreadyFlushed.has(symbol)) + this.reachedBlockedBodyDiagnostics.push(...diags); + else for (const d of diags) this.pushDiag(d); + } + /** Whether run() counts a signature-blocked declaration in * stats.functionsSkipped: whole-program passes and the coverage * remainder do; an externally-gated emit pass leaves the counting to the diff --git a/packages/compiler/src/frontend/type-mapper.ts b/packages/compiler/src/frontend/type-mapper.ts index 9abd5a0b0..93349054a 100644 --- a/packages/compiler/src/frontend/type-mapper.ts +++ b/packages/compiler/src/frontend/type-mapper.ts @@ -5731,6 +5731,19 @@ function constAssertedEmptyArrayProp(p: ts.Symbol): boolean { return underConstAssertion(d); } +/** A TypeScript property assignment `name: []` whose type is `never[]`. */ +function emptyNeverArrayProp(p: ts.Symbol, fieldTs: ts.Type, checker: ts.TypeChecker): boolean { + const d = p.valueDeclaration; + if (d === undefined || !ts.isPropertyAssignment(d)) return false; + if (isJsSourceFile(ts.getSourceFileOfNode(d))) return false; + if (!ts.isArrayLiteralExpression(d.initializer) || d.initializer.elements.nodes.length > 0) + return false; + return ( + checker.isArrayType(fieldTs) && + (checker.getTypeArguments(fieldTs as ts.TypeReference)[0]!.flags & ts.TypeFlags.Never) !== 0 + ); +} + function mapGenericAwaitedAlias(type: ts.Type, ctx: TypeMapperCtx): IrType | null { const { resolveTypeParam } = ctx; if (!resolveTypeParam) return null; @@ -7767,6 +7780,14 @@ function mapRecordTypeInner( ) { pt = arrayOf(unitOnlyUnion(ctx.unions)); } + // A TypeScript property initialized with an empty literal and typed + // `never[]` (`const out = { files: [] }`) holds no element, so it maps + // like the empty tuple: the unit-only element array, which a + // destination's typed array field (`files: string[]`) receives through + // the checked empty-array lift. never's f64 element would make it a + // `number[]` field. + if (pt?.kind === "array" && emptyNeverArrayProp(p, fieldTs, checker)) + pt = arrayOf(unitOnlyUnion(ctx.unions)); if (pt === null && (fieldTs.flags & ts.TypeFlags.Any) !== 0) pt = DYN; // An `any[]` field (`list: any[]` in an SDK's message type, also // optional or nullable) is the checked-dynamic array, exactly as diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index 8f5928d12..70ec7d502 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -3570,7 +3570,7 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ "weakSet.new": { argTypes: [DYN], result: DYN }, "dyn.fromEntries": { argTypes: [DYN], result: DYN }, "bytes.constructor": { argTypes: [STRING], result: DYN }, - "bytes.instanceOf": { argTypes: [DYN, DYN], result: BOOL }, + "bytes.instanceOf": { argTypes: [DYN, DYN, STRING], result: BOOL }, "bytes.construct": { argTypes: [DYN, DYN, STRING], result: DYN }, /** The ambient receiver — JS `this` in a plain (non-method) function * body: the innermost binding the current firing/dispatch window diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7002-empty-array-contextual-slots/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7002-empty-array-contextual-slots/main.ts.json new file mode 100644 index 000000000..d22715215 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7002-empty-array-contextual-slots/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7002-empty-array-contextual-slots/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7005-dyn-constructor-instanceof/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7005-dyn-constructor-instanceof/main.ts.json new file mode 100644 index 000000000..313888b49 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7005-dyn-constructor-instanceof/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7005-dyn-constructor-instanceof/main.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_dyn.h b/packages/runtime/src/scr_dyn.h index a427fc9a1..95f251bd4 100644 --- a/packages/runtime/src/scr_dyn.h +++ b/packages/runtime/src/scr_dyn.h @@ -868,6 +868,7 @@ void scr_dyn_proxy_unsupported(const char *operation); ScrDyn *scr_dyn_new_bytes(const ScrBytes *b); ScrDyn *scr_bytes_constructor(const ScrStr *name); bool scr_bytes_instanceof(const ScrDyn *value, const ScrDyn *callee); +bool scr_bytes_instanceof_at(const ScrDyn *value, const ScrDyn *callee, const ScrStr *refusal); bool scr_dyn_data_view_is(const ScrDyn *value); ScrDyn *scr_dyn_construct(const ScrDyn *callee, const ScrDyn *args, const ScrStr *what); ScrDyn *scr_bytes_construct(const ScrDyn *callee, const ScrDyn *args, const ScrStr *what); diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index 4e21fa397..906be0ccd 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -3920,7 +3920,10 @@ static int scr_dyn_global_constructor_instanceof(const ScrDyn *value, const ScrD return -1; } -bool scr_bytes_instanceof(const ScrDyn *value, const ScrDyn *callee) { +/* `value instanceof callee` for an untyped constructor value. `refusal`, when + * given, is the site's SC1090 message for a runtime constructor with no + * instance test. */ +bool scr_bytes_instanceof_at(const ScrDyn *value, const ScrDyn *callee, const ScrStr *refusal) { if (!callee || callee->kind != SCR_DYN_FUNC) { static const char message[] = "Right-hand side of 'instanceof' is not callable"; scr_throw_error_msg(SCR_ERR_TYPE, message, sizeof message - 1); @@ -3962,12 +3965,29 @@ bool scr_bytes_instanceof(const ScrDyn *value, const ScrDyn *callee) { scr_dyn_release(prototype); return found; } + bool undefined_prototype = prototype->kind == SCR_DYN_UNDEF; scr_dyn_release(prototype); + /* A function the program compiled with no prototype (an arrow function + * or a method): Node's TypeError for an object, false for a primitive. + * The runtime's own constructors carry "native:" signatures. */ + if (undefined_prototype && (!callee->v.fn.sig || strncmp(callee->v.fn.sig, "native:", 7) != 0)) { + if (value->kind != SCR_DYN_OBJ && value->kind != SCR_DYN_ARR && value->kind != SCR_DYN_FUNC) return false; + static const char message[] = "Function has non-object prototype 'undefined' in instanceof check"; + scr_throw_error_msg(SCR_ERR_TYPE, message, sizeof message - 1); + return false; + } + } + if (refusal) { + scr_throw_error_msg_code(SCR_ERR_ERROR, refusal->data, refusal->len, "SC1090"); + return false; } static const char message[] = "instanceof against this native constructor is not supported yet"; scr_throw_error_msg_code(SCR_ERR_ERROR, message, sizeof message - 1, "SC1090"); return false; } +bool scr_bytes_instanceof(const ScrDyn *value, const ScrDyn *callee) { + return scr_bytes_instanceof_at(value, callee, NULL); +} ScrDyn *scr_bytes_construct(const ScrDyn *callee, const ScrDyn *args, const ScrStr *what) { if (callee->kind == SCR_DYN_FUNC && scr_array_ctor && callee->v.fn.clo == scr_array_ctor->v.fn.clo) return scr_array_ctor_call(callee->v.fn.clo, args->v.arr.items, args->v.arr.len); diff --git a/tests/corpus/7002-empty-array-contextual-slots/main.ts b/tests/corpus/7002-empty-array-contextual-slots/main.ts new file mode 100644 index 000000000..ec08a297a --- /dev/null +++ b/tests/corpus/7002-empty-array-contextual-slots/main.ts @@ -0,0 +1,58 @@ +// Empty array literals whose own type is `never[]`: `return []` in an async +// function (its contextual type is `string[] | PromiseLike`) and +// an object literal's `files: []` field returned or passed where typed +// arrays are declared. Each value holds the declared element type. +interface Listing { + files: string[]; + sizes: number[]; +} + +async function readNames(text: string): Promise { + try { + return JSON.parse(text) as string[]; + } catch { + return []; + } +} + +const loadIds = async (fail: boolean): Promise => { + if (fail) return []; + return [1, 2]; +}; + +function emptyListing(): Listing { + const listing = { files: [], sizes: [] }; + return listing; +} + +function index(): Record<"files" | "dirs", string[]> { + const out = { files: [], dirs: [] }; + return out; +} + +function total(listing: Listing): number { + return listing.sizes.reduce((sum, size) => sum + size, 0); +} + +async function main(): Promise { + const names = await readNames("not json"); + names.push("fallback"); + console.log(names, (await readNames('["a","b"]')).join("+")); + const ids = await loadIds(true); + ids.push(7); + console.log(ids, await loadIds(false)); + + const listing = emptyListing(); + listing.files.push("a.txt"); + listing.sizes.push(3, 4); + console.log(listing, total(listing)); + + const dirs = index(); + dirs.dirs.push("src"); + console.log(JSON.stringify(dirs), dirs.files.length); + + const blank = { files: [], sizes: [] }; + console.log(total(blank), JSON.stringify(blank)); +} + +main(); diff --git a/tests/corpus/7005-dyn-constructor-instanceof/main.ts b/tests/corpus/7005-dyn-constructor-instanceof/main.ts new file mode 100644 index 000000000..c32a820c2 --- /dev/null +++ b/tests/corpus/7005-dyn-constructor-instanceof/main.ts @@ -0,0 +1,11 @@ +// A package's own functions, read off an object the checker cannot type, +// on the right of `instanceof`: Node walks the value's prototype chain for +// an ES5 constructor's prototype, at module load and in calls, and throws a +// TypeError for an arrow function, which has no prototype, when the value +// is an object. +import { arrowCheck, describe, rejects, tags } from "tag-types"; + +console.log(tags()); +console.log(describe("{}"), describe("[1]"), describe("null"), describe('"s"')); +console.log(rejects('{"tag":"!x"}')); +console.log(arrowCheck("{}"), arrowCheck("1")); diff --git a/tests/corpus/7005-dyn-constructor-instanceof/node_modules/tag-types/index.d.ts b/tests/corpus/7005-dyn-constructor-instanceof/node_modules/tag-types/index.d.ts new file mode 100644 index 000000000..adeea728e --- /dev/null +++ b/tests/corpus/7005-dyn-constructor-instanceof/node_modules/tag-types/index.d.ts @@ -0,0 +1,4 @@ +export declare function describe(json: string): string; +export declare function tags(): string; +export declare function rejects(json: string): string; +export declare function arrowCheck(json: string): string; diff --git a/tests/corpus/7005-dyn-constructor-instanceof/node_modules/tag-types/index.js b/tests/corpus/7005-dyn-constructor-instanceof/node_modules/tag-types/index.js new file mode 100644 index 000000000..20eee362f --- /dev/null +++ b/tests/corpus/7005-dyn-constructor-instanceof/node_modules/tag-types/index.js @@ -0,0 +1,61 @@ +"use strict"; +// A bundler's lazy module object holding an ES5 constructor: the checker +// cannot type the object, so the constructor is an untyped value where +// `instanceof` tests it. +var types = {}; +var hasTypes; +function requireTypes() { + if (hasTypes) return types; + hasTypes = 1; + function Type(tag, options) { + this.tag = tag; + this.kind = (options && options.kind) || "scalar"; + } + function Schema(list) { + this.list = list; + } + types.Type = Type; + types.Schema = Schema; + types.make = (tag) => new Type(tag); + return types; +} +var loaded = requireTypes(); +var Type = loaded.Type; +var Schema = loaded.Schema; +var make = loaded.make; + +function compileList(list) { + var tags = []; + list.forEach(function (item) { + if (!(item instanceof Type)) throw new Error("not a Type: " + JSON.stringify(item)); + tags.push(item.tag + ":" + item.kind); + }); + return tags; +} +// Runs when the module loads, as js-yaml's default schema does. +var defaults = new Schema(compileList([new Type("!str"), new Type("!seq", { kind: "sequence" })])); + +module.exports.describe = function (json) { + var value = JSON.parse(json); + return [value instanceof Type, value instanceof Schema, defaults instanceof Schema, defaults instanceof Type].join(","); +}; +module.exports.tags = function () { + return defaults.list.join(" "); +}; +function rejects(json) { + try { + compileList([JSON.parse(json)]); + return "accepted"; + } catch (error) { + return error.message; + } +} +module.exports.rejects = rejects; +// An arrow function has no prototype: Node throws a TypeError for an object. +module.exports.arrowCheck = function (json) { + try { + return String(JSON.parse(json) instanceof make); + } catch (error) { + return error.name + ": " + error.message; + } +}; diff --git a/tests/corpus/7005-dyn-constructor-instanceof/node_modules/tag-types/package.json b/tests/corpus/7005-dyn-constructor-instanceof/node_modules/tag-types/package.json new file mode 100644 index 000000000..a469a45f3 --- /dev/null +++ b/tests/corpus/7005-dyn-constructor-instanceof/node_modules/tag-types/package.json @@ -0,0 +1 @@ +{ "name": "tag-types", "version": "1.0.0", "main": "./index.js", "types": "./index.d.ts" } diff --git a/tests/coverage-fixtures/blocked-signature-body.ts b/tests/coverage-fixtures/blocked-signature-body.ts new file mode 100644 index 000000000..7f8db4eb8 --- /dev/null +++ b/tests/coverage-fixtures/blocked-signature-body.ts @@ -0,0 +1,15 @@ +// Functions whose signatures cannot compile: the report still lists what +// their bodies would report. A use of the blocked parameter names the +// declaration; the body's own unsupported statement reports itself, with +// the reached diagnostics for a function the entry calls and as unreached +// for one nothing calls. +function measure(target: WeakRef, label: string): number { + const extra = new WeakRef({ label }); + console.log(target); + return label.length + (extra === undefined ? 0 : 1); +} +function unused(target: WeakRef): number { + const extra = new WeakRef({ target }); + return extra === undefined ? 0 : 1; +} +console.log(measure(new WeakRef({}), "abc")); diff --git a/tests/coverage-fixtures/startup-fences.js b/tests/coverage-fixtures/startup-fences.js new file mode 100644 index 000000000..22aad4e0a --- /dev/null +++ b/tests/coverage-fixtures/startup-fences.js @@ -0,0 +1,9 @@ +// Deferred fences in a module's top-level code run when the module loads; +// a fence inside a function runs when the function is called. +"use strict"; +var value = JSON.parse("{}"); +console.log(value instanceof Object); +function check(other) { + return other instanceof Object; +} +console.log(check([])); diff --git a/tests/harness/__snapshots__/coverage-js-builtin-callable-alias.txt b/tests/harness/__snapshots__/coverage-js-builtin-callable-alias.txt index 53eac6c3c..6291c291f 100644 --- a/tests/harness/__snapshots__/coverage-js-builtin-callable-alias.txt +++ b/tests/harness/__snapshots__/coverage-js-builtin-callable-alias.txt @@ -3,7 +3,7 @@ scriptc coverage tests/coverage-fixtures/js-builtin-callable-alias.js statements analyzed 3 compile statically 2 (66%) - deferred to runtime 1 site (JS statements that throw their fence if executed) + runs at startup 1 site (JS statements in module top-level code, which runs when the module loads) ×1 SC1090 a builtin function alias as an escaping value (call 'ext' directly) at tests/coverage-fixtures/js-builtin-callable-alias.js:8:13 hint: this form is valid TypeScript but does not compile yet — rewrite it with a supported equivalent of the construct the message names (https://scriptc.dev/limitations lists the boundary) diff --git a/tests/harness/corpus-costs/7002-empty-array-contextual-slots/main.ts.json b/tests/harness/corpus-costs/7002-empty-array-contextual-slots/main.ts.json new file mode 100644 index 000000000..9495bb65c --- /dev/null +++ b/tests/harness/corpus-costs/7002-empty-array-contextual-slots/main.ts.json @@ -0,0 +1 @@ +{ "plain": 2282, "san": 1173 } diff --git a/tests/harness/corpus-costs/7005-dyn-constructor-instanceof/main.ts.json b/tests/harness/corpus-costs/7005-dyn-constructor-instanceof/main.ts.json new file mode 100644 index 000000000..f84e21437 --- /dev/null +++ b/tests/harness/corpus-costs/7005-dyn-constructor-instanceof/main.ts.json @@ -0,0 +1 @@ +{ "plain": 2308, "san": 955 } diff --git a/tests/harness/coverage.test.ts b/tests/harness/coverage.test.ts index 86d202b46..40b55b078 100644 --- a/tests/harness/coverage.test.ts +++ b/tests/harness/coverage.test.ts @@ -182,6 +182,53 @@ test("statements of a signature-blocked function count as not static", () => { expect(coverage.stats.statementsFailed + (un?.statementsFailed ?? 0)).toBe(5); }); +test("the body of a signature-blocked function reports its diagnostics", () => { + const { coverage } = analyze(fixture("blocked-signature-body.ts")); + const sites = (diags: readonly { code: string; message: string }[]) => + diags.map((d) => `${d.code} ${d.message}`).sort(); + // measure is called: its body's diagnostics join the reached ones. + expect(sites(coverage.diagnostics)).toEqual([ + "SC2004 uses of 'extra' inherit the blocker on its declaration", + "SC2004 uses of 'measure' inherit the blocker on its declaration", + "SC2004 uses of 'target' inherit the blocker on its declaration", + "SC2020 'WeakRef' is part of the standard library types but has no scriptc lowering yet", + "SC2020 'new WeakRef' is part of the standard library types but has no scriptc lowering yet", + ]); + // Nothing calls unused: its signature and body report as unreached. + expect(sites(coverage.unreached?.diagnostics ?? [])).toEqual([ + "SC2004 uses of 'extra' inherit the blocker on its declaration", + "SC2020 'WeakRef' is part of the standard library types but has no scriptc lowering yet", + "SC2020 'new WeakRef' is part of the standard library types but has no scriptc lowering yet", + ]); + // The bodies' statements still count once each, as not static. + const un = coverage.unreached?.stats; + expect(coverage.stats.functionsSkipped + (un?.functionsSkipped ?? 0)).toBe(2); + expect(coverage.stats.statementsTotal + (un?.statementsTotal ?? 0)).toBe(6); + expect(coverage.stats.statementsFailed + (un?.statementsFailed ?? 0)).toBe(6); +}); + +test("deferred fences in a module's top-level code report as running at startup", () => { + const { coverage, sourceTexts } = analyze(fixture("startup-fences.js")); + expect( + (coverage.runtimeFences ?? []).map((d) => [d.code, d.loc.start, d.startup === true]), + ).toEqual([ + ["SC1090", expect.any(Number), true], + ["SC1090", expect.any(Number), false], + ]); + const out = report(fixture("startup-fences.js")); + expect(out).toMatch( + /runs at startup {3}1 site .*\n.*instanceof against this native constructor\n\s+at tests\/coverage-fixtures\/startup-fences\.js:5:13/, + ); + expect(out).toMatch( + /deferred to runtime {3}1 site .*\n.*instanceof against this native constructor\n\s+at tests\/coverage-fixtures\/startup-fences\.js:7:10/, + ); + const envelope = coverageEnvelope(coverage, { compilerVersion: "test", sourceTexts }); + expect(envelope.diagnostics.map((d) => [d.scope, d.line, d.startup])).toEqual([ + ["deferred", 5, true], + ["deferred", 7, undefined], + ]); +}); + test("a failed unreached JavaScript statement is always explained", () => { const { coverage } = analyze(fixture("unreached-js-fence.js")); const failed = From 44efd4b55bf1fb0b4500699aa5134fd829ef1c18 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 05:49:11 -0500 Subject: [PATCH 007/197] Reuse the cached build after whitespace and comment edits --- .../compiler/src/frontend/input-tracker.ts | 52 +-- .../src/frontend/token-equivalence.test.ts | 49 +++ .../src/frontend/token-equivalence.ts | 370 ++++++++++++++++++ packages/compiler/src/native/prepare.test.ts | 60 ++- packages/compiler/src/native/prepare.ts | 80 +++- 5 files changed, 564 insertions(+), 47 deletions(-) create mode 100644 packages/compiler/src/frontend/token-equivalence.test.ts create mode 100644 packages/compiler/src/frontend/token-equivalence.ts diff --git a/packages/compiler/src/frontend/input-tracker.ts b/packages/compiler/src/frontend/input-tracker.ts index fa2e92568..ed2f7b176 100644 --- a/packages/compiler/src/frontend/input-tracker.ts +++ b/packages/compiler/src/frontend/input-tracker.ts @@ -410,36 +410,38 @@ export function frontendInputsSemanticallyMatch( exclusions: FrontendInputExclusions = {}, ): FrontendSemanticMatch | null { if (!validFrontendInputSnapshot(snapshot)) return null; - const sourceProbes = new Map( - snapshot.probes - .filter((probe): probe is Extract => probe.op === "file") - .map((probe) => [probe.path, probe]), - ); + const sourceProbes = new Map>(); + for (const probe of snapshot.probes) if (probe.op === "file") sourceProbes.set(probe.path, probe); for (const [path, previous] of previousSources) { if (sourceProbes.get(path)?.digest !== frontendSourceDigest(previous)) return null; } const changed: FrontendSemanticChange[] = []; const currentSources = new Map(); - const adjusted: FrontendInputSnapshot = { - ...snapshot, - probes: snapshot.probes.map((probe) => { - if (probe.op !== "file") return probe; - const previous = previousSources.get(probe.path); - if (previous === undefined || frontendSourceDigest(previous) !== probe.digest) return probe; - let current: string; - try { - current = readFileSync(probe.path, "utf8"); - } catch { - return probe; - } - const currentDigest = frontendSourceDigest(current); - currentSources.set(probe.path, current); - if (currentDigest === probe.digest) return probe; - if (!isSemanticEquivalent(probe.path, previous, current)) return probe; - changed.push({ path: probe.path, previous, current }); - return { ...probe, digest: currentDigest }; - }), - }; + const probes: FrontendInputProbe[] = []; + for (const probe of snapshot.probes) { + const previous = probe.op === "file" ? previousSources.get(probe.path) : undefined; + if (probe.op !== "file" || previous === undefined) { + probes.push(probe); + continue; + } + let current: string; + try { + current = readFileSync(probe.path, "utf8"); + } catch { + return null; + } + const currentDigest = frontendSourceDigest(current); + currentSources.set(probe.path, current); + if (currentDigest === probe.digest) { + probes.push(probe); + continue; + } + // One edit that is not equivalent already rules out the match. + if (!isSemanticEquivalent(probe.path, previous, current)) return null; + changed.push({ path: probe.path, previous, current }); + probes.push({ ...probe, digest: currentDigest }); + } + const adjusted: FrontendInputSnapshot = { ...snapshot, probes }; if (!frontendInputsStillMatch(adjusted, exclusions)) return null; for (const [path, previous] of previousSources) { if (!currentSources.has(path)) { diff --git a/packages/compiler/src/frontend/token-equivalence.test.ts b/packages/compiler/src/frontend/token-equivalence.test.ts new file mode 100644 index 000000000..2c7b8ea0a --- /dev/null +++ b/packages/compiler/src/frontend/token-equivalence.test.ts @@ -0,0 +1,49 @@ +import { expect, test } from "vitest"; +import { tokenEquivalentSource } from "./token-equivalence.js"; + +const equivalent = (previous: string, current: string, path = "/src/a.ts") => + tokenEquivalentSource(path, previous, current); + +test("whitespace and plain comments are trivia", () => { + expect(equivalent("const a = 1;\n", "const a = 1; // one\n")).toBe(true); + expect(equivalent("f(a,b);\n", "f( a, /* first */ b );\n")).toBe(true); + expect(equivalent("const s = `x${a}y`;\n", "const s = `x${ a /* the value */ }y`;\n")).toBe(true); +}); + +test("tokens, literals and line breaks that change semicolon insertion are not", () => { + expect(equivalent("const a = 1;\n", "const a = 2;\n")).toBe(false); + expect(equivalent('const s = "a // b";\n', 'const s = "a // c";\n')).toBe(false); + expect(equivalent("const s = `a // ${b} /* c */`;\n", "const s = `a // ${b} /* d */`;\n")).toBe( + false, + ); + // `return` followed by a line break returns undefined. + expect(equivalent("return a;\n", "return\na;\n")).toBe(false); + expect(equivalent("return a;\n", "return /*\n*/ a;\n")).toBe(false); +}); + +test("comment-like text inside regular expressions is code", () => { + expect(equivalent("const r = /[//]/; x = 1;\n", "const r = /[//]/; x = 2;\n")).toBe(false); + expect(equivalent("if (a) /\\/*/.test(b); c = 1;\n", "if (a) /\\/*/.test(b); c = 2;\n")).toBe( + false, + ); + expect(equivalent("const q = a / b; // ratio\n", "const q = a / b; // the ratio\n")).toBe(true); + expect(equivalent("return /a/; // one\n", "return /a/; // two\n")).toBe(true); +}); + +test("comments that can affect compilation are kept", () => { + expect(equivalent("/** @type {number} */ let a;\n", "/** @type {string} */ let a;\n")).toBe( + false, + ); + expect(equivalent("// @ts-ignore\nf();\n", "f();\n")).toBe(false); + expect(equivalent('/// \n', "\n")).toBe(false); + // Every comment is kept in JavaScript, whose types come from JSDoc. + expect(equivalent("let a; // one\n", "let a; // two\n", "/src/a.js")).toBe(false); +}); + +test("text the scanner cannot classify is never equivalent", () => { + expect(equivalent("x = {} / 2; // a\n", "x = {} / 2; // b\n")).toBe(false); + expect(equivalent("++/a/.lastIndex; // a\n", "++/a/.lastIndex; // b\n")).toBe(false); + expect(equivalent("const \\u0061 = 1; // a\n", "const \\u0061 = 1; // b\n")).toBe(false); + expect(equivalent("const a = ; // a\n", "const a = ; // b\n", "/src/a.tsx")).toBe(false); + expect(equivalent('const s = "open\n', 'const s = "open // \n')).toBe(false); +}); diff --git a/packages/compiler/src/frontend/token-equivalence.ts b/packages/compiler/src/frontend/token-equivalence.ts new file mode 100644 index 000000000..6387a0e4b --- /dev/null +++ b/packages/compiler/src/frontend/token-equivalence.ts @@ -0,0 +1,370 @@ +/* Token equivalence of two versions of a source file, for frontend cache + * reuse after edits that touch only comments and whitespace. + * + * Two versions are equivalent when they have the same tokens, in the same + * order, with line breaks between the same pairs of tokens (automatic + * semicolon insertion depends on them), and differ only in whitespace and + * in comments that cannot affect compilation. The scanner answers "not + * equivalent" whenever it cannot classify text with certainty, so a wrong + * answer can only cost a cache hit: + * + * - Every comment in a file that is not TypeScript (`.ts`, `.mts`, `.cts`) + * is kept as a token: JavaScript types come from JSDoc. TypeScript + * comments are kept when they start with `///` (references and + * directives) or contain `@` or `#` (checker directives, JSDoc tags, and + * annotations lowering reads). + * - A `/` starts a regular expression or divides according to the token + * before it. After `)` it divides unless the parenthesis closes an `if`, + * `while`, `for`, or `with` header. After `}`, `++` or `--` the scanner + * cannot tell which, so it gives up, as it does for escaped identifiers, + * which can spell keywords. + * - Template literals track their `${` substitutions on a brace stack. + * - JSX files, unterminated strings, comments, templates and regular + * expressions, and any line terminator outside ASCII give up. + * + * Plain character scans keep the scanner free of Unicode property escapes: + * any non-ASCII code unit outside strings, templates, regular expressions + * and comments belongs to the token around it, so an edit there compares + * literally. */ + +interface Token { + text: string; + comment: boolean; + lineBreakBefore: boolean; +} + +/** Keywords after which `/` starts a regular expression. */ +const EXPRESSION_KEYWORDS = new Set([ + "return", + "typeof", + "instanceof", + "in", + "of", + "new", + "delete", + "void", + "throw", + "case", + "do", + "else", + "yield", + "await", + "default", + "extends", +]); + +/** Keywords whose parenthesized header is followed by a statement. */ +const HEADER_KEYWORDS = new Set(["if", "while", "for", "with"]); + +const PUNCTUATORS = [ + ">>>=", + "...", + "===", + "!==", + "**=", + "<<=", + ">>=", + ">>>", + "&&=", + "||=", + "??=", + "=>", + "==", + "!=", + "<=", + ">=", + "&&", + "||", + "??", + "?.", + "++", + "--", + "+=", + "-=", + "*=", + "%=", + "&=", + "|=", + "^=", + "**", + "<<", + ">>", +]; + +function isTypeScript(path: string): boolean { + return path.endsWith(".ts") || path.endsWith(".mts") || path.endsWith(".cts"); +} + +function isIdentifierCode(code: number): boolean { + return ( + (code >= 97 && code <= 122) || + (code >= 65 && code <= 90) || + (code >= 48 && code <= 57) || + code === 36 || + code === 95 || + code === 92 || + code > 127 + ); +} + +function isLineBreak(code: number): boolean { + return code === 10 || code === 13; +} + +function isSpace(code: number): boolean { + return code === 32 || code === 9 || code === 11 || code === 12 || isLineBreak(code); +} + +/** The tokens of `source`, or null when the scanner cannot classify it. */ +function scan(path: string, source: string): Token[] | null { + if (path.endsWith(".tsx") || path.endsWith(".jsx")) return null; + const typeScript = isTypeScript(path); + const tokens: Token[] = []; + // Brace kinds: true for a template substitution, false otherwise. + const braces: boolean[] = []; + // Paren kinds: true for a statement header. + const parens: boolean[] = []; + let lineBreakBefore = false; + // The previous token, and whether a `)` closed a statement header. + const last: { token: string | null } = { token: null }; + let closedHeader = false; + const push = (text: string, comment = false): void => { + tokens.push({ text, comment, lineBreakBefore }); + lineBreakBefore = false; + if (!comment) last.token = text; + }; + let index = 0; + const length = source.length; + // A shebang is one token. + if (source.startsWith("#!")) { + while (index < length && !isLineBreak(source.charCodeAt(index))) index++; + push(source.slice(0, index)); + } + /** Scans template text from `index` (after a backtick or a closing + * substitution brace) through the closing backtick or `${`. */ + const template = (start: number): boolean => { + let position = index; + while (position < length) { + const code = source.charCodeAt(position); + if (code === 92) { + position += 2; + continue; + } + if (code === 96) { + position++; + index = position; + push(source.slice(start, position)); + closedHeader = false; + return true; + } + if (code === 36 && source.charCodeAt(position + 1) === 123) { + position += 2; + index = position; + push(source.slice(start, position)); + braces.push(true); + last.token = "${"; + return true; + } + position++; + } + return false; + }; + while (index < length) { + const code = source.charCodeAt(index); + if (isSpace(code)) { + if (isLineBreak(code)) lineBreakBefore = true; + index++; + continue; + } + if (code > 127 && (code === 0x2028 || code === 0x2029 || code === 0xa0 || code === 0xfeff)) + return null; + const next = source.charCodeAt(index + 1); + // Comments. + if (code === 47 && (next === 47 || next === 42)) { + const start = index; + if (next === 47) { + while (index < length && !isLineBreak(source.charCodeAt(index))) index++; + } else { + const end = source.indexOf("*/", index + 2); + if (end < 0) return null; + index = end + 2; + } + const text = source.slice(start, index); + const semantic = + !typeScript || text.startsWith("///") || text.includes("@") || text.includes("#"); + if (semantic) push(text, true); + else if (/[\r\n]/.test(text)) lineBreakBefore = true; + continue; + } + // Strings. + if (code === 34 || code === 39) { + const start = index; + index++; + while (index < length) { + const inner = source.charCodeAt(index); + if (inner === 92) { + index += 2; + continue; + } + if (isLineBreak(inner)) return null; + index++; + if (inner === code) break; + if (index >= length) return null; + } + if (source.charCodeAt(index - 1) !== code || index - start < 2) return null; + push(source.slice(start, index)); + closedHeader = false; + continue; + } + // Templates. + if (code === 96) { + const start = index; + index++; + if (!template(start)) return null; + continue; + } + // Identifiers, keywords and numbers. + if (isIdentifierCode(code) || (code === 46 && next >= 48 && next <= 57)) { + const start = index; + index++; + while (index < length) { + const inner = source.charCodeAt(index); + if (isIdentifierCode(inner) || inner === 46) { + // A dot continues only a number (`1.5`), not member access. + if (inner === 46) { + const first = source.charCodeAt(start); + if (!((first >= 48 && first <= 57) || first === 46)) break; + } + index++; + continue; + } + // Exponent signs (`1e-5`). + if ( + (inner === 43 || inner === 45) && + (source.charCodeAt(index - 1) === 101 || source.charCodeAt(index - 1) === 69) + ) { + const first = source.charCodeAt(start); + if ((first >= 48 && first <= 57) || first === 46) { + index++; + continue; + } + } + break; + } + const word = source.slice(start, index); + // An escaped identifier can spell a keyword (`\u0072eturn`). + if (word.includes("\\")) return null; + push(word); + closedHeader = false; + continue; + } + // Division or a regular expression. + if (code === 47) { + const before = last.token; + let regex: boolean; + if (before === null) regex = true; + else if (before === "}") return null; + else if (before === ")") regex = closedHeader; + // `++/re/.lastIndex` is valid, `x++ / 2` too. + else if (before === "++" || before === "--") return null; + else if (before === "]") regex = false; + else if (/^[A-Za-z0-9_$\\]/.test(before) || before.charCodeAt(0) > 127) + regex = EXPRESSION_KEYWORDS.has(before); + else if (before.startsWith("'") || before.startsWith('"') || before.endsWith("`")) + regex = false; + else if (before.startsWith("/") && before.length > 1 && before !== "/=") regex = false; + else regex = true; + if (regex) { + const start = index; + index++; + let inClass = false; + while (index < length) { + const inner = source.charCodeAt(index); + if (isLineBreak(inner)) return null; + if (inner === 92) { + index += 2; + continue; + } + if (inner === 91) inClass = true; + else if (inner === 93) inClass = false; + else if (inner === 47 && !inClass) break; + index++; + } + if (index >= length) return null; + index++; + while (index < length && isIdentifierCode(source.charCodeAt(index))) index++; + push(source.slice(start, index)); + closedHeader = false; + continue; + } + const text = next === 61 ? "/=" : "/"; + index += text.length; + push(text); + closedHeader = false; + continue; + } + // Brackets. + if (code === 40) { + parens.push(last.token !== null && HEADER_KEYWORDS.has(last.token)); + index++; + push("("); + closedHeader = false; + continue; + } + if (code === 41) { + const header = parens.pop() ?? false; + index++; + push(")"); + closedHeader = header; + continue; + } + if (code === 123) { + braces.push(false); + index++; + push("{"); + closedHeader = false; + continue; + } + if (code === 125) { + if (braces.pop() === true) { + const start = index; + index++; + if (!template(start)) return null; + continue; + } + index++; + push("}"); + closedHeader = false; + continue; + } + // Other punctuators. + let text = source[index]!; + for (const punctuator of PUNCTUATORS) + if (source.startsWith(punctuator, index)) { + text = punctuator; + break; + } + index += text.length; + push(text); + closedHeader = false; + } + if (braces.length !== 0 || parens.length !== 0) return null; + return tokens; +} + +/** Whether `current` differs from `previous` only in whitespace and in + * comments that cannot affect compilation. */ +export function tokenEquivalentSource(path: string, previous: string, current: string): boolean { + if (previous === current) return true; + const left = scan(path, previous); + if (left === null) return false; + const right = scan(path, current); + if (right === null || left.length !== right.length) return false; + for (let index = 0; index < left.length; index++) { + const a = left[index]!; + const b = right[index]!; + if (a.text !== b.text || a.comment !== b.comment) return false; + if (index > 0 && a.lineBreakBefore !== b.lineBreakBefore) return false; + } + return true; +} diff --git a/packages/compiler/src/native/prepare.test.ts b/packages/compiler/src/native/prepare.test.ts index 7e22450da..35e866d6a 100644 --- a/packages/compiler/src/native/prepare.test.ts +++ b/packages/compiler/src/native/prepare.test.ts @@ -16,6 +16,22 @@ afterEach(() => { for (const path of scratch.splice(0)) rmSync(path, { recursive: true, force: true }); }); +function fixtureToolchain(): NativeToolchain { + const target = nativeCodegenTarget()!; + return { + compilerVersion: "test", + target, + helper: target.helper, + helperExecutable: "unused", + helperPackageRoot: "unused", + runtimePackRoot: "unused", + linker: "unused", + linkerArgs: [], + dsymutil: "unused", + ts7Executable: ts7Executable(), + }; +} + function fixture() { const temporary = mkdtempSync(join(tmpdir(), "scriptc-native-frontend-")); scratch.push(temporary); @@ -32,19 +48,7 @@ function fixture() { }), ); writeFileSync(join(root, "package.json"), '{"type":"module"}'); - const target = nativeCodegenTarget()!; - const toolchain: NativeToolchain = { - compilerVersion: "test", - target, - helper: target.helper, - helperExecutable: "unused", - helperPackageRoot: "unused", - runtimePackRoot: "unused", - linker: "unused", - linkerArgs: [], - dsymutil: "unused", - ts7Executable: ts7Executable(), - }; + const toolchain = fixtureToolchain(); const cache = new NativeCache(join(temporary, "cache")); const options = { outPath: join(root, ".scriptc", "main"), @@ -79,6 +83,36 @@ test("warm frontend reuse survives output creation and invalidates imported sour expect(f.loads()).toBe(2); }); +test("comment and whitespace edits reuse the frontend output a fresh build reproduces", () => { + const f = fixture(); + const first = f.prepare(); + expect(first.ok, JSON.stringify(first)).toBe(true); + if (!first.ok) return; + const commented = '// The exported value.\nexport const value = /* unchanged */ "first";\n'; + writeFileSync(f.dependency, commented); + const reused = f.prepare(); + expect(f.loads()).toBe(1); + expect(reused.ok).toBe(true); + if (!reused.ok) return; + expect(reused.llvm).toBe(first.llvm); + expect(new Map(reused.sources).get(f.dependency)).toBe(commented); + // The reused output is the one the edited sources lower to. + const fresh = prepareNativeExecutable( + f.entry, + f.options, + null, + fixtureToolchain(), + (path, npmStatic, externalTypes) => runFrontend(path, loadProgram, npmStatic, externalTypes), + new NativeCache(join(f.root, "fresh-cache")), + ); + expect(fresh.ok && fresh.llvm).toBe(first.llvm); + // The refreshed entry still notices a later code edit. + writeFileSync(f.dependency, 'export const value = "second";\n'); + const edited = f.prepare(); + expect(f.loads()).toBe(2); + expect(edited.ok && edited.llvm).not.toBe(first.llvm); +}); + test("structurally invalid cached features cause a rebuild even with a valid digest", () => { const f = fixture(); expect(f.prepare().ok).toBe(true); diff --git a/packages/compiler/src/native/prepare.ts b/packages/compiler/src/native/prepare.ts index 6b0a2fea5..53b22cfaa 100644 --- a/packages/compiler/src/native/prepare.ts +++ b/packages/compiler/src/native/prepare.ts @@ -16,6 +16,7 @@ import type { FfiProfile } from "../ffi/ffi-manifest.js"; import type { FrontendFactory } from "../frontend/pipeline.js"; import { FrontendInputTracker, + frontendInputsSemanticallyMatch, frontendInputsStillMatch, validFrontendInputSnapshot, type FrontendInputSnapshot, @@ -23,6 +24,8 @@ import { trackedReadFile, } from "../frontend/input-tracker.js"; import { provenanceSources } from "../frontend/provenance-registry.js"; +import { tokenEquivalentSource } from "../frontend/token-equivalence.js"; +import { llvmBytes, wtf8Bytes } from "../backend/literals.js"; import { prepareLibrary } from "../library/prepare.js"; import type { LibraryProfile } from "../library/library-profile.js"; import type { IrModule } from "../ir/ir.js"; @@ -90,18 +93,68 @@ function publishFrontendCache( if (length <= NATIVE_STRING_BYTES) cache.writeParts("frontend", key, parts); } +/** The cached input when every frontend input still matches, else the + * entry the inputs no longer match, for reuse after equivalent edits. */ function readFrontendCache( cache: NativeCache, key: string, exclusions: FrontendInputExclusions, -): NativeExecutableInput | null { +): { input: NativeExecutableInput } | { stale: FrontendCacheEntry } | null { // The entry is decoded as one string; larger entries are misses. const bytes = cache.read("frontend", key, NATIVE_STRING_BYTES); if (bytes === null) return null; const cached: unknown = JSON.parse(bytes.toString("utf8")); - if (!validCachedInput(cached) || !frontendInputsStillMatch(cached.probes, exclusions)) + if (!validCachedInput(cached)) return null; + if (!frontendInputsStillMatch(cached.probes, exclusions)) return { stale: cached }; + return { input: { ...cached.input, llvm: cached.llvm, ir: cached.ir } }; +} + +/** The stale entry's input when every changed source is token-equivalent to + * the text the entry was built from (edits to whitespace and to comments + * that cannot affect compilation; see token-equivalence.ts). Equivalent + * sources lower to the same IR apart from source locations, which only dev + * builds' debug information and the serialized IR carry. Diagnostics carry + * locations too, so an entry that reported warnings is rebuilt. */ +function equivalentFrontendInput( + stale: FrontendCacheEntry, + options: CompileRequestOptions, + exclusions: FrontendInputExclusions, +): { input: NativeExecutableInput; probes: FrontendInputSnapshot } | null { + if ( + options.optimization === "dev" || + options.sanitize === true || + options.emitIr === true || + stale.ir !== null || + (stale.input.warnings?.length ?? 0) > 0 + ) return null; - return { ...cached.input, llvm: cached.llvm, ir: cached.ir }; + const match = frontendInputsSemanticallyMatch( + stale.probes, + new Map(stale.input.sources), + tokenEquivalentSource, + exclusions, + ); + if (match === null || match.changed.length === 0) return null; + // Runtime fences and unsupported-construct messages name their source as + // `[ at :]`, and an equivalent edit can move the line. + if ( + match.changed.some(({ path }) => + stale.llvm.includes(` at ${llvmBytes(wtf8Bytes(path), false)}:`), + ) + ) + return null; + return { + input: { + ...stale.input, + llvm: stale.llvm, + ir: null, + sources: stale.input.sources.map(([path, text]) => [ + path, + match.currentSources.get(path) ?? text, + ]), + }, + probes: match.snapshot, + }; } function implementationIdentity(toolchain: NativeToolchain): string { @@ -260,10 +313,19 @@ export function prepareNativeExecutable( .sort(([a], [b]) => a.localeCompare(b)), }), ); - const input = readFrontendCache(cache, key, exclusions); - if (input !== null) { + const cached = readFrontendCache(cache, key, exclusions); + if (cached !== null && "input" in cached) { timing("frontend-cache-hit"); - return input; + return cached.input; + } + const equivalent = + cached === null || outputKind === "ir" + ? null + : equivalentFrontendInput(cached.stale, options, exclusions); + if (equivalent !== null) { + publishFrontendCache(cache, key, equivalent.probes, equivalent.input); + timing("frontend-cache-equivalent"); + return equivalent.input; } } catch { key = null; @@ -373,10 +435,10 @@ export function prepareNativeLibrary( .sort(([a], [b]) => a.localeCompare(b)), }), ); - const input = readFrontendCache(cache, key, exclusions); - if (input !== null) { + const cached = readFrontendCache(cache, key, exclusions); + if (cached !== null && "input" in cached) { timing("frontend-cache-hit"); - return input; + return cached.input; } } catch { key = null; From cda76716a0557ef05232967ce7721ea8a1dd1d25 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 05:51:17 -0500 Subject: [PATCH 008/197] Enumerate each record in the key order its object literal wrote Record shapes are interned by field set, and their key order was one table per shape. When object literals of one shape wrote the keys in different orders (`{ id, name }` and `{ name, id }`), every value enumerated in the first type's order, and anonymous literal types got no warning. - The key-order witness pass gives such a shape its orders (IrRecordShape.orderVariants, the declared table first) and a hidden number field, %order, that each value sets to the index of its order: a literal of explicit keys stores its own order, a copy from a shape with one order stores that order, and spreads, rest bindings and copies of values with orders read their source's index. Allocation zeroes the field, so checked conversions and runtime producers keep the table as before, and clones copy it. - A write of an optional key a value lacks moves the key to the end of that value's order, as in Node: the orders include each order such writes reach, and the write site updates the index first, evaluating a receiver other than a binding once. - Object.keys, Object.values, Object.entries, for-in, util.inspect, JSON.stringify, the record-to-dynamic conversion (structuredClone, postMessage, untyped views) and live record views branch on the index. The dynamic conversion and checked conversions leave the field out. - A shape keeps its table, and SC6002 still reports its literals, when a construction's order cannot be traced: a spread of a value of unknown order, a clone that adds an optional key, a copy whose source is not a binding, or more than 16 orders. Corpus 7122 to 7128 check each surface against Node. --- docs/content/docs/coverage.mdx | 2 +- .../record-identity-and-property-order.mdx | 7 +- packages/compiler/src/backend/llvm/dyn.ts | 149 +++--- .../compiler/src/backend/llvm/expr-context.ts | 3 + .../src/backend/llvm/expr-stream-bridges.ts | 131 +++-- .../compiler/src/backend/llvm/key-orders.ts | 49 ++ packages/compiler/src/backend/llvm/walkers.ts | 139 +++--- .../src/frontend/lowering/divergences.ts | 5 + .../lowering/key-order-witness.test.ts | 118 ++++- .../frontend/lowering/key-order-witness.ts | 471 +++++++++++++++++- .../src/frontend/lowering/lower-calls.ts | 24 +- .../src/frontend/lowering/lower-inspect.ts | 12 +- packages/compiler/src/frontend/type-mapper.ts | 17 +- .../src/ir/function-references.test.ts | 1 + packages/compiler/src/ir/ir.ts | 18 + .../tests/corpus/7122-key-order-json.ts.json | 6 + ...7123-key-order-keys-values-entries.ts.json | 6 + .../corpus/7124-key-order-for-in.ts.json | 6 + .../corpus/7125-key-order-inspect.ts.json | 6 + .../corpus/7126-key-order-copies.ts.json | 6 + .../corpus/7127-key-order-dynamic.ts.json | 6 + .../corpus/7128-key-order-late-writes.ts.json | 6 + tests/corpus/7122-key-order-json.ts | 31 ++ .../7123-key-order-keys-values-entries.ts | 19 + tests/corpus/7124-key-order-for-in.ts | 15 + tests/corpus/7125-key-order-inspect.ts | 12 + tests/corpus/7126-key-order-copies.ts | 19 + tests/corpus/7127-key-order-dynamic.ts | 30 ++ tests/corpus/7128-key-order-late-writes.ts | 27 + .../corpus-costs/7122-key-order-json.ts.json | 1 + ...7123-key-order-keys-values-entries.ts.json | 1 + .../7124-key-order-for-in.ts.json | 1 + .../7125-key-order-inspect.ts.json | 1 + .../7126-key-order-copies.ts.json | 1 + .../7127-key-order-dynamic.ts.json | 1 + .../7128-key-order-late-writes.ts.json | 1 + 36 files changed, 1147 insertions(+), 201 deletions(-) create mode 100644 packages/compiler/src/backend/llvm/key-orders.ts create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7122-key-order-json.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7123-key-order-keys-values-entries.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7124-key-order-for-in.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7125-key-order-inspect.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7126-key-order-copies.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7127-key-order-dynamic.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7128-key-order-late-writes.ts.json create mode 100644 tests/corpus/7122-key-order-json.ts create mode 100644 tests/corpus/7123-key-order-keys-values-entries.ts create mode 100644 tests/corpus/7124-key-order-for-in.ts create mode 100644 tests/corpus/7125-key-order-inspect.ts create mode 100644 tests/corpus/7126-key-order-copies.ts create mode 100644 tests/corpus/7127-key-order-dynamic.ts create mode 100644 tests/corpus/7128-key-order-late-writes.ts create mode 100644 tests/harness/corpus-costs/7122-key-order-json.ts.json create mode 100644 tests/harness/corpus-costs/7123-key-order-keys-values-entries.ts.json create mode 100644 tests/harness/corpus-costs/7124-key-order-for-in.ts.json create mode 100644 tests/harness/corpus-costs/7125-key-order-inspect.ts.json create mode 100644 tests/harness/corpus-costs/7126-key-order-copies.ts.json create mode 100644 tests/harness/corpus-costs/7127-key-order-dynamic.ts.json create mode 100644 tests/harness/corpus-costs/7128-key-order-late-writes.ts.json diff --git a/docs/content/docs/coverage.mdx b/docs/content/docs/coverage.mdx index ae36f736d..258e55ac8 100644 --- a/docs/content/docs/coverage.mdx +++ b/docs/content/docs/coverage.mdx @@ -95,7 +95,7 @@ Coverage and `scriptc build` report the same diagnostic codes. Common ones inclu Some code compiles but can behave differently than under Node.js. Coverage reports the cases it can detect as `SC6xxx` warnings, and `scriptc build` prints the same warnings after a successful build. They never fail a build, but `--fail-on=divergences` can fail a CI job on them. - `SC6001`: a record passed to, stored as, asserted as, or stored as an array element of a record type with different properties is copied, and the program then writes through the copy or compares it by identity. -- `SC6002`: values of one record type are built in different key orders, so the type keeps its declaration order, an object literal's key order differs from it, and the program observes the order through `JSON.stringify`, `Object.keys`, `Object.values`, `Object.entries`, `for...in`, or console output. +- `SC6002`: values of one record type are built in key orders scriptc cannot track per value, so the type keeps its declaration order, an object literal's key order differs from it, and the program observes the order through `JSON.stringify`, `Object.keys`, `Object.values`, `Object.entries`, `for...in`, or console output. - `SC6003`: `localeCompare` compares a string literal outside the built-in Latin-script collation. - `SC6004`: a call passes a class instance to a function that casts it to an unrelated class. diff --git a/docs/content/limitations/record-identity-and-property-order.mdx b/docs/content/limitations/record-identity-and-property-order.mdx index 0381803f6..cc16f74d5 100644 --- a/docs/content/limitations/record-identity-and-property-order.mdx +++ b/docs/content/limitations/record-identity-and-property-order.mdx @@ -10,9 +10,10 @@ A type whose values are only read field by field, from fields the class does not Object spread of a class instance is refused. Dense array literals created directly in checked storage preserve identity, and supported native handles pass by reference. -`Object.keys`, `Object.values`, `Object.entries`, `for...in`, `JSON.stringify`, and `util.inspect` enumerate a record's keys in one order per record type rather than per object. scriptc chooses that order from the order in which the program's object literals create the keys, follows object spread, rest destructuring, and record conversions back to the values they copy, and places an optional key that the program assigns after construction after the keys the object was built with. -The order matches Node.js when every value of the type is created in the same order. -When values of one record type are created in different orders, including values of two types with the same properties and property types, or when a key is deleted and assigned again, the type keeps the declaration order of its first type instead, and `SC6002` reports object literals whose order differs where the program observes it. +`Object.keys`, `Object.values`, `Object.entries`, `for...in`, `JSON.stringify`, and `util.inspect` enumerate a record's keys in the order the object created them, as in Node.js. +scriptc chooses one order per record type from the order in which the program's object literals create the keys, follows object spread, rest destructuring, and record conversions back to the values they copy, and places an optional key that the program assigns after construction after the keys the object was built with. +When the object literals of one record type create its keys in different orders, including literals of two types with the same properties and property types, each value records the order its literal used: copies made by object spread, rest destructuring, and record conversions keep their source's order, and assigning an optional key the object lacks, including after `delete`, moves the key last. +A type keeps the declaration order of its first type, and `SC6002` reports object literals whose order differs where the program observes it, when one of its values gets its keys from a spread whose source order is not known, from a spread that also sets an optional key the source lacks, or from a copy of a value scriptc cannot trace, or when the type's values can take more than 16 orders. A record converted from a checked value, such as `const rows: { score: number }[] = JSON.parse(text)`, keeps the properties its type does not declare. `JSON.stringify`, `Object.keys`, `Object.values`, `Object.entries`, `util.inspect`, and untyped views of the record see them, together with the declared fields, in the source object's key order. diff --git a/packages/compiler/src/backend/llvm/dyn.ts b/packages/compiler/src/backend/llvm/dyn.ts index 6911ef98b..c48468886 100644 --- a/packages/compiler/src/backend/llvm/dyn.ts +++ b/packages/compiler/src/backend/llvm/dyn.ts @@ -1,3 +1,4 @@ +import { emitPerKeyOrder } from "./key-orders.js"; import { typedRefConstructor } from "./shapes.js"; import { bindLiveArrayOps } from "./expr-stream-bridges.js"; import { BYTES_ELEM_NUM, SCR_ARR_FIELD, f64Lit, sizeToDouble } from "./common.js"; @@ -37,6 +38,8 @@ import { accessorSlotProp, DYN_HANDLE_KINDS, isDirentShape, + KEY_ORDER_FIELD, + shapeKeyOrders, isDynTypedRefType, isRefCounted, isUnitType, @@ -1044,9 +1047,10 @@ export class LlDyn { } B.condBr(readable, l0, fail); B.startBlock(l0); - // dyn ('unknown') fields match ANY value, present or missing. + // dyn ('unknown') fields match ANY value, present or missing. The + // order index is no key. for (const f of shape.fields) { - if (f.type.kind === "dyn") continue; + if (f.type.kind === "dyn" || f.name === KEY_ORDER_FIELD) continue; const { value: m, release } = this.memberRead(B, f.name, f.type); const lNext = B.newLabel("dm.n"); this.pendingBail( @@ -1228,9 +1232,10 @@ export class LlDyn { return ( shape.fields.every( (f) => - !f.name.startsWith("%") && - f.name !== "__proto__" && - this.jsonSchemaSupported(f.type, visiting, true), + f.name === KEY_ORDER_FIELD || + (!f.name.startsWith("%") && + f.name !== "__proto__" && + this.jsonSchemaSupported(f.type, visiting, true)), ) && (!index || this.jsonSchemaSupported(index, visiting)) ); @@ -1315,7 +1320,9 @@ export class LlDyn { const base = host.nullableUnions.fieldBase(t.shapeId); const index = new Map(shape.fields.map((f, i) => [f.name, i + base])); // The runtime names kept-source keys by this index (record-sources.ts). - const order = recordSourceFieldOrder(shape); + // The order index is no key: parsed values keep the table's order + // and their source's own order. + const order = recordSourceFieldOrder(shape).filter((name) => name !== KEY_ORDER_FIELD); if (recordMayKeepSource(shape)) this.keepsRecordSources = true; const fieldTy = `{ ptr, ${S}, ${S}, ptr }`; const entries = order.map((fieldName) => { @@ -2058,7 +2065,10 @@ export class LlDyn { } B.line(`%r0 = call ptr @${mangleRecordNew(t.shapeId)}()`); if (recursive) this.recordBuilt(B, name, "%r0"); + // The order index stays 0: the source's own order is kept below + // when it differs from the table. for (const f of shape.fields) { + if (f.name === KEY_ORDER_FIELD) continue; const { value: m, release } = this.memberRead(B, f.name, f.type); this.pendingBail( B, @@ -2789,70 +2799,79 @@ export class LlDyn { host.declare(`declare ptr @scr_dyn_new_obj()`); B.line(`${d} = call ptr @scr_dyn_new_obj()`); } - // Keys insert in DECLARED order (JS insertion order); internal - // '%'-fields follow so a record→dyn→record round trip keeps them. + // Keys insert in DECLARED order (JS insertion order), or in the + // value's own order for a shape with orderVariants; internal + // '%'-fields follow so a record→dyn→record round trip keeps them + // (all but the order index, which is no key). const byName = new Map(shape.fields.map((f) => [f.name, f])); // An accessor property is one key whose getter and setter are the // slot closures (see accessorSlotProp), in accessorOrder's place. - const order = shape.accessorOrder ?? shape.declaredOrder ?? shape.fields.map((f) => f.name); - const inOrder = new Set(order); - const accessors = new Set( - shape.accessorOrder === undefined - ? [] - : shape.fields.flatMap((f) => { - const slot = accessorSlotProp(f.name); - return slot === null ? [] : [slot.prop]; - }), - ); - type DynField = { name: string; type: IrType } | { accessor: string }; - const dynFields: DynField[] = [ - ...order.flatMap((n) => { - if (accessors.has(n)) return [{ accessor: n } as DynField]; - const f = byName.get(n); - return f === undefined ? [] : [f as DynField]; - }), - ...shape.fields.filter( - (f) => - !inOrder.has(f.name) && !(accessors.size > 0 && accessorSlotProp(f.name) !== null), - ), - ]; - for (const entry of dynFields) { - if ("accessor" in entry) { - host.declare( - `declare void @scr_dyn_obj_set_accessor(ptr, ptr, ${host.sizeType}, ptr, ptr)`, - ); - const boxed = (kind: "get" | "set"): string => { - const slot = byName.get(`%${kind}:${entry.accessor}`); - if (slot === undefined) return "null"; - const fv = loadFieldOf(slot.name, slot.type); - const conv = B.tmp(); - B.line( - `${conv} = call ptr @${this.toDynHelper(slot.type)}(${this.valTy(slot.type)} ${fv})`, - ); - return conv; - }; - const getter = boxed("get"); - const setter = boxed("set"); - B.line( - `call void @scr_dyn_obj_set_accessor(ptr ${d}, ptr ${host.cstr(entry.accessor)}, ${host.sizeType} ${Buffer.byteLength(entry.accessor, "utf8")}, ptr ${getter}, ptr ${setter}) ; ${llvmCommentText(entry.accessor)}`, - ); - continue; - } - const f = entry; - const klen = Buffer.byteLength(f.name, "utf8"); - const fv = loadFieldOf(f.name, f.type); - // An ABSENT optional field contributes no key. - const lSkip = f.type.kind === "union" ? this.skipIfAbsent(B, fv, f.type) : null; - const conv = B.tmp(); - B.line(`${conv} = call ptr @${this.toDynHelper(f.type)}(${this.valTy(f.type)} ${fv})`); - B.line( - `call void @scr_dyn_obj_set(ptr ${d}, ptr ${host.cstr(f.name)}, ${host.sizeType} ${klen}, ptr ${conv}) ; ${llvmCommentText(f.name)}`, + const orders = + shape.accessorOrder !== undefined ? [shape.accessorOrder] : shapeKeyOrders(shape); + const perOrder = (order: readonly string[]): void => { + const inOrder = new Set(order); + const accessors = new Set( + shape.accessorOrder === undefined + ? [] + : shape.fields.flatMap((f) => { + const slot = accessorSlotProp(f.name); + return slot === null ? [] : [slot.prop]; + }), ); - if (lSkip !== null) { - B.br(lSkip); - B.startBlock(lSkip); + type DynField = { name: string; type: IrType } | { accessor: string }; + const dynFields: DynField[] = [ + ...order.flatMap((n) => { + if (accessors.has(n)) return [{ accessor: n } as DynField]; + const f = byName.get(n); + return f === undefined ? [] : [f as DynField]; + }), + ...shape.fields.filter( + (f) => + !inOrder.has(f.name) && + f.name !== KEY_ORDER_FIELD && + !(accessors.size > 0 && accessorSlotProp(f.name) !== null), + ), + ]; + for (const entry of dynFields) { + if ("accessor" in entry) { + host.declare( + `declare void @scr_dyn_obj_set_accessor(ptr, ptr, ${host.sizeType}, ptr, ptr)`, + ); + const boxed = (kind: "get" | "set"): string => { + const slot = byName.get(`%${kind}:${entry.accessor}`); + if (slot === undefined) return "null"; + const fv = loadFieldOf(slot.name, slot.type); + const conv = B.tmp(); + B.line( + `${conv} = call ptr @${this.toDynHelper(slot.type)}(${this.valTy(slot.type)} ${fv})`, + ); + return conv; + }; + const getter = boxed("get"); + const setter = boxed("set"); + B.line( + `call void @scr_dyn_obj_set_accessor(ptr ${d}, ptr ${host.cstr(entry.accessor)}, ${host.sizeType} ${Buffer.byteLength(entry.accessor, "utf8")}, ptr ${getter}, ptr ${setter}) ; ${llvmCommentText(entry.accessor)}`, + ); + continue; + } + const f = entry; + const klen = Buffer.byteLength(f.name, "utf8"); + const fv = loadFieldOf(f.name, f.type); + // An ABSENT optional field contributes no key. + const lSkip = f.type.kind === "union" ? this.skipIfAbsent(B, fv, f.type) : null; + const conv = B.tmp(); + B.line(`${conv} = call ptr @${this.toDynHelper(f.type)}(${this.valTy(f.type)} ${fv})`); + B.line( + `call void @scr_dyn_obj_set(ptr ${d}, ptr ${host.cstr(f.name)}, ${host.sizeType} ${klen}, ptr ${conv}) ; ${llvmCommentText(f.name)}`, + ); + if (lSkip !== null) { + B.br(lSkip); + B.startBlock(lSkip); + } } - } + }; + if (orders.length === 1) perOrder(orders[0]!); + else emitPerKeyOrder(B, shape, base, "%v", perOrder); // A Dirent row takes Node's Dirent prototype (the type probes) and // its Symbol(type) key; %dtype stays readable but non-enumerable. if (isDirentShape(shape)) { diff --git a/packages/compiler/src/backend/llvm/expr-context.ts b/packages/compiler/src/backend/llvm/expr-context.ts index 9dd6bbcac..af1a9a486 100644 --- a/packages/compiler/src/backend/llvm/expr-context.ts +++ b/packages/compiler/src/backend/llvm/expr-context.ts @@ -32,6 +32,9 @@ export interface LlStreamTypedRefAdapter { arrayOps?: string; /** `@sym` of the ScrDynTypedRecordOps table for live record capsules. */ recordOps?: string; + /** For a shape with orderVariants: the defined function that binds a + * live record capsule the table of its value's key order. */ + recordBind?: string; /** `@sym` of the ScrDynTypedClassOps table for class instance capsules. */ classOps?: string; } diff --git a/packages/compiler/src/backend/llvm/expr-stream-bridges.ts b/packages/compiler/src/backend/llvm/expr-stream-bridges.ts index a0a0cf24a..b64204122 100644 --- a/packages/compiler/src/backend/llvm/expr-stream-bridges.ts +++ b/packages/compiler/src/backend/llvm/expr-stream-bridges.ts @@ -13,6 +13,8 @@ import { classFieldAttributesName, isClassOwnEnumerableFieldName, isDirentShape, + KEY_ORDER_FIELD, + shapeKeyOrders, isDynTypedRefType, isRefCounted, shapeHasAccessorSlots, @@ -44,6 +46,7 @@ import type { NullableRefField } from "./nullable-fields.js"; import { emitUnionPeek, emitUnionTag } from "./union-repr.js"; import { inlineUnionCopyLines } from "./inline-unions.js"; import { recordMayKeepSource } from "./record-sources.js"; +import { emitPerKeyOrder } from "./key-orders.js"; /** An owned (+1) union value for a nullable-pointer class field slot `p` * (the unit arm's immortal instance, or a fresh box around a retained @@ -1054,52 +1057,68 @@ export function streamTypedRefMaterializeAdapter( const out = B.tmp(); B.line(`${out} = call ptr @scr_dyn_new_obj()`); const byName = new Map(shape.fields.map((field) => [field.name, field])); - const order = shape.declaredOrder ?? shape.fields.map((field) => field.name); - const inOrder = new Set(order); - const fields = [ - ...order.map((name) => byName.get(name)).filter((field) => field !== undefined), - ...shape.fields.filter((field) => !inOrder.has(field.name)), - ]; + // A shape with orderVariants has one field list per key order; its + // order index is no key. + const orders = shapeKeyOrders(shape); + const fieldsOf = (order: readonly string[]): IrRecordShape["fields"][number][] => { + const inOrder = new Set(order); + return [ + ...order.map((name) => byName.get(name)).filter((field) => field !== undefined), + ...shape.fields.filter( + (field) => !inOrder.has(field.name) && field.name !== KEY_ORDER_FIELD, + ), + ]; + }; // Published before the field converters, like the array hooks below. // A Dirent row reads through its finished view (the prototype and // the non-enumerable %dtype). const dirent = isDirentShape(shape); - if (!dirent && liveRecordOpsEligible(fields)) { - adapter.recordOps = `@${snapshot}_rops`; - liveRecordOps(host, t, snapshot, fields); - } - for (const field of fields) { - const index = shape.fields.indexOf(field) + host.nullableUnions.fieldBase(t.shapeId); - const fieldPtr = B.tmp(); - B.line( - `${fieldPtr} = getelementptr inbounds %${mangleRecordStruct(t.shapeId)}, ptr %p, i64 0, i32 ${index}`, - ); - const fieldValue = loadStructField( - B, - host.sizeType, - fieldPtr, - field.type, - host.inlineUnions.record(field.type), - ); - // An ABSENT optional field contributes no key. - const undefinedTag = undefinedArmTag(field.type, host.unionsById); - let skip: string | null = null; - if (undefinedTag >= 0 && field.type.kind === "union") { - const absent = host.fieldAbsentTestIn(B, fieldValue, field.type.unionId); - skip = B.newLabel("live.record.absent"); - const set = B.newLabel("live.record.set"); - B.condBr(absent, skip, set); - B.startBlock(set); - } - const boxed = host.streamTypedRefBoxValue(B, field.type, fieldValue); - B.line( - `call void @scr_dyn_obj_set(ptr ${out}, ptr ${host.cstr(field.name)}, ${host.sizeType} ${Buffer.byteLength(field.name, "utf8")}, ptr ${boxed})`, - ); - if (skip !== null) { - B.br(skip); - B.startBlock(skip); + if (!dirent && orders.every((order) => liveRecordOpsEligible(fieldsOf(order)))) { + if (orders.length === 1) { + adapter.recordOps = `@${snapshot}_rops`; + liveRecordOps(host, t, snapshot, fieldsOf(orders[0]!)); + } else { + orders.forEach((order, i) => + liveRecordOps(host, t, `${snapshot}_o${i}`, fieldsOf(order)), + ); + adapter.recordBind = `${snapshot}_bind`; + liveRecordOrderBind(host, shape, snapshot); } } + emitPerKeyOrder(B, shape, host.nullableUnions.fieldBase(t.shapeId), "%p", (order) => { + for (const field of fieldsOf(order)) { + const index = shape.fields.indexOf(field) + host.nullableUnions.fieldBase(t.shapeId); + const fieldPtr = B.tmp(); + B.line( + `${fieldPtr} = getelementptr inbounds %${mangleRecordStruct(t.shapeId)}, ptr %p, i64 0, i32 ${index}`, + ); + const fieldValue = loadStructField( + B, + host.sizeType, + fieldPtr, + field.type, + host.inlineUnions.record(field.type), + ); + // An ABSENT optional field contributes no key. + const undefinedTag = undefinedArmTag(field.type, host.unionsById); + let skip: string | null = null; + if (undefinedTag >= 0 && field.type.kind === "union") { + const absent = host.fieldAbsentTestIn(B, fieldValue, field.type.unionId); + skip = B.newLabel("live.record.absent"); + const set = B.newLabel("live.record.set"); + B.condBr(absent, skip, set); + B.startBlock(set); + } + const boxed = host.streamTypedRefBoxValue(B, field.type, fieldValue); + B.line( + `call void @scr_dyn_obj_set(ptr ${out}, ptr ${host.cstr(field.name)}, ${host.sizeType} ${Buffer.byteLength(field.name, "utf8")}, ptr ${boxed})`, + ); + if (skip !== null) { + B.br(skip); + B.startBlock(skip); + } + } + }); if (dirent) { host.declare(`declare void @scr_dyn_dirent_finish(ptr)`); B.line(`call void @scr_dyn_dirent_finish(ptr ${out})`); @@ -1366,6 +1385,32 @@ function liveRecordOps( ); } +/** `@_bind(ptr capsule, ptr unused)` for a shape with + * orderVariants: binds the capsule the record ops of its value's key order + * (the `_o_rops` tables liveRecordOps made). */ +function liveRecordOrderBind( + host: LlvmEmitterContext, + shape: IrRecordShape, + snapshot: string, +): void { + host.declare(`declare void @scr_dyn_typed_ref_bind_record(ptr, ptr)`); + const B = new BlockBuilder(); + // The capsule's native value (ScrDyn v.typed_ref.ptr, byte 16 on every + // target). + B.line(`%pp = getelementptr inbounds i8, ptr %c, i64 16`); + B.line(`%p = load ptr, ptr %pp`); + emitPerKeyOrder(B, shape, host.nullableUnions.fieldBase(shape.id), "%p", (_, i) => { + B.line(`call void @scr_dyn_typed_ref_bind_record(ptr %c, ptr @${snapshot}_o${i}_rops)`); + }); + B.terminate(`ret void`); + host.resolveThunkDefs.push( + `define internal void @${snapshot}_bind(ptr %c, ptr %unused) ${FN_ATTRS} { ; live record ops by key order`, + B.render(), + `}`, + ``, + ); +} + /** Compiled classes whose instance views have no prototype object. Runtime * classes and JavaScript classes with prototype data answer through their * own views. */ @@ -1437,7 +1482,9 @@ function liveClassOps(host: LlvmEmitterContext, snapshot: string, meta: LlClassM * records (1 array, 2 record, 3 class). */ export function liveRefView( adapter: LlStreamTypedRefAdapter, -): { bind: string; ops: string; code: 1 | 2 | 3 } | null { +): { bind: string; ops: string; code: 1 | 2 | 3; defined?: true } | null { + if (adapter.recordBind !== undefined) + return { bind: adapter.recordBind, ops: "null", code: 2, defined: true }; if (adapter.classOps !== undefined) return { bind: "scr_dyn_typed_ref_bind_class", ops: adapter.classOps, code: 3 }; if (adapter.recordOps !== undefined) @@ -1455,7 +1502,7 @@ export function bindLiveArrayOps( ): string[] { const view = liveRefView(adapter); if (view === null) return []; - host.declare(`declare void @${view.bind}(ptr, ptr)`); + if (view.defined !== true) host.declare(`declare void @${view.bind}(ptr, ptr)`); return [`call void @${view.bind}(ptr ${capsule}, ptr ${view.ops})`]; } diff --git a/packages/compiler/src/backend/llvm/key-orders.ts b/packages/compiler/src/backend/llvm/key-orders.ts new file mode 100644 index 000000000..9dbda1414 --- /dev/null +++ b/packages/compiler/src/backend/llvm/key-orders.ts @@ -0,0 +1,49 @@ +/* Key-order dispatch for shapes whose values record the order their keys + * were written in (IrRecordShape.orderVariants): an emitter that walks a + * record's keys emits one walk per order and branches on the value's + * KEY_ORDER_FIELD. */ +import type { IrRecordShape } from "../../ir/ir.js"; +import { KEY_ORDER_FIELD, shapeKeyOrders } from "../../ir/ir.js"; +import { mangleRecordStruct } from "../mangle.js"; +import type { BlockBuilder } from "./blocks.js"; + +/** Emits `walk(order, index)` for record `rec` of `shape`, once per order behind a + * switch on the value's order index when the shape has several. Each walk + * starts in its own block, and the code after this call continues in the + * block they all branch to. */ +export function emitPerKeyOrder( + B: BlockBuilder, + shape: IrRecordShape, + fieldBase: number, + rec: string, + walk: (order: readonly string[], index: number) => void, +): void { + const orders = shapeKeyOrders(shape); + if (orders.length === 1) { + walk(orders[0]!, 0); + return; + } + const index = shape.fields.findIndex((f) => f.name === KEY_ORDER_FIELD) + fieldBase; + const p = B.tmp(); + const n = B.tmp(); + const i = B.tmp(); + B.line( + `${p} = getelementptr inbounds %${mangleRecordStruct(shape.id)}, ptr ${rec}, i64 0, i32 ${index} ; .${KEY_ORDER_FIELD}`, + ); + B.line(`${n} = load double, ptr ${p}`); + B.line(`${i} = fptosi double ${n} to i32`); + const labels = orders.map(() => B.newLabel("ko")); + const join = B.newLabel("ko.join"); + B.terminate( + `switch i32 ${i}, label %${labels[0]} [ ${labels + .slice(1) + .map((label, k) => `i32 ${k + 1}, label %${label}`) + .join(" ")} ]`, + ); + orders.forEach((order, k) => { + B.startBlock(labels[k]!); + walk(order, k); + B.br(join); + }); + B.startBlock(join); +} diff --git a/packages/compiler/src/backend/llvm/walkers.ts b/packages/compiler/src/backend/llvm/walkers.ts index 46dbc86e1..25e63251e 100644 --- a/packages/compiler/src/backend/llvm/walkers.ts +++ b/packages/compiler/src/backend/llvm/walkers.ts @@ -1,3 +1,4 @@ +import { emitPerKeyOrder } from "./key-orders.js"; import { f64Lit } from "./common.js"; import { InternalCompilerError } from "../../errors.js"; /* Structure-walking helper EMITTERS for the LLVM backend — the .ll mirror @@ -311,78 +312,80 @@ export class LlWalkers { if (cyclic) this.jbLeave(B); return; } - // Fields serialize in DECLARED order (JS insertion order); internal + // Fields serialize in DECLARED order (JS insertion order), or in the + // value's own order for a shape with orderVariants; internal // '%'-fields stay hidden — jsonWriteHelper's contract, ported comment // and all. - const order = shape.declaredOrder ?? shape.fields.map((f) => f.name); - const inOrder = new Set(order); - if (shape.fields.some((f) => !inOrder.has(f.name) && !f.name.startsWith("%"))) { - throw new InternalCompilerError( - `llvm emitter bug: declaredOrder of shape ${shapeId} omits a non-internal field`, - ); - } - const byName = new Map(shape.fields.map((f) => [f.name, f])); - // Function-valued properties are omitted, as JSON.stringify omits them. - const emitFields = order - .map((n) => byName.get(n)) - .filter((f) => f !== undefined) - .filter((f) => f.type.kind !== "func"); - const droppable = - emitFields.some((f) => undefinedArmTag(f.type, this.host.unionsById) >= 0) || - !!shape.indexValue; - this.putc(B, "%b", "123"); // '{' - if (!droppable) { - emitFields.forEach((f, i) => { - this.puts(B, "%b", `${i > 0 ? "," : ""}${jsonObjectKeyLabel(f.name)}`); - if (edgeable(f.type)) this.jbEdgeProp(B, f.name); - const v = this.loadField(B, "%v", shapeId, fieldIndex.get(f.name)!, f.type); - B.line( - `call void @${this.jsonWriteHelper(f.type)}(ptr %b, ${this.valTy(f.type)} ${v}) ; ${llvmCommentText(f.name)}`, + emitPerKeyOrder(B, shape, fieldBase, "%v", (order) => { + const inOrder = new Set(order); + if (shape.fields.some((f) => !inOrder.has(f.name) && !f.name.startsWith("%"))) { + throw new InternalCompilerError( + `llvm emitter bug: declaredOrder of shape ${shapeId} omits a non-internal field`, ); - }); - } else { - const first = B.slot(); - B.entryAllocas.push(`${first} = alloca i1`); - B.line(`store i1 true, ptr ${first}`); - const comma = (): void => { - const isf = B.tmp(); - const lc = B.newLabel("jwc.c"); - const lj = B.newLabel("jwc.j"); - B.line(`${isf} = load i1, ptr ${first}`); - B.condBr(isf, lj, lc); - B.startBlock(lc); - this.putc(B, "%b", "44"); // ',' - B.br(lj); - B.startBlock(lj); - B.line(`store i1 false, ptr ${first}`); - }; - for (const f of emitFields) { - const utag = undefinedArmTag(f.type, this.host.unionsById); - const v = this.loadField(B, "%v", shapeId, fieldIndex.get(f.name)!, f.type); - let skip: string | null = null; - if (utag >= 0 && f.type.kind === "union") { - // Undefined-valued field: dropped, like Node. - const tag = this.unionTag(B, v, f.type.unionId); - const isu = B.tmp(); - B.line(`${isu} = icmp eq i32 ${tag}, ${utag}`); - const lw = B.newLabel("jwf.w"); - skip = B.newLabel("jwf.s"); - B.condBr(isu, skip, lw); - B.startBlock(lw); - } - comma(); - this.puts(B, "%b", jsonObjectKeyLabel(f.name)); - if (edgeable(f.type)) this.jbEdgeProp(B, f.name); - B.line( - `call void @${this.jsonWriteHelper(f.type)}(ptr %b, ${this.valTy(f.type)} ${v}) ; ${llvmCommentText(f.name)}`, - ); - if (skip !== null) { - B.br(skip); - B.startBlock(skip); - } } - if (shape.indexValue) this.emitOverflowEntries(B, shape, first, edgeable(shape.indexValue)); - } + const byName = new Map(shape.fields.map((f) => [f.name, f])); + // Function-valued properties are omitted, as JSON.stringify omits them. + const emitFields = order + .map((n) => byName.get(n)) + .filter((f) => f !== undefined) + .filter((f) => f.type.kind !== "func"); + const droppable = + emitFields.some((f) => undefinedArmTag(f.type, this.host.unionsById) >= 0) || + !!shape.indexValue; + this.putc(B, "%b", "123"); // '{' + if (!droppable) { + emitFields.forEach((f, i) => { + this.puts(B, "%b", `${i > 0 ? "," : ""}${jsonObjectKeyLabel(f.name)}`); + if (edgeable(f.type)) this.jbEdgeProp(B, f.name); + const v = this.loadField(B, "%v", shapeId, fieldIndex.get(f.name)!, f.type); + B.line( + `call void @${this.jsonWriteHelper(f.type)}(ptr %b, ${this.valTy(f.type)} ${v}) ; ${llvmCommentText(f.name)}`, + ); + }); + } else { + const first = B.slot(); + B.entryAllocas.push(`${first} = alloca i1`); + B.line(`store i1 true, ptr ${first}`); + const comma = (): void => { + const isf = B.tmp(); + const lc = B.newLabel("jwc.c"); + const lj = B.newLabel("jwc.j"); + B.line(`${isf} = load i1, ptr ${first}`); + B.condBr(isf, lj, lc); + B.startBlock(lc); + this.putc(B, "%b", "44"); // ',' + B.br(lj); + B.startBlock(lj); + B.line(`store i1 false, ptr ${first}`); + }; + for (const f of emitFields) { + const utag = undefinedArmTag(f.type, this.host.unionsById); + const v = this.loadField(B, "%v", shapeId, fieldIndex.get(f.name)!, f.type); + let skip: string | null = null; + if (utag >= 0 && f.type.kind === "union") { + // Undefined-valued field: dropped, like Node. + const tag = this.unionTag(B, v, f.type.unionId); + const isu = B.tmp(); + B.line(`${isu} = icmp eq i32 ${tag}, ${utag}`); + const lw = B.newLabel("jwf.w"); + skip = B.newLabel("jwf.s"); + B.condBr(isu, skip, lw); + B.startBlock(lw); + } + comma(); + this.puts(B, "%b", jsonObjectKeyLabel(f.name)); + if (edgeable(f.type)) this.jbEdgeProp(B, f.name); + B.line( + `call void @${this.jsonWriteHelper(f.type)}(ptr %b, ${this.valTy(f.type)} ${v}) ; ${llvmCommentText(f.name)}`, + ); + if (skip !== null) { + B.br(skip); + B.startBlock(skip); + } + } + if (shape.indexValue) this.emitOverflowEntries(B, shape, first, edgeable(shape.indexValue)); + } + }); this.putc(B, "%b", "125"); // '}' if (cyclic) this.jbLeave(B); } diff --git a/packages/compiler/src/frontend/lowering/divergences.ts b/packages/compiler/src/frontend/lowering/divergences.ts index 1c362cb09..cf3c632d7 100644 --- a/packages/compiler/src/frontend/lowering/divergences.ts +++ b/packages/compiler/src/frontend/lowering/divergences.ts @@ -724,6 +724,11 @@ function keyOrderFinding( written: readonly string[], assignedNames: ReadonlySet, ): { native: string; node: string; detail: "reordered" | "added-later" } | null { + // Values of a shape with orderVariants enumerate in the order their + // literal wrote (key-order-witness.ts). + const mapped = safely(() => lowerer.mapTypeOf(contextual)); + if (mapped?.kind === "record" && lowerer.shapes.get(mapped.shapeId)?.orderVariants !== undefined) + return null; // A shape whose table the witness pass re-chose (key-order-witness.ts) // enumerates that table, not the checker's declaration order. const table = witnessTable(lowerer, contextual); diff --git a/packages/compiler/src/frontend/lowering/key-order-witness.test.ts b/packages/compiler/src/frontend/lowering/key-order-witness.test.ts index de16e5d61..56b65a3d3 100644 --- a/packages/compiler/src/frontend/lowering/key-order-witness.test.ts +++ b/packages/compiler/src/frontend/lowering/key-order-witness.test.ts @@ -1,5 +1,12 @@ import { expect, test } from "vitest"; -import { F64, type IrExpr, type IrFunction, type IrStmt, type IrType } from "../../ir/ir.js"; +import { + F64, + KEY_ORDER_FIELD, + type IrExpr, + type IrFunction, + type IrStmt, + type IrType, +} from "../../ir/ir.js"; import { observeRecordLiteralRebuilds, transformExpr } from "../../ir/traverse.js"; import { ShapeRegistry } from "../type-mapper.js"; import { @@ -21,7 +28,7 @@ interface Fixture { notes: KeyOrderNotes; record(fields: [string, IrType][], order: string[]): IrType & { kind: "record" }; literal(type: IrType, keys: string[], segments?: KeyOrderSegment[]): IrExpr; - run(body: IrStmt[]): void; + run(body: IrStmt[]): IrFunction; order(type: IrType & { kind: "record" }): string[] | undefined; } @@ -35,6 +42,14 @@ function fixture(): Fixture { classes: new Map(), armTag: (unionId: string, arm: IrType): number => unionId === "u0" && arm.kind === "undefinedT" ? 1 : -1, + recordFieldPresent: (obj: IrExpr, shapeId: string, field: string): IrExpr => ({ + kind: "recordHas", + obj, + shapeId, + field, + type: { kind: "bool" }, + loc, + }), } as unknown as Lowerer; const record = (fields: [string, IrType][], order: string[]): IrType & { kind: "record" } => ({ kind: "record", @@ -55,15 +70,20 @@ function fixture(): Fixture { segments ?? keys.map((name) => ({ kind: "key", name, definite: true })), ); }; - const run = (body: IrStmt[]): void => { + const run = (body: IrStmt[]): IrFunction => { const fn: IrFunction = { name: "main", params: [], returnType: F64, locals: [], body, loc }; applyKeyOrderWitnesses(lowerer, [fn]); + return fn; }; const order = (type: IrType & { kind: "record" }): string[] | undefined => shapes.get(type.shapeId)?.declaredOrder; return { lowerer, shapes, notes, record, literal, run, order }; } +/** The order index a literal stores (KEY_ORDER_FIELD), when it stores one. */ +const orderIndexOf = (lit: IrExpr): IrExpr | undefined => + lit.kind === "recordLit" ? lit.fields.find((x) => x.name === KEY_ORDER_FIELD)?.value : undefined; + const observe = (value: IrExpr): IrStmt => ({ kind: "exprStmt", expr: { kind: "jsonStringify", value, type: { kind: "string" }, loc }, @@ -109,7 +129,7 @@ test("literals that agree with the table leave the same array in place", () => { expect(f.notes.changed.size).toBe(0); }); -test("values built in two orders keep today's table", () => { +test("values built in two orders each record their literal's order", () => { const f = fixture(); const ab = f.record( [ @@ -118,8 +138,71 @@ test("values built in two orders keep today's table", () => { ], ["a", "b"], ); - f.run([observe(f.literal(ab, ["b", "a"])), observe(f.literal(ab, ["a", "b"]))]); + const ba = f.literal(ab, ["b", "a"]); + const aOrder = f.literal(ab, ["a", "b"]); + f.run([observe(ba), observe(aOrder)]); expect(f.order(ab)).toEqual(["a", "b"]); + const shape = f.shapes.get(ab.shapeId)!; + expect(shape.orderVariants).toEqual([ + ["a", "b"], + ["b", "a"], + ]); + expect(shape.fields.map((x) => x.name)).toEqual([KEY_ORDER_FIELD, "a", "b"]); + expect(orderIndexOf(ba)).toMatchObject({ kind: "numLit", value: 1 }); + expect(orderIndexOf(aOrder)).toMatchObject({ kind: "numLit", value: 0 }); +}); + +test("a copy of a value with orders stores its source's index", () => { + const f = fixture(); + const abc = f.record( + [ + ["a", F64], + ["b", F64], + ["c", F64], + ], + ["a", "b", "c"], + ); + const ab = f.record( + [ + ["a", F64], + ["b", F64], + ], + ["a", "b"], + ); + const read = (field: string): IrExpr => ({ + kind: "recordGet", + obj: ref("x", abc), + shapeId: abc.shapeId, + field, + type: F64, + loc, + }); + const rest = f.notes.copy( + { + kind: "recordLit", + fields: [ + { name: "a", value: read("a") }, + { name: "b", value: read("b") }, + ], + type: ab, + loc, + }, + abc.shapeId, + ); + f.run([ + decl("x", f.literal(abc, ["c", "b", "a"])), + observe(f.literal(abc, ["a", "b", "c"])), + observe(rest), + ]); + expect(f.shapes.get(ab.shapeId)!.orderVariants).toEqual([ + ["a", "b"], + ["b", "a"], + ]); + expect(orderIndexOf(rest)).toMatchObject({ + kind: "recordGet", + obj: { kind: "varRef", localId: "x" }, + field: KEY_ORDER_FIELD, + }); }); test("an unobserved shape keeps today's table", () => { @@ -307,7 +390,7 @@ test("straight-line writes of late-only keys keep program order", () => { expect(f.order(o)).toEqual(["id", "y", "x"]); }); -test("late-only keys written through different bindings keep today's table", () => { +test("late-only keys written through different bindings move to the end per value", () => { const f = fixture(); const o = f.record( [ @@ -324,9 +407,13 @@ test("late-only keys written through different bindings keep today's table", () observe(ref("o", o)), ]); expect(f.order(o)).toEqual(["id", "x", "y"]); + expect(f.shapes.get(o.shapeId)!.orderVariants).toEqual([ + ["id", "x", "y"], + ["id", "y", "x"], + ]); }); -test("a late-only key written in a loop over an outer binding keeps today's table", () => { +test("a late-only key written in a loop guards the write with its move", () => { const f = fixture(); const o = f.record( [ @@ -336,7 +423,7 @@ test("a late-only key written in a loop over an outer binding keeps today's tabl ], ["id", "x", "y"], ); - f.run([ + const body: IrStmt[] = [ decl("o", f.literal(o, ["id"])), set(ref("o", o), "y"), { @@ -346,11 +433,18 @@ test("a late-only key written in a loop over an outer binding keeps today's tabl loc, }, observe(ref("o", o)), - ]); + ]; + const fn = f.run(body); expect(f.order(o)).toEqual(["id", "x", "y"]); + const loop = fn.body.find((st) => st.kind === "while") as IrStmt & { kind: "while" }; + const guarded = loop.body[0]!; + expect(guarded.kind).toBe("block"); + const [move, write] = (guarded as IrStmt & { kind: "block" }).body; + expect(move).toMatchObject({ kind: "if", cond: { kind: "unary", op: "!" } }); + expect(write).toMatchObject({ kind: "recordSet", field: "x" }); }); -test("re-adding a key a literal creates earlier keeps today's table", () => { +test("re-adding a key a literal creates earlier moves it to the end of that value", () => { const f = fixture(); const o = f.record( [ @@ -369,4 +463,8 @@ test("re-adding a key a literal creates earlier keeps today's table", () => { }; f.run([decl("o", f.literal(o, ["a", "b"])), del, set(ref("o", o), "a"), observe(ref("o", o))]); expect(f.order(o)).toEqual(["a", "b"]); + expect(f.shapes.get(o.shapeId)!.orderVariants).toEqual([ + ["a", "b"], + ["b", "a"], + ]); }); diff --git a/packages/compiler/src/frontend/lowering/key-order-witness.ts b/packages/compiler/src/frontend/lowering/key-order-witness.ts index 2fc9a6cbd..c35860d43 100644 --- a/packages/compiler/src/frontend/lowering/key-order-witness.ts +++ b/packages/compiler/src/frontend/lowering/key-order-witness.ts @@ -41,11 +41,56 @@ * value: that shape, and every shape copied from it, keeps today's table * (and today's SC6002 warnings). Shapes that no order surface observes * keep today's table too. */ -import type { IrExpr, IrFunction, IrRecordShape, IrStmt, IrType } from "../../ir/ir.js"; -import { shapeHasAccessorSlots, typeEquals, UNDEFINED_T } from "../../ir/ir.js"; -import { everyExprChild, everyStmtChild, everyTypeChild } from "../../ir/traverse.js"; +import type { IrExpr, IrFunction, IrRecordShape, IrStmt, IrType, SrcLoc } from "../../ir/ir.js"; +import { + BOOL, + F64, + KEY_ORDER_FIELD, + shapeHasAccessorSlots, + typeEquals, + UNDEFINED_T, +} from "../../ir/ir.js"; +import { + everyExprChild, + everyStmtChild, + everyTypeChild, + transformStmtList, +} from "../../ir/traverse.js"; import type { Lowerer } from "./lowerer.js"; +/** The statements a key-order helper runs for record `ref`: `walks[0]`, + * its walk over the shape's table, or, for a shape with orderVariants, a + * branch per order (walks[i] for order i) on the value's KEY_ORDER_FIELD. + * `walks` follows shapeKeyOrders. */ +export function perKeyOrder( + shape: IrRecordShape, + ref: IrExpr, + walks: readonly IrStmt[][], + loc: SrcLoc, +): IrStmt[] { + let tail = walks[0]!; + for (let i = walks.length - 1; i >= 1; i--) { + const index: IrExpr = { + kind: "recordGet", + obj: ref, + shapeId: shape.id, + field: KEY_ORDER_FIELD, + type: F64, + loc, + }; + const cond: IrExpr = { + kind: "bin", + op: "===", + left: index, + right: { kind: "numLit", value: i, type: F64, loc }, + type: BOOL, + loc, + }; + tail = [{ kind: "if", cond, then: walks[i]!, else_: tail, loc }]; + } + return tail; +} + /** One property of an object literal, in source order. */ export type KeyOrderSegment = /** An explicit key (`definite`), or a conditional spread's key, which @@ -708,4 +753,424 @@ export function applyKeyOrderWitnesses(lowerer: Lowerer, fns: readonly IrFunctio shape.declaredOrder = [...shape.declaredOrder!.filter(isArrayIndex), ...resolved.order]; notes.changed.add(id); } + + // ── per-instance orders ───────────────────────────────────────────── + // A shape whose literals write its keys in more than one order has no + // one table. Each value then records which of the orders it was written + // in (KEY_ORDER_FIELD), when every construction's order is known + // statically or copied from a value that records its own: + // - a literal of explicit keys writes its order; + // - a copy from a shape with one resolved order writes that order; + // - a copy from a value of a shape with orders writes the source's + // order, filtered to the copy's keys (rest bindings, width copies); + // - clones copy the field, and every other construction (synthesized + // literals, checked conversions, runtime producers) leaves 0, today's + // table, as before. + // A write of an optional key a value lacks moves the key to the end of + // the value's order, as in Node: the orders include every order such + // writes reach, and each write site updates the index first. + const MAX_ORDERS = 16; + const unresolved = (id: string): boolean => result.get(id) === "unknown"; + /** `order` over every key of `table`: keys the order lacks follow it. */ + const complete = (order: readonly string[], table: readonly string[]): string[] => { + const keys = new Set(table); + const out = order.filter((k) => keys.has(k)); + const seen = new Set(out); + return [...out, ...table.filter((k) => !seen.has(k))]; + }; + const literalOrder = (segments: readonly KeyOrderSegment[]): string[] | null => { + const out: string[] = []; + for (const segment of segments) { + if (segment.kind !== "key" || !segment.definite) return null; + if (!isArrayIndex(segment.name)) out.push(segment.name); + } + return out; + }; + /** `{ ...value }` of a value of the literal's own shape: a copy of the + * value's order. */ + const selfSpread = (segments: readonly KeyOrderSegment[], id: string): boolean => + segments.length === 1 && + segments[0]!.kind === "spread" && + segments[0]!.shapeId === id && + !segments[0]!.maybeAbsent; + const sameOrder = (a: readonly string[], b: readonly string[]): boolean => + a.join("\0") === b.join("\0"); + const indexOfOrder = (orders: readonly string[][], order: readonly string[]): number => + orders.findIndex((o) => sameOrder(o, order)); + /** The keys writes can add to a value of the shape, or null when a + * clone override can (the clone's order is its source's). */ + const movingKeys = (id: string, f: ShapeFacts, table: readonly string[]): string[] | null => { + const keys = new Set(table); + const moving = new Set(); + for (const w of f.writes) { + if (w.kind === "delete" || !keys.has(w.field) || !maybeAbsent(id, w.field)) continue; + if (w.kind === "clone") return null; + moving.add(w.field); + } + return [...moving]; + }; + const moves = new Map>(); + /** Adds to `orders` every order writes of `moving` keys reach from them + * and records each key's transitions; false past MAX_ORDERS. */ + const closeUnderMoves = (id: string, orders: string[][], moving: readonly string[]): boolean => { + const transitions = new Map(moving.map((k) => [k, []])); + for (let i = 0; i < orders.length; i++) { + for (const key of moving) { + const moved = [...orders[i]!.filter((k) => k !== key), key]; + let index = indexOfOrder(orders, moved); + if (index < 0) { + if (orders.length === MAX_ORDERS) return false; + index = orders.push(moved) - 1; + } + transitions.get(key)![i] = index; + } + } + if (moving.length > 0) moves.set(id, transitions); + return true; + }; + // Observed shapes, and the shapes their values copy their orders from. + const wanted = new Set(observed); + const pending = [...observed]; + while (pending.length > 0) { + const f = facts.get(pending.pop()!); + if (f === undefined) continue; + const sources = [ + ...f.copies.map((c) => c.source), + ...f.literals.flatMap((segments) => + segments.flatMap((s) => (s.kind === "spread" ? [s.shapeId] : [])), + ), + ]; + for (const source of sources) { + if (wanted.has(source)) continue; + wanted.add(source); + pending.push(source); + } + } + const orderable = ( + id: string, + f: ShapeFacts, + ): { shape: IrRecordShape; moving: string[] } | null => { + const shape = shapes.get(id); + if (!wanted.has(id) || !unresolved(id) || !inScope(shape)) return null; + const moving = movingKeys(id, f, tableOf(shape)); + return moving === null ? null : { shape, moving }; + }; + const variants = new Map(); + for (const [id, f] of facts) { + const candidate = orderable(id, f); + if (candidate === null || f.literals.length === 0) continue; + const table = tableOf(candidate.shape); + const orders: string[][] = [table]; + let ok = true; + const add = (order: readonly string[]): void => { + const full = complete(order, table); + if (indexOfOrder(orders, full) < 0) orders.push(full); + }; + for (const segments of f.literals) { + if (selfSpread(segments, id)) continue; + const order = literalOrder(segments); + if (order === null) ok = false; + else add(order); + } + for (const c of f.copies) { + if (c.source === id) continue; + const source = result.get(c.source); + if (source === undefined || source === "unknown") ok = false; + else add(source.order.filter((k) => c.keys.has(k))); + } + if (!ok || orders.length > MAX_ORDERS || !closeUnderMoves(id, orders, candidate.moving)) + continue; + if (orders.length > 1) variants.set(id, orders); + else moves.delete(id); + } + // Shapes built only from values of one shape with orders, all in one + // way: copies (rest bindings, width copies), or literals spreading the + // value between the same explicit keys. Order i is the source's order i + // over the shape's keys, after the keys before the spread and before + // the keys after it. + type Template = { source: string; prefix: string[]; suffix: string[] }; + const templateOf = (segments: readonly KeyOrderSegment[], id: string): Template | null => { + const prefix: string[] = []; + const suffix: string[] = []; + let source: string | null = null; + for (const segment of segments) { + if (segment.kind === "opaque") return null; + if (segment.kind === "spread") { + if (source !== null || segment.maybeAbsent || segment.shapeId === id) return null; + source = segment.shapeId; + continue; + } + if (!segment.definite) return null; + (source === null ? prefix : suffix).push(segment.name); + } + return source === null ? null : { source, prefix, suffix }; + }; + const templateKey = (t: Template): string => JSON.stringify([t.source, t.prefix, t.suffix]); + const copyTemplate = (source: string): Template => ({ source, prefix: [], suffix: [] }); + const mirrors = new Map(); + for (const [id, f] of facts) { + if (variants.has(id) || f.pinned || f.runtime || f.opaque) continue; + const candidate = orderable(id, f); + if (candidate === null || f.literals.length + f.copies.length === 0) continue; + const templates: (Template | null)[] = f.literals.map((segments) => templateOf(segments, id)); + for (const c of f.copies) templates.push(copyTemplate(c.source)); + const first = templates[0]; + if ( + first === null || + first === undefined || + templates.some((t) => t === null || templateKey(t) !== templateKey(first)) + ) + continue; + const orders = variants.get(first.source); + if (orders === undefined || mirrors.has(first.source)) continue; + const table = tableOf(candidate.shape); + const mirrored = orders.map((o) => { + const order = [...first.prefix]; + for (const k of [...o, ...first.suffix]) if (!order.includes(k)) order.push(k); + return complete(order, table); + }); + if (!closeUnderMoves(id, mirrored, candidate.moving)) continue; + if (mirrored.every((o) => sameOrder(o, mirrored[0]!))) { + // Every order of the source agrees over these keys: one table. + moves.delete(id); + const shape = candidate.shape; + const only: string[] = mirrored[0] ?? table; + if (!sameOrder(only, table)) + shape.declaredOrder = [...shape.declaredOrder!.filter(isArrayIndex), ...only]; + continue; + } + variants.set(id, mirrored); + mirrors.set(id, first); + } + if (variants.size === 0) return; + + // Every construction of those shapes, to give each its order index. A + // copy reads its source's index, so its source must be a plain binding. + const sites = new Map(); + /** The statements a spread literal's sequence evaluates first: the + * spread value's binding, then its fields. */ + const prologues = new Map(); + const siteOf = (e: IrExpr): boolean => { + if (e.kind === "seqExpr" && e.result.kind === "recordLit") prologues.set(e.result, e.stmts); + if (e.kind === "recordLit" && e.type.kind === "record" && variants.has(e.type.shapeId)) { + const list = sites.get(e.type.shapeId) ?? []; + list.push(e); + sites.set(e.type.shapeId, list); + } + return everyExprChild(e, siteOf, siteStmt); + }; + const siteStmt = (st: IrStmt): boolean => everyStmtChild(st, siteOf, siteStmt); + for (const fn of fns) for (const st of fn.body) siteStmt(st); + /** The binding a copy or spread reads its fields from. */ + const copySource = (lit: IrExpr & { kind: "recordLit" }, source: string): IrExpr | null => { + for (const field of lit.fields) { + const value = field.value; + if (value.kind === "recordGet" && value.shapeId === source && value.obj.kind === "varRef") + return value.obj; + } + for (const st of prologues.get(lit) ?? []) { + if (st.kind !== "varDecl" || st.init === null) continue; + const t = st.init.type; + if (t.kind === "record" && t.shapeId === source) + return { kind: "varRef", localId: st.localId, type: t, loc: lit.loc }; + } + return null; + }; + const orderIndex = (id: string, lit: IrExpr & { kind: "recordLit" }): IrExpr | number | null => { + const orders = variants.get(id)!; + const segments = notes.literalOf(lit); + if (segments !== undefined && selfSpread(segments, id)) { + const obj = copySource(lit, id); + if (obj === null) return null; + return { + kind: "recordGet", + obj, + shapeId: id, + field: KEY_ORDER_FIELD, + type: F64, + loc: lit.loc, + }; + } + const mirror = mirrors.get(id); + if (segments !== undefined && mirror !== undefined) { + const obj = copySource(lit, mirror.source); + if (obj === null) return null; + return { + kind: "recordGet", + obj, + shapeId: mirror.source, + field: KEY_ORDER_FIELD, + type: F64, + loc: lit.loc, + }; + } + if (segments !== undefined) { + const order = literalOrder(segments); + return order === null ? null : indexOfOrder(orders, complete(order, orders[0]!)); + } + const source = notes.copyOf(lit); + if (source === undefined) return 0; + if (variants.has(source) && (source === id || mirrors.get(id)?.source === source)) { + const obj = copySource(lit, source); + if (obj === null) return null; + return { + kind: "recordGet", + obj, + shapeId: source, + field: KEY_ORDER_FIELD, + type: F64, + loc: lit.loc, + }; + } + const resolved = result.get(source); + if (resolved === undefined || resolved === "unknown") return null; + const keys = new Set( + lit.fields.filter((x) => x.value.kind !== "fieldAbsent").map((x) => x.name), + ); + return indexOfOrder( + orders, + complete( + resolved.order.filter((k) => keys.has(k)), + orders[0]!, + ), + ); + }; + const plan = new Map(); + const dropped = new Set(); + for (const [id, list] of sites) { + for (const lit of list) { + const index = orderIndex(id, lit); + if (index === null || (typeof index === "number" && index < 0)) dropped.add(id); + else plan.set(lit, index); + } + } + // Write sites of keys that move: each reads and updates the value's + // index before the write. + const writeSites: { st: IrStmt & { kind: "recordSet" }; fn: IrFunction }[] = []; + let writeFn: IrFunction | null = null; + const writeExpr = (e: IrExpr): boolean => everyExprChild(e, writeExpr, writeOf); + const writeOf = (st: IrStmt): boolean => { + if ( + st.kind === "recordSet" && + st.value.kind !== "fieldAbsent" && + moves.get(st.shapeId)?.has(st.field) + ) + writeSites.push({ st, fn: writeFn! }); + return everyStmtChild(st, writeExpr, writeOf); + }; + for (const fn of fns) { + writeFn = fn; + for (const st of fn.body) writeOf(st); + } + // A shape whose copies cannot read their source keeps today's table, and + // so do the shapes that mirror it. + for (const [id, mirror] of mirrors) if (dropped.has(mirror.source)) dropped.add(id); + for (const [lit, index] of plan) { + const id = (lit.type as IrType & { kind: "record" }).shapeId; + if (dropped.has(id)) continue; + const value: IrExpr = + typeof index === "number" ? { kind: "numLit", value: index, type: F64, loc: lit.loc } : index; + lit.fields.push({ name: KEY_ORDER_FIELD, value }); + } + let receivers = 0; + const replacements = new Map>(); + for (const { st, fn } of writeSites) { + if (dropped.has(st.shapeId)) continue; + const transitions = moves.get(st.shapeId)!.get(st.field)!; + const loc = st.loc; + // A receiver other than a binding evaluates once, into a local. + let plain = st.obj; + while (plain.kind === "unionNarrow" || plain.kind === "downcast") plain = plain.value; + const prologue: IrStmt[] = []; + let obj = st.obj; + if (plain.kind !== "varRef") { + const localId = `%keyOrderReceiver.${receivers++}`; + fn.locals.push({ id: localId, name: "%keyOrderReceiver", type: st.obj.type, mutable: false }); + prologue.push({ kind: "varDecl", localId, init: st.obj, loc }); + obj = { kind: "varRef", localId, type: st.obj.type, loc }; + } + const current: IrExpr = { + kind: "recordGet", + obj, + shapeId: st.shapeId, + field: KEY_ORDER_FIELD, + type: F64, + loc, + }; + // transitions[i] for the current index i, as a chain of tests. + let next: IrExpr = { + kind: "numLit", + value: transitions[transitions.length - 1]!, + type: F64, + loc, + }; + for (let i = transitions.length - 2; i >= 0; i--) { + next = { + kind: "ternary", + cond: { + kind: "bin", + op: "===", + left: current, + right: { kind: "numLit", value: i, type: F64, loc }, + type: BOOL, + loc, + }, + then: { kind: "numLit", value: transitions[i]!, type: F64, loc }, + else_: next, + type: F64, + loc, + }; + } + const absent: IrExpr = { + kind: "unary", + op: "!", + operand: lowerer.recordFieldPresent(obj, st.shapeId, st.field, loc), + type: BOOL, + loc, + }; + const update: IrStmt = { + kind: "recordSet", + obj, + shapeId: st.shapeId, + field: KEY_ORDER_FIELD, + value: next, + loc, + }; + // The write becomes a block that moves the key first. + const write: IrStmt = { + kind: "recordSet", + obj, + shapeId: st.shapeId, + field: st.field, + value: st.value, + loc, + }; + const guard: IrStmt = { kind: "if", cond: absent, then: [update], else_: null, loc }; + const block: IrStmt = { kind: "block", body: [...prologue, guard, write], loc }; + let replaced = replacements.get(fn); + if (replaced === undefined) { + replaced = new Map(); + replacements.set(fn, replaced); + } + replaced.set(st, block); + } + for (const [fn, replaced] of replacements) { + fn.body = transformStmtList(fn.body, { + expr: (e) => e, + stmt: (st) => replaced.get(st) ?? st, + }); + } + for (const [id, orders] of variants) { + if (dropped.has(id)) continue; + const shape = shapes.get(id)!; + const indexKeys: string[] = (shape.declaredOrder ?? []).filter(isArrayIndex); + shapes.addKeyOrderField(id); + const tables: string[][] = []; + for (const order of orders) { + const table: string[] = [...indexKeys]; + for (const key of order) table.push(key); + tables.push(table); + } + shape.orderVariants = tables; + } } diff --git a/packages/compiler/src/frontend/lowering/lower-calls.ts b/packages/compiler/src/frontend/lowering/lower-calls.ts index bd5068d05..0d45628ed 100644 --- a/packages/compiler/src/frontend/lowering/lower-calls.ts +++ b/packages/compiler/src/frontend/lowering/lower-calls.ts @@ -1,4 +1,5 @@ import { keyedHelperName } from "./stable-names.js"; +import { perKeyOrder } from "./key-order-witness.js"; import { isForwardedTupleSpread, lowerForwardingApply } from "./typed-arguments.js"; import { guardNullProto, nullProtoRecordShape } from "./null-prototype.js"; import { lowerDynStreamMemberCall } from "./dyn-streams.js"; @@ -71,6 +72,7 @@ import { isDynTypedRefType, isUnitType, shapeHasAccessorSlots, + shapeKeyOrders, typeEquals, RUNTIME_EMITTER_CLASS, } from "../../ir/ir.js"; @@ -7029,8 +7031,12 @@ export function recordKeysArrayCall( loc, }, ]; - const order = current.declaredOrder ?? current.fields.map((f) => f.name); - for (const name of order) { + // One walk per key order (see perKeyOrder). + const orders = shapeKeyOrders(current); + const walks = orders.map((): IrStmt[] => []); + for (const [k, name] of orders.flatMap((order, k) => + order.map((name) => [k, name] as const), + )) { const f = current.fields.find((x) => x.name === name)!; const pushStmt: IrStmt = { kind: "exprStmt", @@ -7047,12 +7053,13 @@ export function recordKeysArrayCall( // Fields that can be absent push behind their presence test (the // key exists exactly when Object.keys would list it). const present = lowerer.recordFieldPresent(ref, argIr.shapeId, f.name, loc); - body.push( + walks[k]!.push( present.kind === "recordHas" ? { kind: "if", cond: present, then: [pushStmt], else_: null, loc } : pushStmt, ); } + body.push(...perKeyOrder(current, ref, walks, loc)); body.push({ kind: "return", value: outRef, loc }); fn.body = body; }; @@ -8709,8 +8716,12 @@ function lowerObjectStaticCall( loc, }, ]; - const order = current.declaredOrder ?? current.fields.map((f) => f.name); - for (const name of order) { + // One walk per key order (see perKeyOrder). + const orders = shapeKeyOrders(current); + const walks = orders.map((): IrStmt[] => []); + for (const [k, name] of orders.flatMap((order, k) => + order.map((name) => [k, name] as const), + )) { const f = current.fields.find((x) => x.name === name)!; const raw: IrExpr = { kind: "recordGet", @@ -8872,8 +8883,9 @@ function lowerObjectStaticCall( loc, }; } - body.push(stmt); + walks[k]!.push(stmt); } + body.push(...perKeyOrder(current, ref, walks, loc)); body.push({ kind: "return", value: outRef, loc }); fn.body = body; }; diff --git a/packages/compiler/src/frontend/lowering/lower-inspect.ts b/packages/compiler/src/frontend/lowering/lower-inspect.ts index 47f490c7b..57f16bcc7 100644 --- a/packages/compiler/src/frontend/lowering/lower-inspect.ts +++ b/packages/compiler/src/frontend/lowering/lower-inspect.ts @@ -1,4 +1,5 @@ import { keyedHelperName } from "./stable-names.js"; +import { perKeyOrder } from "./key-order-witness.js"; import { dynUndefinedExpr, boolLit, @@ -40,6 +41,7 @@ import { isDynTypedRefType, recordTextCodecClass, shapeHasAccessorSlots, + shapeKeyOrders, typeKey, } from "../../ir/ir.js"; import type { ClassInfo } from "./lower-classes.js"; @@ -1647,7 +1649,12 @@ function inspectHelper(lowerer: Lowerer, t: IrType, loc: SrcLoc): string { }); } body.push(depthGate("[Object]"), ...begin()); - for (const fname of order) { + // One walk per key order (see perKeyOrder). + const orders = shapeKeyOrders(current); + const walks = orders.map((): IrStmt[] => []); + for (const [k, fname] of orders.flatMap((order, k) => + order.map((name) => [k, name] as const), + )) { const ft = byName.get(fname); if (!ft) continue; const show = entry( @@ -1655,12 +1662,13 @@ function inspectHelper(lowerer: Lowerer, t: IrType, loc: SrcLoc): string { boolLit(false, loc), ); const present = lowerer.recordFieldPresent(v(), t.shapeId, fname, loc); - body.push( + walks[k]!.push( present.kind === "recordHas" ? { kind: "if", cond: present, then: [show], else_: null, loc } : show, ); } + body.push(...perKeyOrder(current, v(), walks, loc)); body.push( ret(end(strLit("", loc), strLit("{", loc), strLit("}", loc), false, boolLit(false, loc))), ); diff --git a/packages/compiler/src/frontend/type-mapper.ts b/packages/compiler/src/frontend/type-mapper.ts index 9abd5a0b0..ee942c032 100644 --- a/packages/compiler/src/frontend/type-mapper.ts +++ b/packages/compiler/src/frontend/type-mapper.ts @@ -42,7 +42,7 @@ import { UNDEFINED_T, VOID, } from "../ir/ir.js"; -import { BIGINT_T } from "../ir/ir.js"; +import { BIGINT_T, KEY_ORDER_FIELD } from "../ir/ir.js"; import { literalValues } from "./literal-values.js"; import { remapUnionDiscriminant } from "./union-discriminants.js"; @@ -523,6 +523,21 @@ export class ShapeRegistry { return missing.length === 0 ? order : [...order, ...missing]; } + /** Gives a shape with orderVariants its KEY_ORDER_FIELD, in place, + * after the bodies lowered (key-order-witness.ts adds the field to each + * of the shape's literals). Allocation zeroes it, which every other + * construction keeps. */ + addKeyOrderField(shapeId: string): void { + const shape = this.byId.get(shapeId); + if (!shape || shape.fields.some((f) => f.name === KEY_ORDER_FIELD)) return; + shape.fields = [...shape.fields, { name: KEY_ORDER_FIELD, type: F64 }].sort((a, b) => + a.name < b.name ? -1 : a.name > b.name ? 1 : 0, + ); + this.revision++; + const key = this.keyOf(shape.fields, false, shape.indexValue); + if (!this.byKey.has(key)) this.byKey.set(key, shapeId); + } + /** Adds optional fields to an interned shape in place: members a type * assertion attaches to the shape's values (`r as R & { tag: string }` * followed by `view.tag = ...`; see assertion-members.ts). Every value diff --git a/packages/compiler/src/ir/function-references.test.ts b/packages/compiler/src/ir/function-references.test.ts index 7bef40aa6..0a802c03d 100644 --- a/packages/compiler/src/ir/function-references.test.ts +++ b/packages/compiler/src/ir/function-references.test.ts @@ -172,6 +172,7 @@ const MODULE_FIELDS: Record>, Role> = { "records[].fields[].name": "other", "records[].declaredOrder[]": "other", "records[].accessorOrder[]": "other", + "records[].orderVariants[][]": "other", "unions[].id": "other", "unions[].discriminant.field": "other", "unions[].discriminant.cases[].values[]": "other", diff --git a/packages/compiler/src/ir/ir.ts b/packages/compiler/src/ir/ir.ts index 4a395feb2..89a5ee73a 100644 --- a/packages/compiler/src/ir/ir.ts +++ b/packages/compiler/src/ir/ir.ts @@ -1665,6 +1665,24 @@ export interface IrRecordShape { * an accessor property named once (see accessorSlotProp). The * checked-dynamic copy lists its keys in this order. */ accessorOrder?: string[]; + /** Every order the shape's values enumerate their keys in, when its + * object literals write the keys in more than one order (see + * key-order-witness.ts); declaredOrder is the first. Each value holds the + * index of its order in the hidden number field KEY_ORDER_FIELD: the + * literal that wrote it stores its order's index, a copy its source's, + * and every other construction leaves 0. Each key-order surface + * dispatches on that field. */ + orderVariants?: string[][]; +} + +/** The hidden field of a shape with orderVariants: the index of the value's + * key order. */ +export const KEY_ORDER_FIELD = "%order"; + +/** The orders a record shape's values enumerate their keys in: each order + * of a shape with orderVariants, or its one table. */ +export function shapeKeyOrders(shape: IrRecordShape): readonly (readonly string[])[] { + return shape.orderVariants ?? [shape.declaredOrder ?? shape.fields.map((f) => f.name)]; } /** fs.Dirent rows: the hidden %dtype entry kind (libuv's UV_DIRENT diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7122-key-order-json.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7122-key-order-json.ts.json new file mode 100644 index 000000000..3551ad6b1 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7122-key-order-json.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7122-key-order-json.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7123-key-order-keys-values-entries.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7123-key-order-keys-values-entries.ts.json new file mode 100644 index 000000000..4451e935a --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7123-key-order-keys-values-entries.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7123-key-order-keys-values-entries.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7124-key-order-for-in.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7124-key-order-for-in.ts.json new file mode 100644 index 000000000..e87101734 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7124-key-order-for-in.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7124-key-order-for-in.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7125-key-order-inspect.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7125-key-order-inspect.ts.json new file mode 100644 index 000000000..b13a13261 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7125-key-order-inspect.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7125-key-order-inspect.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7126-key-order-copies.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7126-key-order-copies.ts.json new file mode 100644 index 000000000..a9f55663d --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7126-key-order-copies.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7126-key-order-copies.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7127-key-order-dynamic.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7127-key-order-dynamic.ts.json new file mode 100644 index 000000000..5a091d157 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7127-key-order-dynamic.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7127-key-order-dynamic.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7128-key-order-late-writes.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7128-key-order-late-writes.ts.json new file mode 100644 index 000000000..407a2179e --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7128-key-order-late-writes.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7128-key-order-late-writes.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7122-key-order-json.ts b/tests/corpus/7122-key-order-json.ts new file mode 100644 index 000000000..0cd3dff61 --- /dev/null +++ b/tests/corpus/7122-key-order-json.ts @@ -0,0 +1,31 @@ +// JSON.stringify of object literals that write the same keys in different +// orders: each value serializes in the order its literal wrote, as in Node. +type Point = { x: number; y: number; label?: string }; + +const a = { id: 1, name: "a", ok: true }; +const b = { name: "b", ok: false, id: 2 }; +const c = { ok: true, id: 3, name: "c" }; +for (const value of [a, b, c]) console.log(JSON.stringify(value)); +console.log(JSON.stringify([a, b, c])); +console.log(JSON.stringify({ first: b, rest: [c, a] }, null, 2)); + +// Optional keys the literal leaves out, and writes to keys it has. +const p: Point = { y: 2, x: 1 }; +const q: Point = { label: "q", x: 3, y: 4 }; +const r: Point = { x: 5, y: 6, label: "r" }; +q.x = 30; +r.label = "r2"; +for (const point of [p, q, r]) console.log(JSON.stringify(point)); + +// A function typed by one literal's shape receiving the others. +function describe(value: { id: number; name: string; ok: boolean }): string { + return JSON.stringify(value); +} +console.log(describe(c), describe(a)); + +// JSON.parse into the same record type keeps each text's own key order. +type Entry = { id: number; name: string; ok: boolean }; +for (const text of ['{"ok":true,"name":"p","id":7}', '{"id":8,"name":"q","ok":false}']) { + const parsed = JSON.parse(text) as Entry; + console.log(JSON.stringify(parsed), Object.keys(parsed).join(","), describe(parsed)); +} diff --git a/tests/corpus/7123-key-order-keys-values-entries.ts b/tests/corpus/7123-key-order-keys-values-entries.ts new file mode 100644 index 000000000..29f725c24 --- /dev/null +++ b/tests/corpus/7123-key-order-keys-values-entries.ts @@ -0,0 +1,19 @@ +// Object.keys, Object.values, Object.entries and Object.fromEntries over +// object literals that write the same keys in different orders. +const a = { id: 1, name: "a", ok: true }; +const b = { name: "b", ok: false, id: 2 }; +const c = { ok: true, id: 3, name: "c" }; +for (const value of [a, b, c]) { + console.log(Object.keys(value).join(",")); + console.log(Object.values(value).join(",")); + console.log(Object.entries(value).map(([key, item]) => `${key}=${String(item)}`).join(" ")); + console.log(JSON.stringify(Object.fromEntries(Object.entries(value)))); +} + +type Row = { year: number; title?: string; score: number }; +const rows: Row[] = [ + { score: 1, year: 2001 }, + { title: "t", year: 2002, score: 2 }, + { year: 2003, score: 3, title: "u" }, +]; +for (const row of rows) console.log(Object.keys(row).join(","), Object.values(row).length); diff --git a/tests/corpus/7124-key-order-for-in.ts b/tests/corpus/7124-key-order-for-in.ts new file mode 100644 index 000000000..89ed16edb --- /dev/null +++ b/tests/corpus/7124-key-order-for-in.ts @@ -0,0 +1,15 @@ +// for-in over object literals that write the same keys in different +// orders, and over values a function received. +const a = { id: 1, name: "a", ok: true }; +const b = { name: "b", ok: false, id: 2 }; +const c = { ok: true, id: 3, name: "c" }; +function keysOf(value: { id: number; name: string; ok: boolean }): string { + const keys: string[] = []; + for (const key in value) keys.push(key); + return keys.join(","); +} +for (const value of [a, b, c]) { + const seen: string[] = []; + for (const key in value) seen.push(key.toUpperCase()); + console.log(seen.join(" "), keysOf(value)); +} diff --git a/tests/corpus/7125-key-order-inspect.ts b/tests/corpus/7125-key-order-inspect.ts new file mode 100644 index 000000000..6a11eaf2c --- /dev/null +++ b/tests/corpus/7125-key-order-inspect.ts @@ -0,0 +1,12 @@ +import { inspect } from "node:util"; + +// console.log and util.inspect of object literals that write the same keys +// in different orders, alone and nested. +const a = { id: 1, name: "a", ok: true }; +const b = { name: "b", ok: false, id: 2 }; +const c = { ok: true, id: 3, name: "c" }; +console.log(a, b, c); +console.log([a, b, c]); +console.log({ outer: { inner: b }, list: [c] }); +console.log(inspect(b, { depth: 0 }), inspect({ deep: { deeper: c } }, { depth: 0 })); +console.log(`${inspect(c)} / ${inspect(a)}`); diff --git a/tests/corpus/7126-key-order-copies.ts b/tests/corpus/7126-key-order-copies.ts new file mode 100644 index 000000000..daafba23b --- /dev/null +++ b/tests/corpus/7126-key-order-copies.ts @@ -0,0 +1,19 @@ +// Copies of object literals that write the same keys in different orders: +// spreads, spreads with other keys before and after, rest bindings, +// Object.assign and structuredClone keep each value's own order. +const a = { id: 1, name: "a", ok: true }; +const b = { name: "b", ok: false, id: 2 }; +const c = { ok: true, id: 3, name: "c" }; +for (const value of [a, b, c]) { + const copy = { ...value }; + const changed = { ...value, id: 10 }; + const after = { ...value, extra: "e" }; + const before = { first: 0, ...value }; + const { name, ...rest } = value; + const assigned = Object.assign({}, value); + const cloned = structuredClone(value); + console.log(JSON.stringify(copy), JSON.stringify(changed)); + console.log(JSON.stringify(after), Object.keys(before).join(",")); + console.log(name, JSON.stringify(rest), Object.keys(rest).join(",")); + console.log(JSON.stringify(assigned), JSON.stringify(cloned)); +} diff --git a/tests/corpus/7127-key-order-dynamic.ts b/tests/corpus/7127-key-order-dynamic.ts new file mode 100644 index 000000000..55970cb40 --- /dev/null +++ b/tests/corpus/7127-key-order-dynamic.ts @@ -0,0 +1,30 @@ +import { MessageChannel } from "node:worker_threads"; + +// Object literals that write the same keys in different orders, seen +// through untyped values: their keys, JSON text, own property names, and a +// MessageChannel message. +const a = { id: 1, name: "a", ok: true }; +const b = { name: "b", ok: false, id: 2 }; +const c = { ok: true, id: 3, name: "c" }; +const values: unknown[] = [a, b, c]; +for (const value of values) { + const object = value as Record; + console.log(Object.keys(object).join(","), JSON.stringify(object)); + console.log(Object.getOwnPropertyNames(object).join(",")); +} +const { port1, port2 } = new MessageChannel(); +let received = 0; +port1.on("message", (message: unknown) => { + console.log("message", JSON.stringify(message), Object.keys(message as object).join(",")); + if (++received === 3) port1.close(); +}); +for (const value of [a, b, c]) port2.postMessage(value); + +// A nested value seen through an untyped view keeps its identity and its +// own key order, and writes through the view reach it. +const holder = { inner: b, tag: "h" }; +const view = holder as unknown as Record; +const inner = view.inner as Record; +console.log(view.inner === view.inner, Object.keys(inner).join(",")); +inner.id = 20; +console.log(b.id, JSON.stringify(b), JSON.stringify(view)); diff --git a/tests/corpus/7128-key-order-late-writes.ts b/tests/corpus/7128-key-order-late-writes.ts new file mode 100644 index 000000000..b1afd8761 --- /dev/null +++ b/tests/corpus/7128-key-order-late-writes.ts @@ -0,0 +1,27 @@ +// Optional keys added after construction go last, as in Node, also for +// shapes whose literals write their keys in different orders: a write of a +// missing key, a delete and a later write, and writes in a loop. +type Item = { id: number; name?: string; note?: string; count: number }; + +const items: Item[] = [ + { id: 1, count: 0 }, + { count: 1, name: "b", id: 2 }, + { note: "c", id: 3, count: 2 }, +]; +items[0]!.note = "added"; +items[0]!.name = "later"; +items[1]!.name = "kept"; +delete items[2]!.note; +items[2]!.note = "again"; +for (const item of items) { + if (item.count > 0) item.name ??= "filled"; + console.log(JSON.stringify(item), Object.keys(item).join(",")); +} +const built: Item = { count: 9, id: 9 }; +for (let i = 0; i < 2; i++) { + if (i === 1) built.note = `n${i}`; + console.log(Object.keys(built).join(",")); +} +console.log(built); +// d +// e diff --git a/tests/harness/corpus-costs/7122-key-order-json.ts.json b/tests/harness/corpus-costs/7122-key-order-json.ts.json new file mode 100644 index 000000000..aa8658007 --- /dev/null +++ b/tests/harness/corpus-costs/7122-key-order-json.ts.json @@ -0,0 +1 @@ +{ "plain": 2200, "san": 3600 } diff --git a/tests/harness/corpus-costs/7123-key-order-keys-values-entries.ts.json b/tests/harness/corpus-costs/7123-key-order-keys-values-entries.ts.json new file mode 100644 index 000000000..58d7b29f8 --- /dev/null +++ b/tests/harness/corpus-costs/7123-key-order-keys-values-entries.ts.json @@ -0,0 +1 @@ +{ "plain": 1500, "san": 2800 } diff --git a/tests/harness/corpus-costs/7124-key-order-for-in.ts.json b/tests/harness/corpus-costs/7124-key-order-for-in.ts.json new file mode 100644 index 000000000..e0b1e4dd1 --- /dev/null +++ b/tests/harness/corpus-costs/7124-key-order-for-in.ts.json @@ -0,0 +1 @@ +{ "plain": 1600, "san": 2100 } diff --git a/tests/harness/corpus-costs/7125-key-order-inspect.ts.json b/tests/harness/corpus-costs/7125-key-order-inspect.ts.json new file mode 100644 index 000000000..932ea4809 --- /dev/null +++ b/tests/harness/corpus-costs/7125-key-order-inspect.ts.json @@ -0,0 +1 @@ +{ "plain": 1600, "san": 2500 } diff --git a/tests/harness/corpus-costs/7126-key-order-copies.ts.json b/tests/harness/corpus-costs/7126-key-order-copies.ts.json new file mode 100644 index 000000000..ba3e270ee --- /dev/null +++ b/tests/harness/corpus-costs/7126-key-order-copies.ts.json @@ -0,0 +1 @@ +{ "plain": 1700, "san": 2400 } diff --git a/tests/harness/corpus-costs/7127-key-order-dynamic.ts.json b/tests/harness/corpus-costs/7127-key-order-dynamic.ts.json new file mode 100644 index 000000000..12680c6af --- /dev/null +++ b/tests/harness/corpus-costs/7127-key-order-dynamic.ts.json @@ -0,0 +1 @@ +{ "plain": 1800, "san": 2700 } diff --git a/tests/harness/corpus-costs/7128-key-order-late-writes.ts.json b/tests/harness/corpus-costs/7128-key-order-late-writes.ts.json new file mode 100644 index 000000000..0b945a1c9 --- /dev/null +++ b/tests/harness/corpus-costs/7128-key-order-late-writes.ts.json @@ -0,0 +1 @@ +{ "plain": 1700, "san": 2500 } From ce47b9bad95fd3284f151cd5c04d0ecc9dea881a Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 05:54:53 -0500 Subject: [PATCH 009/197] Number the key-order corpus programs 7132 to 7138 Corpus 7120 to 7129 are taken on main. --- .../order-parity/tests/corpus/7122-key-order-json.ts.json | 6 ------ .../tests/corpus/7123-key-order-keys-values-entries.ts.json | 6 ------ .../order-parity/tests/corpus/7124-key-order-for-in.ts.json | 6 ------ .../tests/corpus/7125-key-order-inspect.ts.json | 6 ------ .../order-parity/tests/corpus/7126-key-order-copies.ts.json | 6 ------ .../tests/corpus/7127-key-order-dynamic.ts.json | 6 ------ .../tests/corpus/7128-key-order-late-writes.ts.json | 6 ------ .../order-parity/tests/corpus/7132-key-order-json.ts.json | 6 ++++++ .../tests/corpus/7133-key-order-keys-values-entries.ts.json | 6 ++++++ .../order-parity/tests/corpus/7134-key-order-for-in.ts.json | 6 ++++++ .../tests/corpus/7135-key-order-inspect.ts.json | 6 ++++++ .../order-parity/tests/corpus/7136-key-order-copies.ts.json | 6 ++++++ .../tests/corpus/7137-key-order-dynamic.ts.json | 6 ++++++ .../tests/corpus/7138-key-order-late-writes.ts.json | 6 ++++++ .../{7122-key-order-json.ts => 7132-key-order-json.ts} | 0 ...ues-entries.ts => 7133-key-order-keys-values-entries.ts} | 0 .../{7124-key-order-for-in.ts => 7134-key-order-for-in.ts} | 0 ...{7125-key-order-inspect.ts => 7135-key-order-inspect.ts} | 0 .../{7126-key-order-copies.ts => 7136-key-order-copies.ts} | 0 ...{7127-key-order-dynamic.ts => 7137-key-order-dynamic.ts} | 0 ...y-order-late-writes.ts => 7138-key-order-late-writes.ts} | 0 ...2-key-order-json.ts.json => 7132-key-order-json.ts.json} | 0 ...s.ts.json => 7133-key-order-keys-values-entries.ts.json} | 0 ...y-order-for-in.ts.json => 7134-key-order-for-in.ts.json} | 0 ...order-inspect.ts.json => 7135-key-order-inspect.ts.json} | 0 ...y-order-copies.ts.json => 7136-key-order-copies.ts.json} | 0 ...order-dynamic.ts.json => 7137-key-order-dynamic.ts.json} | 0 ...te-writes.ts.json => 7138-key-order-late-writes.ts.json} | 0 28 files changed, 42 insertions(+), 42 deletions(-) delete mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7122-key-order-json.ts.json delete mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7123-key-order-keys-values-entries.ts.json delete mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7124-key-order-for-in.ts.json delete mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7125-key-order-inspect.ts.json delete mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7126-key-order-copies.ts.json delete mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7127-key-order-dynamic.ts.json delete mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7128-key-order-late-writes.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7132-key-order-json.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7133-key-order-keys-values-entries.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7134-key-order-for-in.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7135-key-order-inspect.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7136-key-order-copies.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7137-key-order-dynamic.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7138-key-order-late-writes.ts.json rename tests/corpus/{7122-key-order-json.ts => 7132-key-order-json.ts} (100%) rename tests/corpus/{7123-key-order-keys-values-entries.ts => 7133-key-order-keys-values-entries.ts} (100%) rename tests/corpus/{7124-key-order-for-in.ts => 7134-key-order-for-in.ts} (100%) rename tests/corpus/{7125-key-order-inspect.ts => 7135-key-order-inspect.ts} (100%) rename tests/corpus/{7126-key-order-copies.ts => 7136-key-order-copies.ts} (100%) rename tests/corpus/{7127-key-order-dynamic.ts => 7137-key-order-dynamic.ts} (100%) rename tests/corpus/{7128-key-order-late-writes.ts => 7138-key-order-late-writes.ts} (100%) rename tests/harness/corpus-costs/{7122-key-order-json.ts.json => 7132-key-order-json.ts.json} (100%) rename tests/harness/corpus-costs/{7123-key-order-keys-values-entries.ts.json => 7133-key-order-keys-values-entries.ts.json} (100%) rename tests/harness/corpus-costs/{7124-key-order-for-in.ts.json => 7134-key-order-for-in.ts.json} (100%) rename tests/harness/corpus-costs/{7125-key-order-inspect.ts.json => 7135-key-order-inspect.ts.json} (100%) rename tests/harness/corpus-costs/{7126-key-order-copies.ts.json => 7136-key-order-copies.ts.json} (100%) rename tests/harness/corpus-costs/{7127-key-order-dynamic.ts.json => 7137-key-order-dynamic.ts.json} (100%) rename tests/harness/corpus-costs/{7128-key-order-late-writes.ts.json => 7138-key-order-late-writes.ts.json} (100%) diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7122-key-order-json.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7122-key-order-json.ts.json deleted file mode 100644 index 3551ad6b1..000000000 --- a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7122-key-order-json.ts.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "order": [ - "/tests/corpus/7122-key-order-json.ts" - ], - "diags": [] -} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7123-key-order-keys-values-entries.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7123-key-order-keys-values-entries.ts.json deleted file mode 100644 index 4451e935a..000000000 --- a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7123-key-order-keys-values-entries.ts.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "order": [ - "/tests/corpus/7123-key-order-keys-values-entries.ts" - ], - "diags": [] -} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7124-key-order-for-in.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7124-key-order-for-in.ts.json deleted file mode 100644 index e87101734..000000000 --- a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7124-key-order-for-in.ts.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "order": [ - "/tests/corpus/7124-key-order-for-in.ts" - ], - "diags": [] -} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7125-key-order-inspect.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7125-key-order-inspect.ts.json deleted file mode 100644 index b13a13261..000000000 --- a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7125-key-order-inspect.ts.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "order": [ - "/tests/corpus/7125-key-order-inspect.ts" - ], - "diags": [] -} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7126-key-order-copies.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7126-key-order-copies.ts.json deleted file mode 100644 index a9f55663d..000000000 --- a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7126-key-order-copies.ts.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "order": [ - "/tests/corpus/7126-key-order-copies.ts" - ], - "diags": [] -} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7127-key-order-dynamic.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7127-key-order-dynamic.ts.json deleted file mode 100644 index 5a091d157..000000000 --- a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7127-key-order-dynamic.ts.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "order": [ - "/tests/corpus/7127-key-order-dynamic.ts" - ], - "diags": [] -} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7128-key-order-late-writes.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7128-key-order-late-writes.ts.json deleted file mode 100644 index 407a2179e..000000000 --- a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7128-key-order-late-writes.ts.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "order": [ - "/tests/corpus/7128-key-order-late-writes.ts" - ], - "diags": [] -} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7132-key-order-json.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7132-key-order-json.ts.json new file mode 100644 index 000000000..2609ff929 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7132-key-order-json.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7132-key-order-json.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7133-key-order-keys-values-entries.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7133-key-order-keys-values-entries.ts.json new file mode 100644 index 000000000..40729240a --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7133-key-order-keys-values-entries.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7133-key-order-keys-values-entries.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7134-key-order-for-in.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7134-key-order-for-in.ts.json new file mode 100644 index 000000000..e44503626 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7134-key-order-for-in.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7134-key-order-for-in.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7135-key-order-inspect.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7135-key-order-inspect.ts.json new file mode 100644 index 000000000..431bf5c9c --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7135-key-order-inspect.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7135-key-order-inspect.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7136-key-order-copies.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7136-key-order-copies.ts.json new file mode 100644 index 000000000..fea64a145 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7136-key-order-copies.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7136-key-order-copies.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7137-key-order-dynamic.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7137-key-order-dynamic.ts.json new file mode 100644 index 000000000..3b7cab36f --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7137-key-order-dynamic.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7137-key-order-dynamic.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7138-key-order-late-writes.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7138-key-order-late-writes.ts.json new file mode 100644 index 000000000..7f5302cd5 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7138-key-order-late-writes.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7138-key-order-late-writes.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7122-key-order-json.ts b/tests/corpus/7132-key-order-json.ts similarity index 100% rename from tests/corpus/7122-key-order-json.ts rename to tests/corpus/7132-key-order-json.ts diff --git a/tests/corpus/7123-key-order-keys-values-entries.ts b/tests/corpus/7133-key-order-keys-values-entries.ts similarity index 100% rename from tests/corpus/7123-key-order-keys-values-entries.ts rename to tests/corpus/7133-key-order-keys-values-entries.ts diff --git a/tests/corpus/7124-key-order-for-in.ts b/tests/corpus/7134-key-order-for-in.ts similarity index 100% rename from tests/corpus/7124-key-order-for-in.ts rename to tests/corpus/7134-key-order-for-in.ts diff --git a/tests/corpus/7125-key-order-inspect.ts b/tests/corpus/7135-key-order-inspect.ts similarity index 100% rename from tests/corpus/7125-key-order-inspect.ts rename to tests/corpus/7135-key-order-inspect.ts diff --git a/tests/corpus/7126-key-order-copies.ts b/tests/corpus/7136-key-order-copies.ts similarity index 100% rename from tests/corpus/7126-key-order-copies.ts rename to tests/corpus/7136-key-order-copies.ts diff --git a/tests/corpus/7127-key-order-dynamic.ts b/tests/corpus/7137-key-order-dynamic.ts similarity index 100% rename from tests/corpus/7127-key-order-dynamic.ts rename to tests/corpus/7137-key-order-dynamic.ts diff --git a/tests/corpus/7128-key-order-late-writes.ts b/tests/corpus/7138-key-order-late-writes.ts similarity index 100% rename from tests/corpus/7128-key-order-late-writes.ts rename to tests/corpus/7138-key-order-late-writes.ts diff --git a/tests/harness/corpus-costs/7122-key-order-json.ts.json b/tests/harness/corpus-costs/7132-key-order-json.ts.json similarity index 100% rename from tests/harness/corpus-costs/7122-key-order-json.ts.json rename to tests/harness/corpus-costs/7132-key-order-json.ts.json diff --git a/tests/harness/corpus-costs/7123-key-order-keys-values-entries.ts.json b/tests/harness/corpus-costs/7133-key-order-keys-values-entries.ts.json similarity index 100% rename from tests/harness/corpus-costs/7123-key-order-keys-values-entries.ts.json rename to tests/harness/corpus-costs/7133-key-order-keys-values-entries.ts.json diff --git a/tests/harness/corpus-costs/7124-key-order-for-in.ts.json b/tests/harness/corpus-costs/7134-key-order-for-in.ts.json similarity index 100% rename from tests/harness/corpus-costs/7124-key-order-for-in.ts.json rename to tests/harness/corpus-costs/7134-key-order-for-in.ts.json diff --git a/tests/harness/corpus-costs/7125-key-order-inspect.ts.json b/tests/harness/corpus-costs/7135-key-order-inspect.ts.json similarity index 100% rename from tests/harness/corpus-costs/7125-key-order-inspect.ts.json rename to tests/harness/corpus-costs/7135-key-order-inspect.ts.json diff --git a/tests/harness/corpus-costs/7126-key-order-copies.ts.json b/tests/harness/corpus-costs/7136-key-order-copies.ts.json similarity index 100% rename from tests/harness/corpus-costs/7126-key-order-copies.ts.json rename to tests/harness/corpus-costs/7136-key-order-copies.ts.json diff --git a/tests/harness/corpus-costs/7127-key-order-dynamic.ts.json b/tests/harness/corpus-costs/7137-key-order-dynamic.ts.json similarity index 100% rename from tests/harness/corpus-costs/7127-key-order-dynamic.ts.json rename to tests/harness/corpus-costs/7137-key-order-dynamic.ts.json diff --git a/tests/harness/corpus-costs/7128-key-order-late-writes.ts.json b/tests/harness/corpus-costs/7138-key-order-late-writes.ts.json similarity index 100% rename from tests/harness/corpus-costs/7128-key-order-late-writes.ts.json rename to tests/harness/corpus-costs/7138-key-order-late-writes.ts.json From 4cb46d6472af2f82aa2161c0ae6bccf438389449 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 06:39:06 -0500 Subject: [PATCH 010/197] Keep a key-order divergence in the coverage fixture The fixture's Point literals now each record their own key order, so they no longer diverge. The fixture adds a spread that sets an optional key its source lacks, which keeps the type's declaration order, so SC6002 still reports the literal written in another order. --- tests/coverage-fixtures/divergences.ts | 10 +++++++--- tests/harness/__snapshots__/coverage-divergences.txt | 10 +++++----- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/tests/coverage-fixtures/divergences.ts b/tests/coverage-fixtures/divergences.ts index 83c3ede49..0cf8bcae5 100644 --- a/tests/coverage-fixtures/divergences.ts +++ b/tests/coverage-fixtures/divergences.ts @@ -20,14 +20,18 @@ console.log(tagged.label); const view: Labeled = tagged; console.log(view === tagged); -// Key order: values of one type are built in two orders and the program -// observes them, so the type keeps its declaration order. +// Key order: values of one type are built in two orders, and a spread sets +// an optional key its source lacks, so the values cannot record their own +// orders and the type keeps its declaration order. interface Point { x: number; y: number; + label?: string; } const point: Point = { y: 1, x: 2 }; -console.log(JSON.stringify(point), JSON.stringify({ x: 3, y: 4 })); +const other: Point = { x: 3, y: 4 }; +const labeled: Point = { ...point, label: "p" }; +console.log(JSON.stringify(point), JSON.stringify(other), JSON.stringify(labeled)); // Text outside the built-in collation compares by code point natively. console.log(["б", "а"].sort((p, q) => p.localeCompare(q)).join(","), "яблоко".localeCompare("груша")); diff --git a/tests/harness/__snapshots__/coverage-divergences.txt b/tests/harness/__snapshots__/coverage-divergences.txt index 66373655f..8ab286873 100644 --- a/tests/harness/__snapshots__/coverage-divergences.txt +++ b/tests/harness/__snapshots__/coverage-divergences.txt @@ -1,7 +1,7 @@ scriptc coverage tests/coverage-fixtures/divergences.ts - statements analyzed 18 - compile statically 18 (100%) + statements analyzed 20 + compile statically 20 (100%) in dynamic tier 1 (operate on values typed only at runtime; --explain-dynamic lists them) fully static — all analyzed statements compile to native code. @@ -14,11 +14,11 @@ scriptc coverage tests/coverage-fixtures/divergences.ts at tests/coverage-fixtures/divergences.ts:20:23 hint: declare the parameter or binding with the value's full type ('Tagged'), or make the shape a class — class instances keep their identity when passed as a base class ×1 SC6002 this object's keys enumerate as 'x, y' natively (the declaration order of 'Point') but as 'y, x' under Node (insertion order), and the program observes the order (JSON.stringify, Object.keys, for-in … - at tests/coverage-fixtures/divergences.ts:29:22 + at tests/coverage-fixtures/divergences.ts:31:22 hint: write the properties in the declaration order of 'Point' (or reorder the declaration to match how objects are built) ×1 SC6003 localeCompare over 'яблоко' compares characters outside the built-in collation (Latin letters, digits, punctuation and symbols) by code point natively, while Node applies ICU's full collation - at tests/coverage-fixtures/divergences.ts:33:70 + at tests/coverage-fixtures/divergences.ts:37:70 hint: results agree with Node for Latin-script text; for other scripts compare explicitly (for example by code point with < and >) so both runtimes agree ×1 SC6004 this 'Circle' reaches a cast to the unrelated class 'Square' in the callee: natively the checked cast throws a TypeError when it executes, while Node reads the members structurally - at tests/coverage-fixtures/divergences.ts:48:20 + at tests/coverage-fixtures/divergences.ts:52:20 hint: narrow with 'instanceof Square' before casting, or cast to a base class or interface both classes share \ No newline at end of file From 41ef12e4633e678272bb07eef8f2cee5af9f2242 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 06:44:59 -0500 Subject: [PATCH 011/197] Construct builtins through stored constructors and fix function-local class names and dispatch --- .../limitations/builtin-apis/prototypes.mdx | 4 +- .../local-class-expressions.mdx | 3 +- .../lowering/class-runtime-statics.ts | 38 +++- .../lowering/expressions/identifiers.ts | 3 +- .../lowering/expressions/object-literals.ts | 3 + .../src/frontend/lowering/lower-classes.ts | 173 +++++++++++++++++- .../src/frontend/lowering/lower-stmts.ts | 8 + .../compiler/src/frontend/lowering/lowerer.ts | 7 +- .../src/frontend/lowering/surfaces.ts | 13 +- ...6913-builtin-constructor-shorthand.ts.json | 6 + .../6914-dynamic-builtin-construct.ts.json | 6 + .../6915-dynamic-builtin-construct.mjs.json | 6 + ...local-class-static-self-reference.mjs.json | 6 + ...al-class-expression-self-reference.ts.json | 6 + ...l-class-expression-self-reference.mjs.json | 6 + .../6919-local-class-inspect-name.mjs.json | 6 + ...1-local-subclass-override-dispatch.ts.json | 6 + ...-local-subclass-override-dispatch.mjs.json | 6 + packages/runtime/src/scr_dyn.h | 9 + packages/runtime/src/scr_dyn_invoke.c | 32 ++++ packages/runtime/src/scr_json.c | 33 ++++ packages/runtime/src/scr_map.c | 2 +- packages/runtime/src/scr_map.h | 4 +- .../6913-builtin-constructor-shorthand.ts | 10 + .../corpus/6914-dynamic-builtin-construct.ts | 54 ++++++ .../corpus/6915-dynamic-builtin-construct.mjs | 16 ++ ...6916-local-class-static-self-reference.mjs | 52 ++++++ ...7-local-class-expression-self-reference.ts | 41 +++++ ...-local-class-expression-self-reference.mjs | 50 +++++ .../corpus/6919-local-class-inspect-name.mjs | 51 ++++++ .../6921-local-subclass-override-dispatch.ts | 51 ++++++ .../6922-local-subclass-override-dispatch.mjs | 51 ++++++ tests/diagnostics/local-class-boundaries.ts | 2 - .../local-class-boundaries.ts.txt | 17 +- ...6913-builtin-constructor-shorthand.ts.json | 1 + .../6914-dynamic-builtin-construct.ts.json | 1 + .../6915-dynamic-builtin-construct.mjs.json | 1 + ...local-class-static-self-reference.mjs.json | 1 + ...al-class-expression-self-reference.ts.json | 1 + ...l-class-expression-self-reference.mjs.json | 1 + .../6919-local-class-inspect-name.mjs.json | 1 + ...1-local-subclass-override-dispatch.ts.json | 1 + ...-local-subclass-override-dispatch.mjs.json | 1 + 43 files changed, 758 insertions(+), 32 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6913-builtin-constructor-shorthand.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6914-dynamic-builtin-construct.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6915-dynamic-builtin-construct.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6916-local-class-static-self-reference.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6917-local-class-expression-self-reference.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6918-local-class-expression-self-reference.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6919-local-class-inspect-name.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6921-local-subclass-override-dispatch.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6922-local-subclass-override-dispatch.mjs.json create mode 100644 tests/corpus/6913-builtin-constructor-shorthand.ts create mode 100644 tests/corpus/6914-dynamic-builtin-construct.ts create mode 100644 tests/corpus/6915-dynamic-builtin-construct.mjs create mode 100644 tests/corpus/6916-local-class-static-self-reference.mjs create mode 100644 tests/corpus/6917-local-class-expression-self-reference.ts create mode 100644 tests/corpus/6918-local-class-expression-self-reference.mjs create mode 100644 tests/corpus/6919-local-class-inspect-name.mjs create mode 100644 tests/corpus/6921-local-subclass-override-dispatch.ts create mode 100644 tests/corpus/6922-local-subclass-override-dispatch.mjs create mode 100644 tests/harness/corpus-costs/6913-builtin-constructor-shorthand.ts.json create mode 100644 tests/harness/corpus-costs/6914-dynamic-builtin-construct.ts.json create mode 100644 tests/harness/corpus-costs/6915-dynamic-builtin-construct.mjs.json create mode 100644 tests/harness/corpus-costs/6916-local-class-static-self-reference.mjs.json create mode 100644 tests/harness/corpus-costs/6917-local-class-expression-self-reference.ts.json create mode 100644 tests/harness/corpus-costs/6918-local-class-expression-self-reference.mjs.json create mode 100644 tests/harness/corpus-costs/6919-local-class-inspect-name.mjs.json create mode 100644 tests/harness/corpus-costs/6921-local-subclass-override-dispatch.ts.json create mode 100644 tests/harness/corpus-costs/6922-local-subclass-override-dispatch.mjs.json diff --git a/docs/content/limitations/builtin-apis/prototypes.mdx b/docs/content/limitations/builtin-apis/prototypes.mdx index 3e0b5789d..170b11793 100644 --- a/docs/content/limitations/builtin-apis/prototypes.mdx +++ b/docs/content/limitations/builtin-apis/prototypes.mdx @@ -3,4 +3,6 @@ Plain objects, arrays, functions, class instances, and Date, RegExp, Map, Set, WeakMap, WeakSet, and Promise values inherit `constructor`, `valueOf`, `toString`, `hasOwnProperty`, and `propertyIsEnumerable` from `Object.prototype` as in Node, so `value.constructor === Object` and `value.valueOf === Object.prototype.valueOf` compare identities exactly. Named reads such as `value.valueOf` answer these inherited members; computed-key reads such as `value[key]` do not. `Object.prototype`'s own members read as values; its mutation and other reflection, such as `isPrototypeOf` and `__proto__`, are unsupported. - The constructors `Object`, `Function`, `Date`, `RegExp`, `Map`, `Set`, `WeakMap`, `WeakSet`, and `Promise` read as function values; through a stored value only `Object` can be called, and `instanceof` and `new` against them remain unsupported. + The constructors `Object`, `Function`, `Date`, `RegExp`, `Map`, `Set`, `WeakMap`, `WeakSet`, and `Promise` read as function values; through a stored value only `Object` can be called. + `new` through a stored `Object`, `Date`, `Map`, `Set`, `WeakMap`, or `WeakSet` builds the same value as the `new` expression. + `new` through a stored `RegExp` or `Promise`, `new` of a stored `Object` with a primitive argument, and `instanceof` against these constructors are unsupported. diff --git a/docs/content/limitations/types-and-class-definitions/local-class-expressions.mdx b/docs/content/limitations/types-and-class-definitions/local-class-expressions.mdx index 173bdc0c6..de24038cc 100644 --- a/docs/content/limitations/types-and-class-definitions/local-class-expressions.mdx +++ b/docs/content/limitations/types-and-class-definitions/local-class-expressions.mdx @@ -1,5 +1,6 @@ -- **Local class expressions:** Static fields and static blocks in TypeScript files, decorators, private brands, generic members, and references through the class's internal name are unsupported. +- **Local class expressions:** Static fields and static blocks in TypeScript files, decorators, private brands, and generic members are unsupported. Static methods read the enclosing function's bindings and are inherited by subclasses. + A named class expression's methods and static members can refer to the class by its name. Supported expressions preserve fresh constructor identity and shared captures across constructors, methods, accessors, and arrow fields. Local inheritance requires a compiled base class. Constructors retain identity, `typeof`, `name`, and `length` through `unknown`; cast back to their exact type before construction. diff --git a/packages/compiler/src/frontend/lowering/class-runtime-statics.ts b/packages/compiler/src/frontend/lowering/class-runtime-statics.ts index 310c8f389..707c916ec 100644 --- a/packages/compiler/src/frontend/lowering/class-runtime-statics.ts +++ b/packages/compiler/src/frontend/lowering/class-runtime-statics.ts @@ -1,6 +1,6 @@ import { dynUndefinedExpr, varRef } from "../../ir/build.js"; import * as ts from "../ts7/adapter.js"; -import { BOOL, DYN, STRING, type IrExpr, type IrStmt } from "../../ir/ir.js"; +import { BOOL, DYN, STRING, type IrExpr, type IrLocal, type IrStmt } from "../../ir/ir.js"; import { locOf } from "../program.js"; import type { Lowerer } from "./lowerer.js"; import { classValueRef, type ClassInfo } from "./lower-classes.js"; @@ -15,9 +15,29 @@ export function hasRuntimeStatics(info: ClassInfo): boolean { /** Constructor descriptors preserve inherited getter receivers and static * initialization order. Factory evaluations additionally own independent - * fields and closures when the same factory is called repeatedly. */ -export function initializeRuntimeStatics(lowerer: Lowerer, info: ClassInfo, value: IrExpr): IrExpr { - if (!info.runtimeStatics && !info.runtimePrototypeMembers) return value; + * fields and closures when the same factory is called repeatedly. + * + * `binding` is the local that holds the class under its own name (a + * function-local class declaration's binding, or the inner name of a + * function-local class expression whose body names itself), declared + * without a value. It is assigned before any static member is defined, so + * static initializers and methods read the class through it. */ +export function initializeRuntimeStatics( + lowerer: Lowerer, + info: ClassInfo, + value: IrExpr, + binding?: IrLocal, +): IrExpr { + if (!info.runtimeStatics && !info.runtimePrototypeMembers) { + if (!binding) return value; + return { + kind: "seqExpr", + stmts: [{ kind: "assign", localId: binding.id, value, loc: value.loc }], + result: varRef(binding.id, binding.type, value.loc), + type: binding.type, + loc: value.loc, + }; + } const loc = value.loc; const local = lowerer.declareHiddenLocal("%staticClass", value.type); const receiver = varRef(local.id, value.type, loc); @@ -28,6 +48,16 @@ export function initializeRuntimeStatics(lowerer: Lowerer, info: ClassInfo, valu : dynUndefinedExpr(loc); const statements: IrStmt[] = [ { kind: "varDecl", localId: local.id, init: value, loc }, + ...(binding + ? [ + { + kind: "assign" as const, + localId: binding.id, + value: lowerer.coerceToExpected(receiver, binding.type), + loc, + }, + ] + : []), { kind: "exprStmt", expr: { kind: "libCall", fn: "dyn.classInherit", args: [boxed, base], type: DYN, loc }, diff --git a/packages/compiler/src/frontend/lowering/expressions/identifiers.ts b/packages/compiler/src/frontend/lowering/expressions/identifiers.ts index 1f74cea42..63175d50a 100644 --- a/packages/compiler/src/frontend/lowering/expressions/identifiers.ts +++ b/packages/compiler/src/frontend/lowering/expressions/identifiers.ts @@ -69,6 +69,7 @@ import { builtinModuleConstOf, builtinModuleFnOf, builtinModulesArrayLit, + identifierReadSymbol, stdlibGlobalNameOf, } from "../surfaces.js"; import { builtinConstantsObject } from "../lower-builtin-values.js"; @@ -753,7 +754,7 @@ export function lowerIdentifierExpr(lowerer: Lowerer, expr: ts.Identifier, loc: return lowerGlobalValue(lowerer, expr); } { - const sym = lowerer.checker.getSymbolAtLocation(expr); + const sym = identifierReadSymbol(lowerer, expr); // The temporal global is typed (TS 7's lib slice) but Node 24 does // not define it: the read throws Node's ReferenceError. if (isTemporalGlobalRoot(lowerer, expr, sym)) { diff --git a/packages/compiler/src/frontend/lowering/expressions/object-literals.ts b/packages/compiler/src/frontend/lowering/expressions/object-literals.ts index 678b375fe..155529920 100644 --- a/packages/compiler/src/frontend/lowering/expressions/object-literals.ts +++ b/packages/compiler/src/frontend/lowering/expressions/object-literals.ts @@ -3262,6 +3262,9 @@ export function lowerShorthandValue( } return lowerer.classValueRef(classInfo, propName); } + // A standard-library global (`{ Date }`): the value the long form + // `{ Date: Date }` reads. + if (lowerer.isStdlibSymbol(resolved)) return lowerer.lowerExpr(propName); } lowerer.rejectUnresolvedSymbol( symbol diff --git a/packages/compiler/src/frontend/lowering/lower-classes.ts b/packages/compiler/src/frontend/lowering/lower-classes.ts index d8ccc15b7..c74969d10 100644 --- a/packages/compiler/src/frontend/lowering/lower-classes.ts +++ b/packages/compiler/src/frontend/lowering/lower-classes.ts @@ -6034,6 +6034,31 @@ export function lowerClassExpressionInfo( return info; } +/** True when a named class expression's body refers to the class value by + * its own name (`class Box { static of() { return new Box(); } }`); type + * annotations naming it (`again(): Box`) do not count. */ +function classExpressionNamesItself(lowerer: Lowerer, expr: ts.ClassExpression): boolean { + const name = expr.name; + if (!name) return false; + const symbol = lowerer.checker.getSymbolAtLocation(name); + if (!symbol) return false; + let found = false; + for (const member of expr.members.nodes) { + ts.walkPreorder(member, (n) => { + if (found || ts.isTypeNode(n)) return "skip"; + if ( + ts.isIdentifier(n) && + n.text === name.text && + lowerer.checker.getSymbolAtLocation(n) === symbol + ) + found = true; + return undefined; + }); + if (found) return true; + } + return false; +} + /** `class {…}` in expression position: a class definition bound to no * statement — once the static side is a value, the expression IS the * definition plus a classRef over it. A DECORATED class expression @@ -6041,8 +6066,16 @@ export function lowerClassExpressionInfo( * mints a class at all: evaluating the expression IS the * ReferenceError, so it lowers to exactly that read — every evaluation * throws identically, which is why the once-evaluated restriction and - * the member fences don't apply. */ -export function lowerClassExpression(lowerer: Lowerer, expr: ts.ClassLikeDeclaration): IrExpr { + * the member fences don't apply. + * + * `binding` is the local of a function-local class declaration's name, + * declared without a value; the class value assigns it before the class's + * static members are defined (see initializeRuntimeStatics). */ +export function lowerClassExpression( + lowerer: Lowerer, + expr: ts.ClassLikeDeclaration, + binding?: IrLocal, +): IrExpr { if ( decoratorNodesOf(expr).length > 0 || expr.members.nodes.some((m) => decoratorNodesOf(m).length > 0) @@ -6058,6 +6091,41 @@ export function lowerClassExpression(lowerer: Lowerer, expr: ts.ClassLikeDeclara } } const info = lowerClassExpressionInfo(lowerer, expr); + // A function-local class expression's own name is a binding of its body: + // declared before the members lower, so their references to the name + // resolve to it. The class value captures it before it is assigned, as + // it captures a class declaration's binding. + if ( + !binding && + info.localClass && + ts.isClassExpression(expr) && + expr.name && + classExpressionNamesItself(lowerer, expr) + ) { + const own = lowerer.declareLocal( + expr.name, + expr.name.text, + { kind: "classval", className: info.def.name }, + true, + ); + const value = lowerClassValue(lowerer, expr, info, own); + return { + kind: "seqExpr", + stmts: [{ kind: "varDecl", localId: own.id, init: null, loc: value.loc }], + result: value, + type: value.type, + loc: value.loc, + }; + } + return lowerClassValue(lowerer, expr, info, binding); +} + +function lowerClassValue( + lowerer: Lowerer, + expr: ts.ClassLikeDeclaration, + info: ClassInfo, + binding: IrLocal | undefined, +): IrExpr { if ( !info.localClass && !info.expressionInitializationEmitted && @@ -6104,7 +6172,7 @@ export function lowerClassExpression(lowerer: Lowerer, expr: ts.ClassLikeDeclara const clsRef: IrExpr = { kind: "varRef", localId: clsLocal.id, type: value.type, loc }; const boxed: IrExpr = { kind: "dynFrom", value: clsRef, type: DYN, loc }; info.localPrototypeData = true; - return initializeRuntimeStatics(lowerer, info, { + const built: IrExpr = { kind: "seqExpr", stmts: [ { kind: "varDecl", localId: baseLocal.id, init: base, loc }, @@ -6137,7 +6205,8 @@ export function lowerClassExpression(lowerer: Lowerer, expr: ts.ClassLikeDeclara result: clsRef, type: value.type, loc, - }); + }; + return initializeRuntimeStatics(lowerer, info, built, binding); } if (info.localClass?.baseExpression && info.base && value.kind === "classRef") { const loc = locOf(expr); @@ -6152,7 +6221,7 @@ export function lowerClassExpression(lowerer: Lowerer, expr: ts.ClassLikeDeclara const clsRef: IrExpr = { kind: "varRef", localId: clsLocal.id, type: value.type, loc }; const basePrototype = classPrototypeData(lowerer, info.base, loc, baseRef)!; info.localPrototypeData = true; - return initializeRuntimeStatics(lowerer, info, { + const built: IrExpr = { kind: "seqExpr", stmts: [ { kind: "varDecl", localId: baseLocal.id, init: base, loc }, @@ -6191,13 +6260,15 @@ export function lowerClassExpression(lowerer: Lowerer, expr: ts.ClassLikeDeclara result: clsRef, type: value.type, loc, - }); + }; + return initializeRuntimeStatics(lowerer, info, built, binding); } if (!info.localClass) return value; const classValue = initializeRuntimeStatics( lowerer, info, value.kind === "classRef" ? { ...value, captures: info.localClass.captureSources } : value, + binding, ); // The base resolves statically here, but Node still evaluates the // heritage expression each time the class definition runs, first. @@ -7124,12 +7195,102 @@ export function inHierarchy(lowerer: Lowerer, info: ClassInfo): boolean { * dynamically. Abstract re-declarations don't count (they carry no * implementation; the concrete ones below them do, via the recursion). */ export function overrideBelow(lowerer: Lowerer, info: ClassInfo, name: string): boolean { + if (info.decl !== null && localOverrideBelow(lowerer, info.decl, name)) return true; return info.subclasses.some((s) => { const m = s.methods.get(name); return (m !== undefined && m.abstract !== true) || lowerer.overrideBelow(s, name); }); } +/** The class node an `extends` clause names: a class declaration, or the + * class expression a variable is initialized with. */ +function extendedClassNode(lowerer: Lowerer, decl: ts.ClassLikeDeclaration): ts.Node | null { + const heritage = decl.heritageClauses?.nodes.find( + (clause) => clause.token === ts.SyntaxKind.ExtendsKeyword, + )?.types?.nodes[0]?.expression; + if (!heritage || !ts.isIdentifier(heritage)) return null; + let symbol = lowerer.checker.getSymbolAtLocation(heritage); + if (symbol && symbol.flags & ts.SymbolFlags.Alias) + symbol = lowerer.checker.getAliasedSymbol(symbol); + const target = symbol?.valueDeclaration; + if (!target) return null; + if (ts.isClassDeclaration(target) || ts.isClassExpression(target)) return target; + if (ts.isVariableDeclaration(target) && target.initializer) { + let init: ts.Expression = target.initializer; + while (ts.isParenthesizedExpression(init)) init = init.expression; + if (ts.isClassExpression(init)) return init; + } + return null; +} + +/** The function-local classes that extend the class node `base`. Function + * bodies lower on demand, after code that may already call methods of a + * class their local classes extend, so such a subclass is not yet in its + * base's `subclasses` when those calls lower. */ +function localSubclassesOf(lowerer: Lowerer, base: ts.Node): readonly ts.ClassLikeDeclaration[] { + let index = lowerer.localSubclassIndex; + if (index === null) { + index = new Map(); + const files = lowerer.moduleOrder.length > 0 ? lowerer.moduleOrder : [lowerer.entry]; + for (const sf of files) { + if (!sf.text.includes("extends")) continue; + ts.walkPreorder(sf, (node) => { + if ( + (ts.isClassDeclaration(node) || ts.isClassExpression(node)) && + node.heritageClauses !== undefined && + functionLocalClass(node) + ) { + const extended = extendedClassNode(lowerer, node); + if (extended) { + const list = index!.get(extended); + if (list) list.push(node); + else index!.set(extended, [node]); + } + } + return undefined; + }); + } + lowerer.localSubclassIndex = index; + } + return index.get(base) ?? []; +} + +/** True when a function-local class below the class node `base` declares + * a concrete instance method or accessor `name` (accessors are keyed + * `get:x` and `set:x`). A call on `base`'s class then dispatches on the + * receiver; devirtualization turns it back into a direct call when no such + * override is reachable. */ +function localOverrideBelow( + lowerer: Lowerer, + base: ts.Node, + name: string, + visited?: Set, +): boolean { + const subclasses = localSubclassesOf(lowerer, base); + if (subclasses.length === 0) return false; + visited ??= new Set(); + if (visited.has(base)) return false; + visited.add(base); + const accessor = name.startsWith("get:") ? "get" : name.startsWith("set:") ? "set" : null; + const member = accessor ? name.slice(4) : name; + return subclasses.some( + (sub) => + sub.members.nodes.some( + (m) => + (accessor === null + ? ts.isMethodDeclaration(m) + : accessor === "get" + ? ts.isGetAccessorDeclaration(m) + : ts.isSetAccessorDeclaration(m)) && + m.body !== undefined && + m.name !== undefined && + (ts.isIdentifier(m.name) || ts.isStringLiteral(m.name)) && + m.name.text === member && + !ts.getModifiers(m)?.some((modifier) => modifier.kind === ts.SyntaxKind.StaticKeyword), + ) || localOverrideBelow(lowerer, sub, name, visited), + ); +} + /** The nearest GENERIC-method declaration of `name` at/above `info` — * findMethodOn's twin over the genericMethods tables. */ export function findGenericMethodOn( diff --git a/packages/compiler/src/frontend/lowering/lower-stmts.ts b/packages/compiler/src/frontend/lowering/lower-stmts.ts index 213abfec6..92c1946d5 100644 --- a/packages/compiler/src/frontend/lowering/lower-stmts.ts +++ b/packages/compiler/src/frontend/lowering/lower-stmts.ts @@ -1523,6 +1523,14 @@ export function lowerStmt(lowerer: Lowerer, stmt: ts.Statement): IrStmt | IrStmt const info = lowerer.lowerClassExpressionInfo(stmt); const type: IrType = { kind: "classval", className: info.def.name }; const local = lowerer.declareLocal(stmt.name, stmt.name.text, type, true); + // Static members of a function-local class run while the class value is + // built and may read the class by name: the class value assigns the + // binding before them. + if (info.localClass && (info.runtimeStatics || info.runtimePrototypeMembers)) + return [ + { kind: "varDecl", localId: local.id, init: null, loc: locOf(stmt) }, + { kind: "exprStmt", expr: lowerer.lowerClassExpression(stmt, local), loc: locOf(stmt) }, + ]; const init = lowerer.lowerClassExpression(stmt); return { kind: "varDecl", localId: local.id, init, loc: locOf(stmt) }; } diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index c95543c61..7334455af 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -3303,6 +3303,9 @@ export class Lowerer { * the class and constructed-class counts, it tells caches over the class * graph when the graph changed. */ subclassLinks = 0; + /** Function-local classes by the class node they extend, built on first + * use (localSubclassesOf in lower-classes.ts). */ + localSubclassIndex: Map | null = null; /** Set while a subclass constructor's super() call notes its base * constructor edge: that edge does not instantiate the base exactly. */ private superConstructorEdge = false; @@ -11377,8 +11380,8 @@ export class Lowerer { } } - lowerClassExpression(expr: ts.ClassLikeDeclaration): IrExpr { - return lowerClassExpression(this, expr); + lowerClassExpression(expr: ts.ClassLikeDeclaration, binding?: IrLocal): IrExpr { + return lowerClassExpression(this, expr, binding); } /* ── the class graph (single inheritance) ─────────────────────────── */ diff --git a/packages/compiler/src/frontend/lowering/surfaces.ts b/packages/compiler/src/frontend/lowering/surfaces.ts index e814bc0f5..923c857fe 100644 --- a/packages/compiler/src/frontend/lowering/surfaces.ts +++ b/packages/compiler/src/frontend/lowering/surfaces.ts @@ -3366,6 +3366,17 @@ export function isStdlibSymbol(lowerer: Lowerer, symbol: ts.Symbol | undefined): * tamper-guard prologue) registered in stdlibGlobalAliases; * - a const binding of the Map or Set constructor (`const M = Map`, * builtinClassAliasOf). */ +/** The symbol an identifier reads. A shorthand property's name (`{ Date }`) + * reads the binding of the same name, not the property it declares. */ +export function identifierReadSymbol(lowerer: Lowerer, expr: ts.Identifier): ts.Symbol | undefined { + if (expr.parent && ts.isShorthandPropertyAssignment(expr.parent) && expr.parent.name === expr) + return ( + lowerer.checker.getShorthandAssignmentValueSymbol(expr.parent) ?? + lowerer.checker.getSymbolAtLocation(expr) + ); + return lowerer.checker.getSymbolAtLocation(expr); +} + export function stdlibGlobalNameOf(lowerer: Lowerer, expr: ts.Expression): string | null { if (ts.isParenthesizedExpression(expr)) return stdlibGlobalNameOf(lowerer, expr.expression); // The global-object detection chain (`typeof global < "u" ? global : @@ -3377,7 +3388,7 @@ export function stdlibGlobalNameOf(lowerer: Lowerer, expr: ts.Expression): strin return stdlibGlobalNameOf(lowerer, taken ? expr.whenTrue : expr.whenFalse); } if (ts.isIdentifier(expr)) { - const symbol = lowerer.checker.getSymbolAtLocation(expr); + const symbol = identifierReadSymbol(lowerer, expr); // `globalThis` itself: a reserved intrinsic — tsc rejects user // global bindings, but local parameters can shadow it. Its special // symbol has no ordinary declarations for the provenance check. diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6913-builtin-constructor-shorthand.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6913-builtin-constructor-shorthand.ts.json new file mode 100644 index 000000000..bd7194022 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6913-builtin-constructor-shorthand.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6913-builtin-constructor-shorthand.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6914-dynamic-builtin-construct.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6914-dynamic-builtin-construct.ts.json new file mode 100644 index 000000000..9bc7d8342 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6914-dynamic-builtin-construct.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6914-dynamic-builtin-construct.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6915-dynamic-builtin-construct.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6915-dynamic-builtin-construct.mjs.json new file mode 100644 index 000000000..a817ad3d2 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6915-dynamic-builtin-construct.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6915-dynamic-builtin-construct.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6916-local-class-static-self-reference.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6916-local-class-static-self-reference.mjs.json new file mode 100644 index 000000000..aaefe846a --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6916-local-class-static-self-reference.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6916-local-class-static-self-reference.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6917-local-class-expression-self-reference.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6917-local-class-expression-self-reference.ts.json new file mode 100644 index 000000000..41d005bd0 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6917-local-class-expression-self-reference.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6917-local-class-expression-self-reference.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6918-local-class-expression-self-reference.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6918-local-class-expression-self-reference.mjs.json new file mode 100644 index 000000000..afed7a70d --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6918-local-class-expression-self-reference.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6918-local-class-expression-self-reference.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6919-local-class-inspect-name.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6919-local-class-inspect-name.mjs.json new file mode 100644 index 000000000..229c01bcd --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6919-local-class-inspect-name.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6919-local-class-inspect-name.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6921-local-subclass-override-dispatch.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6921-local-subclass-override-dispatch.ts.json new file mode 100644 index 000000000..1878554c1 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6921-local-subclass-override-dispatch.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6921-local-subclass-override-dispatch.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6922-local-subclass-override-dispatch.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6922-local-subclass-override-dispatch.mjs.json new file mode 100644 index 000000000..4da840cbc --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6922-local-subclass-override-dispatch.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6922-local-subclass-override-dispatch.mjs" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_dyn.h b/packages/runtime/src/scr_dyn.h index facf9f2ff..8a29e7c5d 100644 --- a/packages/runtime/src/scr_dyn.h +++ b/packages/runtime/src/scr_dyn.h @@ -659,6 +659,9 @@ ScrDyn *scr_dyn_native_set_new(const ScrDyn *seed); ScrDyn *scr_dyn_native_set(ScrMap *value); /* borrowed backing, +1 box */ bool scr_dyn_native_set_is(const ScrDyn *value); ScrDyn *scr_dyn_native_map(ScrMap *value); /* borrowed backing, +1 box */ +/* `new Map(entries)` over a checked-dynamic seed: +1, or NULL with the + * exception pending. */ +ScrDyn *scr_dyn_native_map_new(const ScrDyn *seed); bool scr_dyn_native_map_is(const ScrDyn *value); void scr_regex_reset_last_index(ScrRegex *re, double index); ScrRegex *scr_regex_new_checked(const ScrDyn *pattern, const ScrDyn *flags); @@ -879,6 +882,12 @@ bool scr_bytes_instanceof(const ScrDyn *value, const ScrDyn *callee); bool scr_dyn_data_view_is(const ScrDyn *value); ScrDyn *scr_dyn_construct(const ScrDyn *callee, const ScrDyn *args, const ScrStr *what); ScrDyn *scr_bytes_construct(const ScrDyn *callee, const ScrDyn *args, const ScrStr *what); +/* Installed by scr_dyn_invoke.c when it interns a builtin constructor value + * (scr_dyn_global_constructor): false when `callee` is not one of them; + * otherwise true with `new callee(...args)` in *result (+1, or NULL with + * the exception pending). */ +extern SCR_TL bool (*scr_dyn_global_construct_fn)(const ScrDyn *callee, const ScrDyn *args, + ScrDyn **result); ScrDyn *scr_dyn_bytes_key_get(const ScrDyn *value, const ScrStr *key); /* The Buffer-flavored twin (stream chunks): string coercion/toString * decode utf8 instead of joining elements. */ diff --git a/packages/runtime/src/scr_dyn_invoke.c b/packages/runtime/src/scr_dyn_invoke.c index 80e01b08c..e978ac7aa 100644 --- a/packages/runtime/src/scr_dyn_invoke.c +++ b/packages/runtime/src/scr_dyn_invoke.c @@ -1854,6 +1854,8 @@ static const char dyn_s29[] SCR_DYN_COMPAT_STR = "Date"; static const char dyn_s30[] SCR_DYN_COMPAT_STR = "Map"; static const char dyn_s31[] SCR_DYN_COMPAT_STR = "Set"; static const char dyn_s32[] SCR_DYN_COMPAT_STR = ""; +static const char dyn_s33[] SCR_DYN_COMPAT_STR = "constructing %s%s through a stored value is not supported yet"; +static const char dyn_s34[] SCR_DYN_COMPAT_STR = " from a primitive"; static const char *const dyn_global_ctor_names[] = { dyn_s26, dyn_s29, dyn_s25, dyn_s30, dyn_s31, dyn_s21, dyn_s22, dyn_s24, @@ -1887,12 +1889,42 @@ static ScrDyn *dyn_global_ctor_call(ScrClosure *closure, ScrDyn *const *args, si return NULL; } +/* `new C(...args)` through a stored Object, Date, Map, Set, WeakMap or + * WeakSet builds what the `new` expression builds. RegExp, Promise, and + * Object over a primitive (a wrapper object) are fenced. */ +static bool dyn_global_construct(const ScrDyn *callee, const ScrDyn *args, ScrDyn **result) { + if (callee->kind != SCR_DYN_FUNC || callee->v.fn.thunk != dyn_global_ctor_call) return false; + const char *name = callee->v.fn.clo->fn; + ScrDyn *const *items = args->v.arr.items; + size_t argc = args->v.arr.len; + const ScrDyn *first = argc ? items[0] : scr_dyn_undefined(); + bool primitive = first->kind == SCR_DYN_NUM || first->kind == SCR_DYN_STR || first->kind == SCR_DYN_BOOL || + first->kind == SCR_DYN_BIGINT || first->kind == SCR_DYN_SYMBOL; + if (strcmp(name, dyn_s26) == 0 && !primitive) *result = dyn_global_ctor_call(callee->v.fn.clo, items, argc); + else if (strcmp(name, dyn_s29) == 0) + *result = argc == 0 ? scr_dyn_native_date_new(args) + : argc == 1 ? scr_dyn_native_date_new_value(args) + : scr_dyn_native_date_new_fields(args); + else if (strcmp(name, dyn_s30) == 0) *result = scr_dyn_native_map_new(first); + else if (strcmp(name, dyn_s31) == 0) *result = scr_dyn_native_set_new(first); + else if (strcmp(name, dyn_s22) == 0) *result = scr_weak_map_new((ScrDyn *)first); + else if (strcmp(name, dyn_s24) == 0) *result = scr_weak_set_new((ScrDyn *)first); + else { + char message[128]; + int length = snprintf(message, sizeof message, dyn_s33, name, strcmp(name, dyn_s26) == 0 ? dyn_s34 : dyn_s32); + scr_throw_error_msg_code(SCR_ERR_ERROR, message, (size_t)length, dyn_s27); + *result = NULL; + } + return true; +} + static ScrDyn *dyn_global_ctor(const char *name) { for (size_t i = 0; i < DYN_GLOBAL_CTORS; i++) { if (strcmp(dyn_global_ctor_names[i], name) != 0) continue; if (!dyn_global_ctors_registered) { dyn_global_ctors_registered = true; scr_atexit(dyn_global_ctors_cleanup); + scr_dyn_global_construct_fn = dyn_global_construct; } if (!dyn_global_ctors[i]) { /* Each constructor's own length: Object 1, Date 7, RegExp 2, Promise 1, diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index f610ffdbb..6986330f5 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -4017,6 +4017,8 @@ bool scr_bytes_instanceof(const ScrDyn *value, const ScrDyn *callee) { scr_throw_error_msg_code(SCR_ERR_ERROR, message, sizeof message - 1, "SC1090"); return false; } +SCR_TL bool (*scr_dyn_global_construct_fn)(const ScrDyn *callee, const ScrDyn *args, ScrDyn **result) = NULL; + ScrDyn *scr_bytes_construct(const ScrDyn *callee, const ScrDyn *args, const ScrStr *what) { if (callee->kind == SCR_DYN_FUNC && scr_array_ctor && callee->v.fn.clo == scr_array_ctor->v.fn.clo) return scr_array_ctor_call(callee->v.fn.clo, args->v.arr.items, args->v.arr.len); @@ -4033,6 +4035,9 @@ ScrDyn *scr_bytes_construct(const ScrDyn *callee, const ScrDyn *args, const ScrS scr_bytes_release(bytes); return result; } + ScrDyn *constructed = NULL; + if (scr_dyn_global_construct_fn && scr_dyn_global_construct_fn(callee, args, &constructed)) + return constructed; ScrJsonBuf message; scr_jb_init(&message); scr_jb_write(&message, what->data, what->len); @@ -4180,6 +4185,13 @@ ScrDyn *scr_dyn_class_prototype(ScrDyn *value, ScrDyn *base) { if (!cls->prototype_data) { ScrDyn *data = base->kind == SCR_DYN_UNDEF ? scr_dyn_new_obj() : scr_dyn_obj_create(base); if (!data) return NULL; + /* util.inspect prints an instance with its class's name. */ + const char *name = value->v.fn.name; + if (name && name[0]) { + ScrStr *class_name = scr_str_new(name, strlen(name)); + scr_dyn_name_class_prototype(data, class_name); + scr_str_release(class_name); + } cls->prototype_data = scr_box_new_obj(scr_dyn_retain_v, scr_dyn_release_v, scr_dyn_trace_v); scr_box_set_ref(cls->prototype_data, data); } @@ -14939,6 +14951,27 @@ ScrDyn *scr_dyn_native_map(ScrMap *value) { return scr_dyn_new_handle(value, SCR_DYNH_MAP); } +ScrDyn *scr_dyn_native_map_new(const ScrDyn *seed) { + ScrDyn *entries = scr_dyn_map_seed_entries(seed); + if (!entries) return NULL; + ScrMap *map = scr_map_new(SCR_MAP_KEY_DYN, SCR_MAP_VAL_REF, &scr_map_dyn_map_ops); + for (size_t i = 0; i < entries->v.arr.len; i++) { + ScrDyn *entry = scr_dyn_map_seed_entry(entries->v.arr.items[i]); + if (!entry) break; + ScrDyn *key = scr_dyn_entry_read(entry, 0); + ScrDyn *value = scr_exc_pending() ? NULL : scr_dyn_entry_read(entry, 1); + scr_dyn_release(entry); + if (value) scr_map_dyn_set(map, key, value, false); + scr_dyn_release(key); + scr_dyn_release(value); + if (scr_exc_pending()) break; + } + scr_dyn_release(entries); + ScrDyn *result = scr_exc_pending() ? NULL : scr_dyn_native_map(map); + scr_map_release(map); + return result; +} + bool scr_dyn_native_map_is(const ScrDyn *value) { return value && value->kind == SCR_DYN_HANDLE && value->v.handle.tag == SCR_DYNH_MAP; } diff --git a/packages/runtime/src/scr_map.c b/packages/runtime/src/scr_map.c index 99caa31a8..9e53b1e98 100644 --- a/packages/runtime/src/scr_map.c +++ b/packages/runtime/src/scr_map.c @@ -1527,7 +1527,7 @@ ScrMap *scr_map_clone(const ScrMap *source, bool keys_only, const ScrMapOps *ops const ScrMapOps scr_map_dyn_set_ops = { scr_dyn_retain_v, scr_dyn_release_v, scr_dyn_trace_v, NULL, NULL, NULL, NULL }; -static const ScrMapOps scr_map_dyn_map_ops = { +const ScrMapOps scr_map_dyn_map_ops = { scr_dyn_retain_v, scr_dyn_release_v, scr_dyn_trace_v, scr_dyn_retain_v, scr_dyn_release_v, scr_dyn_trace_v, NULL }; diff --git a/packages/runtime/src/scr_map.h b/packages/runtime/src/scr_map.h index c61df9a90..cce327508 100644 --- a/packages/runtime/src/scr_map.h +++ b/packages/runtime/src/scr_map.h @@ -214,8 +214,10 @@ void *scr_map_get_span_ref(const ScrMap *m, const char *bytes, size_t length); * adapters); NULL keeps the source's descriptor. */ ScrMap *scr_map_clone(const ScrMap *source, bool keys_only, const ScrMapOps *ops); ScrMap *scr_map_clone_dyn(const ScrMap *source, bool keys_only); -/* The descriptor of a Set of checked values (SCR_MAP_KEY_DYN keys). */ +/* The descriptors of a Set and a Map of checked values (SCR_MAP_KEY_DYN + * keys; the Map's values are checked values too). */ extern const ScrMapOps scr_map_dyn_set_ops; +extern const ScrMapOps scr_map_dyn_map_ops; void scr_map_values_into(ScrMap *set, const ScrMap *source, bool checked); void scr_map_dyn_attach(ScrMap *map, const ScrMapDynOps *ops); diff --git a/tests/corpus/6913-builtin-constructor-shorthand.ts b/tests/corpus/6913-builtin-constructor-shorthand.ts new file mode 100644 index 000000000..7fb5066ed --- /dev/null +++ b/tests/corpus/6913-builtin-constructor-shorthand.ts @@ -0,0 +1,10 @@ +// Builtin constructors in shorthand properties are the constructors themselves. +const direct: Record = { Date: Date, Map: Map, Set: Set }; +const shorthand: Record = { Date, Map, Set, Object, Promise, WeakMap }; +console.log(shorthand.Date === Date, shorthand.Map === direct.Map, shorthand.Set === direct.Set); +console.log(shorthand.Object === Object, shorthand.Promise === Promise, shorthand.WeakMap === WeakMap); +console.log(typeof shorthand.Date, typeof shorthand.Map); +function pick(table: Record, name: string): unknown { + return table[name]; +} +console.log(pick({ Map, Set }, "Set") === Set, pick({ Date }, "Date") === pick(direct, "Date")); diff --git a/tests/corpus/6914-dynamic-builtin-construct.ts b/tests/corpus/6914-dynamic-builtin-construct.ts new file mode 100644 index 000000000..c026a6691 --- /dev/null +++ b/tests/corpus/6914-dynamic-builtin-construct.ts @@ -0,0 +1,54 @@ +// `new` through values that hold builtin constructors. +const ctors: Record = { Date, Map, Set, Object, WeakMap, WeakSet }; +function construct(name: string, ...args: unknown[]): unknown { + const C = ctors[name] as any; + return new C(...args); +} +function make(C: any, ...args: unknown[]): unknown { + return new C(...args); +} +const tag = (value: unknown) => Object.prototype.toString.call(value); + +const d = construct("Date", 0) as Date; +console.log(tag(d), d.getTime(), d.toISOString()); +const d2 = make(Date, 2020, 1, 3, 4, 5, 6, 7) as Date; +console.log(d2.getFullYear(), d2.getMonth(), d2.getDate(), d2.getHours(), d2.getMilliseconds()); +const d3 = make(Date, "2001-02-03T04:05:06Z") as Date; +const d4 = make(Date, d3) as Date; +console.log(d3.getTime(), d4.getTime()); +console.log(typeof make(Date), Number.isNaN((make(Date, "nope") as Date).getTime())); + +const m = construct("Map", [["a", 1], ["b", 2]]) as Map; +console.log(tag(m), m.size, m.get("b"), [...m.keys()].join()); +const m2 = make(Map, m) as Map; +m2.set("c", 3); +console.log(m.size, m2.size, (make(Map) as Map).size, (make(Map, null) as Map).size); + +const s = construct("Set", [1, 2, 2, 3]) as Set; +console.log(tag(s), s.size, s.has(3), (make(Set, "abca") as Set).size); + +const o = construct("Object") as Record; +o.x = 1; +const same = { y: 2 }; +console.log(tag(o), JSON.stringify(o), make(Object, same) === same, tag(make(Object, null))); + +const key = {}; +const w = construct("WeakMap", [[key, "v"]]) as WeakMap; +console.log(tag(w), w.get(key), w.has({})); +const ws = make(WeakSet, [key]) as WeakSet; +console.log(tag(ws), ws.has(key)); + +try { + make(Map, [1]); +} catch (e) { + console.log((e as Error).name, (e as Error).message); +} +try { + make(WeakMap, [[1, 2]]); +} catch (e) { + console.log((e as Error).name, (e as Error).message); +} + +const DateSlot: new () => unknown = Date; +const MapSlot: new (entries: [string, number][]) => unknown = Map; +console.log(tag(new DateSlot()), (new MapSlot([["k", 1]]) as Map).get("k")); diff --git a/tests/corpus/6915-dynamic-builtin-construct.mjs b/tests/corpus/6915-dynamic-builtin-construct.mjs new file mode 100644 index 000000000..dd13aa325 --- /dev/null +++ b/tests/corpus/6915-dynamic-builtin-construct.mjs @@ -0,0 +1,16 @@ +// `new` through stored builtin constructors in JavaScript. +const table = [Date, Map, Set, Object, WeakMap, WeakSet]; +for (const C of table) { + const value = new C(); + console.log(C.name, typeof value, Object.prototype.toString.call(value), value.constructor === C); +} +const Ctor = table[0]; +console.log(new Ctor(86400000).toISOString(), new Ctor(2024, 0, 31).getDate()); +const MapCtor = table[1]; +const map = new MapCtor(Object.entries({ a: 1, b: 2 })); +map.set("c", 3); +console.log(map.size, map.get("a"), [...map.entries()].length); +const SetCtor = table[2]; +console.log(new SetCtor([...map.keys()]).size, new SetCtor([NaN, NaN, 0, -0]).size); +const named = { Date, Map, Set }; +console.log(named.Date === Date, named.Map === table[1], named.Set === table[2]); diff --git a/tests/corpus/6916-local-class-static-self-reference.mjs b/tests/corpus/6916-local-class-static-self-reference.mjs new file mode 100644 index 000000000..bc19cd0e9 --- /dev/null +++ b/tests/corpus/6916-local-class-static-self-reference.mjs @@ -0,0 +1,52 @@ +// Static fields, static blocks and static methods of function-local classes +// that read the class by its name while the class is being defined. +function registry(prefix) { + class Registry { + static label = Registry.name + ":" + prefix; + static instances = []; + static { + Registry.ready = Registry.instances.length === 0; + } + constructor(id) { + this.id = id; + Registry.instances.push(this); + } + static create(id) { + return new Registry(id); + } + } + return Registry; +} +const R = registry("main"); +R.create(1); +R.create(2); +console.log(R.label, R.ready, R.instances.map((r) => r.id)); +const R2 = registry("other"); +console.log(R2.label, R2.instances.length, R2 === R, R.instances.length); + +function counter(step) { + class Counter { + static zero = new Counter(0); + static one = Counter.zero.next(); + constructor(value) { + this.value = value; + } + next() { + return new Counter(this.value + step); + } + } + return Counter; +} +const C = counter(5); +console.log(C.zero.value, C.one.value, C.one.next().value, C.one instanceof C); + +function described() { + class Shape { + static kinds = ["circle", "square"]; + static describe() { + return Shape.kinds.map((kind) => kind + "@" + Shape.name).join(","); + } + } + return Shape.describe(); +} +console.log(described()); diff --git a/tests/corpus/6917-local-class-expression-self-reference.ts b/tests/corpus/6917-local-class-expression-self-reference.ts new file mode 100644 index 000000000..f11746eda --- /dev/null +++ b/tests/corpus/6917-local-class-expression-self-reference.ts @@ -0,0 +1,41 @@ +// Function-local class expressions that refer to themselves by their own name. +function counter(start: number) { + const C = class Counter { + value: number; + constructor(value: number) { this.value = value; } + next(): Counter { return new Counter(this.value + 1); } + static first(): Counter { return new Counter(start); } + static isSelf(value: unknown): boolean { return value === Counter; } + }; + return C; +} +const Counter1 = counter(10); +console.log(Counter1.first().next().next().value); +console.log(Counter1.isSelf(Counter1), Counter1.isSelf(counter(1)), Counter1.name); + +function box(seed: T) { + return class Box { + value: T; + constructor(value: T) { this.value = value; } + again(): Box { return new Box(this.value); } + static of(): Box { return new Box(seed); } + }; +} +const StringBox = box("seed"); +console.log(new StringBox("x").again().value, StringBox.of().value); +const NumberBox = box(7); +console.log(NumberBox.of().again().value + 1); + +function nested(n: number) { + const Outer = class Tree { + children: Tree[] = []; + static build(depth: number): Tree { + const tree = new Tree(); + if (depth > 0) tree.children.push(Tree.build(depth - 1), Tree.build(depth - 1)); + return tree; + } + size(): number { return 1 + this.children.reduce((total, child) => total + child.size(), 0); } + }; + return Outer.build(n).size(); +} +console.log(nested(3)); diff --git a/tests/corpus/6918-local-class-expression-self-reference.mjs b/tests/corpus/6918-local-class-expression-self-reference.mjs new file mode 100644 index 000000000..cfc8de806 --- /dev/null +++ b/tests/corpus/6918-local-class-expression-self-reference.mjs @@ -0,0 +1,50 @@ +// Function-local class expressions in JavaScript whose static members and +// methods read the class by its own name. +function point(scale) { + const P = class Point { + static origin = new Point(0, 0); + static unit = Point.scaled(1); + constructor(x, y) { + this.x = x; + this.y = y; + } + static scaled(n) { + return new Point(n * scale, n * scale); + } + plus(other) { + return new Point(this.x + other.x, this.y + other.y); + } + }; + return P; +} +const P = point(3); +const sum = P.origin.plus(P.unit).plus(P.unit); +console.log(sum.x, sum.y, P.unit instanceof P, P.name); + +function selfName() { + const K = class Named { + static who() { + return Named === this; + } + }; + return [K.who(), K.name, typeof K]; +} +console.log(selfName()); + +function factory(tag) { + return class Tagged { + static count = 0; + constructor() { + Tagged.count++; + } + static make() { + return new Tagged(); + } + get tag() { + return tag + Tagged.count; + } + }; +} +const T = factory("t"); +T.make(); +console.log(T.make().tag, T.count, factory("u").count); diff --git a/tests/corpus/6919-local-class-inspect-name.mjs b/tests/corpus/6919-local-class-inspect-name.mjs new file mode 100644 index 000000000..022895591 --- /dev/null +++ b/tests/corpus/6919-local-class-inspect-name.mjs @@ -0,0 +1,51 @@ +// console.log prints instances of function-local classes with prototype +// methods under their class names. +function point() { + class Point { + constructor(x, y) { + this.x = x; + this.y = y; + } + plus(other) { + return new Point(this.x + other.x, this.y + other.y); + } + } + return Point; +} +const Point = point(); +const a = new Point(1, 2); +console.log(a, a.plus(a)); +console.log([new Point(0, 0)], { p: new Point(5, 6) }); + +function animals() { + class Animal { + constructor(name) { + this.name = name; + } + speak() { + return this.name + " makes a sound"; + } + } + class Dog extends Animal { + speak() { + return this.name + " barks"; + } + } + return [new Animal("cat"), new Dog("rex")]; +} +const [cat, dog] = animals(); +console.log(cat, dog); + +function anonymous() { + const Named = class { + value() { + return 1; + } + }; + return [new Named(), new (class { + value() { + return 2; + } + })()]; +} +console.log(anonymous()); diff --git a/tests/corpus/6921-local-subclass-override-dispatch.ts b/tests/corpus/6921-local-subclass-override-dispatch.ts new file mode 100644 index 000000000..0476e56c7 --- /dev/null +++ b/tests/corpus/6921-local-subclass-override-dispatch.ts @@ -0,0 +1,51 @@ +// Overrides declared by function-local subclasses dispatch on the receiver +// when the call site lowers before the function that declares them. +class Shape { + area(): number { return 0; } + get label(): string { return "shape"; } + describe(): string { return this.label + ":" + this.area(); } +} + +function square(side: number): Shape { + class Square extends Shape { + area(): number { return side * side; } + get label(): string { return "square"; } + } + return new Square(); +} + +function scaled(factor: number): Shape { + class Scaled extends Shape { + area(): number { return factor; } + } + class Doubled extends Scaled { + area(): number { return super.area() * 2; } + } + return new Doubled(); +} + +const shapes: Shape[] = [new Shape(), square(3), scaled(5)]; +for (const shape of shapes) console.log(shape.area(), shape.label, shape.describe()); + +function pair() { + class Base { + name(): string { return "base"; } + } + class Derived extends Base { + name(): string { return "derived"; } + } + return [new Base(), new Derived()]; +} +const [first, second] = pair(); +console.log(first.name(), second.name()); + +function counter() { + const Tally = class { + count(): number { return 1; } + }; + const Double = class extends Tally { + count(): number { return 2; } + }; + return new Double(); +} +console.log(counter().count()); diff --git a/tests/corpus/6922-local-subclass-override-dispatch.mjs b/tests/corpus/6922-local-subclass-override-dispatch.mjs new file mode 100644 index 000000000..8439c563d --- /dev/null +++ b/tests/corpus/6922-local-subclass-override-dispatch.mjs @@ -0,0 +1,51 @@ +// Overrides declared by function-local subclasses in JavaScript dispatch on +// the receiver when the call site lowers before the declaring function. +class Animal { + constructor(name) { + this.name = name; + } + speak() { + return this.name + " makes a sound"; + } +} + +function dog(name) { + class Dog extends Animal { + speak() { + return this.name + " barks"; + } + } + return new Dog(name); +} + +function animals() { + class Bird extends Animal { + speak() { + return this.name + " sings"; + } + } + return [new Animal("cat"), new Bird("robin"), dog("rex")]; +} + +for (const animal of animals()) console.log(animal.speak()); +const speak = (animal) => animal.speak(); +console.log(speak(dog("fido"))); + +function pets() { + class Pet { + constructor(name) { + this.name = name; + } + greet() { + return this.name + " waves"; + } + } + class Cat extends Pet { + greet() { + return this.name + " purrs"; + } + } + return [new Pet("ann"), new Cat("tom")]; +} +const [pet, cat] = pets(); +console.log(pet.greet(), cat.greet()); diff --git a/tests/diagnostics/local-class-boundaries.ts b/tests/diagnostics/local-class-boundaries.ts index 2c43e9900..ac1afb2c2 100644 --- a/tests/diagnostics/local-class-boundaries.ts +++ b/tests/diagnostics/local-class-boundaries.ts @@ -1,10 +1,8 @@ function privateBrand() { return class { #value = 1; has(other: object): boolean { return #value in other; } }; } -function selfName() { return class Local { copy() { return new Local(); } }; } class Base {} function derived() { return class extends Base {}; } function generic() { return class { read(value: T): T { return value; } }; } privateBrand(); -selfName(); derived(); generic(); // End of unsupported local-class forms. diff --git a/tests/harness/__snapshots__/local-class-boundaries.ts.txt b/tests/harness/__snapshots__/local-class-boundaries.ts.txt index 6706a1278..697b1b8ed 100644 --- a/tests/harness/__snapshots__/local-class-boundaries.ts.txt +++ b/tests/harness/__snapshots__/local-class-boundaries.ts.txt @@ -2,18 +2,11 @@ local-class-boundaries.ts:1:34 - error SC1090: function-local classes with priva 1 | function privateBrand() { return class { #value = 1; has(other: object): boolean { return #value in other; } }; } | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - 2 | function selfName() { return class Local { copy() { return new Local(); } }; } + 2 | class Base {} -local-class-boundaries.ts:2:64 - error SC1090: named self references inside function-local classes are not supported yet +local-class-boundaries.ts:4:29 - error SC1090: function-local classes with statics, decorators, or generic members are not supported yet - 1 | function privateBrand() { return class { #value = 1; has(other: object): boolean { return #value in other; } }; } - 2 | function selfName() { return class Local { copy() { return new Local(); } }; } - | ^~~~~ - 3 | class Base {} - -local-class-boundaries.ts:5:29 - error SC1090: function-local classes with statics, decorators, or generic members are not supported yet - - 4 | function derived() { return class extends Base {}; } - 5 | function generic() { return class { read(value: T): T { return value; } }; } + 3 | function derived() { return class extends Base {}; } + 4 | function generic() { return class { read(value: T): T { return value; } }; } | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - 6 | privateBrand(); \ No newline at end of file + 5 | privateBrand(); \ No newline at end of file diff --git a/tests/harness/corpus-costs/6913-builtin-constructor-shorthand.ts.json b/tests/harness/corpus-costs/6913-builtin-constructor-shorthand.ts.json new file mode 100644 index 000000000..3c9170116 --- /dev/null +++ b/tests/harness/corpus-costs/6913-builtin-constructor-shorthand.ts.json @@ -0,0 +1 @@ +{ "plain": 3202, "san": 4630 } diff --git a/tests/harness/corpus-costs/6914-dynamic-builtin-construct.ts.json b/tests/harness/corpus-costs/6914-dynamic-builtin-construct.ts.json new file mode 100644 index 000000000..a3f53b09c --- /dev/null +++ b/tests/harness/corpus-costs/6914-dynamic-builtin-construct.ts.json @@ -0,0 +1 @@ +{ "plain": 3064, "san": 5586 } diff --git a/tests/harness/corpus-costs/6915-dynamic-builtin-construct.mjs.json b/tests/harness/corpus-costs/6915-dynamic-builtin-construct.mjs.json new file mode 100644 index 000000000..0c65d046c --- /dev/null +++ b/tests/harness/corpus-costs/6915-dynamic-builtin-construct.mjs.json @@ -0,0 +1 @@ +{ "plain": 2821, "san": 4440 } diff --git a/tests/harness/corpus-costs/6916-local-class-static-self-reference.mjs.json b/tests/harness/corpus-costs/6916-local-class-static-self-reference.mjs.json new file mode 100644 index 000000000..0d9aa6b13 --- /dev/null +++ b/tests/harness/corpus-costs/6916-local-class-static-self-reference.mjs.json @@ -0,0 +1 @@ +{ "plain": 3135, "san": 4365 } diff --git a/tests/harness/corpus-costs/6917-local-class-expression-self-reference.ts.json b/tests/harness/corpus-costs/6917-local-class-expression-self-reference.ts.json new file mode 100644 index 000000000..2769f5b72 --- /dev/null +++ b/tests/harness/corpus-costs/6917-local-class-expression-self-reference.ts.json @@ -0,0 +1 @@ +{ "plain": 3074, "san": 4624 } diff --git a/tests/harness/corpus-costs/6918-local-class-expression-self-reference.mjs.json b/tests/harness/corpus-costs/6918-local-class-expression-self-reference.mjs.json new file mode 100644 index 000000000..04f903d8a --- /dev/null +++ b/tests/harness/corpus-costs/6918-local-class-expression-self-reference.mjs.json @@ -0,0 +1 @@ +{ "plain": 3050, "san": 4393 } diff --git a/tests/harness/corpus-costs/6919-local-class-inspect-name.mjs.json b/tests/harness/corpus-costs/6919-local-class-inspect-name.mjs.json new file mode 100644 index 000000000..78aea9163 --- /dev/null +++ b/tests/harness/corpus-costs/6919-local-class-inspect-name.mjs.json @@ -0,0 +1 @@ +{ "plain": 3058, "san": 4846 } diff --git a/tests/harness/corpus-costs/6921-local-subclass-override-dispatch.ts.json b/tests/harness/corpus-costs/6921-local-subclass-override-dispatch.ts.json new file mode 100644 index 000000000..0dd7e966b --- /dev/null +++ b/tests/harness/corpus-costs/6921-local-subclass-override-dispatch.ts.json @@ -0,0 +1 @@ +{ "plain": 2459, "san": 4630 } diff --git a/tests/harness/corpus-costs/6922-local-subclass-override-dispatch.mjs.json b/tests/harness/corpus-costs/6922-local-subclass-override-dispatch.mjs.json new file mode 100644 index 000000000..f2ba6c139 --- /dev/null +++ b/tests/harness/corpus-costs/6922-local-subclass-override-dispatch.mjs.json @@ -0,0 +1 @@ +{ "plain": 3216, "san": 5042 } From e07f1be5c297a17769d7a2b69dcea17b6f78c2f5 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 07:07:34 -0500 Subject: [PATCH 012/197] Keep aliases of evolving empty arrays on the array they alias An unannotated `const xs = []` inside a function takes the element type its reads agree on, and is checked-dynamic when they disagree. An alias taken between two writes of different element types (`const alias = xs` after `xs.push(1)`, before `xs.push("two")`) took the element type the checker gives it at that point, so it held a copy and missed the later writes: Node prints `1,two 2` for `xs.join(","), alias.length`, the compiled program printed `1,two 1`. An unannotated alias of such a binding, and an alias of that alias, now takes the binding's checked-dynamic type, so every name holds the one array. A write that stores a value the agreed element type does not hold (`xs.push("t")` after reads typed `boolean[]`, or `xs[1] = "one"`) now makes the binding checked-dynamic too. Those programs were refused with SC1090 "'string' values where 'boolean' is expected". Fixes #1641. --- .../src/frontend/lowering/lower-stmts.ts | 59 ++++++++++++++-- .../7006-evolving-array-aliases.ts.json | 6 ++ tests/corpus/7006-evolving-array-aliases.ts | 70 +++++++++++++++++++ .../7006-evolving-array-aliases.ts.json | 1 + 4 files changed, 131 insertions(+), 5 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7006-evolving-array-aliases.ts.json create mode 100644 tests/corpus/7006-evolving-array-aliases.ts create mode 100644 tests/harness/corpus-costs/7006-evolving-array-aliases.ts.json diff --git a/packages/compiler/src/frontend/lowering/lower-stmts.ts b/packages/compiler/src/frontend/lowering/lower-stmts.ts index 213abfec6..29086d326 100644 --- a/packages/compiler/src/frontend/lowering/lower-stmts.ts +++ b/packages/compiler/src/frontend/lowering/lower-stmts.ts @@ -4799,9 +4799,10 @@ export function lowerVarDeclList( * `never[]` or `any[]` and each later read by the elements written before * it, and reads inside closures as the unsettled array. When every read * with a settled element type maps to the same array type, the binding has - * that type. When reads disagree or the binding is reassigned, it is - * checked-dynamic, as the same binding is at module scope. Null for other - * bindings. */ + * that type. When reads disagree, a write (`xs.push(v)`, `xs.unshift(v)`, + * `xs[i] = v`) stores a value that type's element does not hold, or the + * binding is reassigned, it is checked-dynamic, as the same binding is at + * module scope. Null for other bindings. */ function evolvingArrayType(lowerer: Lowerer, decl: ts.VariableDeclaration): IrType | null { if (!ts.isIdentifier(decl.name) || decl.initializer === undefined) return null; let initializer: ts.Expression = decl.initializer; @@ -4823,6 +4824,9 @@ function evolvingArrayType(lowerer: Lowerer, decl: ts.VariableDeclaration): IrTy }; const scope = owner(decl); let result: IrType | null = null; + // The values the binding's writes store, checked against the element + // type once the reads settle it. + const written: ts.Expression[] = []; // The file's identifiers are indexed by name once (identifiersNamed), so // each binding resolves only the uses spelled like it. for (const node of identifiersNamed(lowerer, ts.getSourceFileOfNode(decl), decl.name.text)) { @@ -4842,6 +4846,22 @@ function evolvingArrayType(lowerer: Lowerer, decl: ts.VariableDeclaration): IrTy parent.operatorToken.kind >= ts.SyntaxKind.FirstAssignment && parent.operatorToken.kind <= ts.SyntaxKind.LastAssignment; if (reassigned) return DYN; + if ( + ts.isPropertyAccessExpression(parent) && + parent.expression === target && + (parent.name.text === "push" || parent.name.text === "unshift") && + ts.isCallExpression(parent.parent) && + parent.parent.expression === parent + ) + written.push(...parent.parent.arguments.nodes); + else if ( + ts.isElementAccessExpression(parent) && + parent.expression === target && + ts.isBinaryExpression(parent.parent) && + parent.parent.left === parent && + parent.parent.operatorToken.kind === ts.SyntaxKind.EqualsToken + ) + written.push(parent.parent.right); const t = lowerer.typeOf(node); if (unsettled(t)) continue; const mapped = lowerer.mapTypeOf(t); @@ -4849,7 +4869,36 @@ function evolvingArrayType(lowerer: Lowerer, decl: ts.VariableDeclaration): IrTy return DYN; result = mapped; } - return result ?? DYN; + if (result === null || result.kind !== "array") return DYN; + for (const value of written) { + if (ts.isSpreadElement(value)) return DYN; + const t = lowerer.mapTypeOf(lowerer.checker.getBaseTypeOfLiteralType(lowerer.typeOf(value))); + if (!t || (!typeEquals(t, result.elem) && lowerer.widthLiftPlan(t, result.elem) === null)) + return DYN; + } + return result; +} + +/** An unannotated alias of an evolving array binding whose reads disagree + * (`const alias = xs` between `xs.push(1)` and `xs.push("two")`): the + * checker types the alias by the elements written before it, but both names + * hold one array, so the alias takes the binding's checked-dynamic type and + * sees the later writes. Aliases of such an alias follow it. Null for other + * bindings. */ +function evolvingArrayAliasType(lowerer: Lowerer, decl: ts.VariableDeclaration): IrType | null { + let current = decl; + for (let depth = 0; depth < 8; depth++) { + let initializer = current.initializer; + while (initializer !== undefined && ts.isParenthesizedExpression(initializer)) + initializer = initializer.expression; + if (initializer === undefined || !ts.isIdentifier(initializer)) return null; + const source = lowerer.checker.getSymbolAtLocation(initializer)?.valueDeclaration; + if (source === undefined || !ts.isVariableDeclaration(source) || source.type) return null; + const evolved = evolvingArrayType(lowerer, source); + if (evolved !== null) return evolved.kind === "dyn" ? evolved : null; + current = source; + } + return null; } /** A single static type for an initializer-less JavaScript `let` whose @@ -5565,7 +5614,7 @@ export function lowerVarDecl( !decl.type && !isJsSourceFile(ts.getSourceFileOfNode(decl)) ) { - const evolved = evolvingArrayType(lowerer, decl); + const evolved = evolvingArrayType(lowerer, decl) ?? evolvingArrayAliasType(lowerer, decl); if (evolved !== null) { type = evolved; init = lowerer.lowerExprExpecting(decl.initializer, evolved); diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7006-evolving-array-aliases.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7006-evolving-array-aliases.ts.json new file mode 100644 index 000000000..cb3d36498 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7006-evolving-array-aliases.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7006-evolving-array-aliases.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7006-evolving-array-aliases.ts b/tests/corpus/7006-evolving-array-aliases.ts new file mode 100644 index 000000000..4810c440e --- /dev/null +++ b/tests/corpus/7006-evolving-array-aliases.ts @@ -0,0 +1,70 @@ +// An unannotated `const xs = []` inside a function whose writes store values +// of more than one element type: an alias taken between the writes +// (`const alias = xs`) and an alias of that alias name the same array, so +// they see every later write, as in Node. A write of another element type +// after reads that agree on one type also keeps the array untyped. +function chain(): string { + const xs = []; + xs.push(1); + const a = xs; + const b = a; + xs.push("two"); + b.push(3); + return `${xs.join(",")} ${a.length} ${b.length} ${a === xs} ${b === xs}`; +} +function viaAlias(): string { + const xs = []; + xs.push("a"); + const alias = xs; + alias.push("b"); + xs.push(2); + return `${xs.join(",")} ${alias.join(",")}`; +} +function letAlias(): string { + const xs = []; + xs.push(true); + let alias = xs; + xs.push("t"); + const seen = alias.length; + alias = []; + return `${seen} ${alias.length} ${xs.length}`; +} +function nested(flag: boolean): string { + const xs = []; + xs.push(1); + if (flag) { + const inner = xs; + xs.push("n"); + return `${inner.length} ${inner.join(":")}`; + } + return `${xs.length}`; +} +function elementWrite(): string { + const xs = []; + xs.push(1); + const alias = xs; + xs[1] = "one"; + return `${alias.length} ${alias.join("+")}`; +} +function agreeing(): string { + const xs = []; + xs.push(1); + const alias = xs; + xs.push(2); + return `${alias.length} ${alias === xs} ${alias.reduce((sum, v) => sum + v, 0)}`; +} +function total(values: number[]): number { + return values.length; +} +function passed(): string { + const xs = []; + xs.push(1); + const alias = xs; + const n = total(alias); + xs.push("z"); + return `${n} ${alias.length}`; +} +console.log(chain()); +console.log(viaAlias(), "|", letAlias()); +console.log(nested(true), "|", nested(false), "|", elementWrite()); +console.log(agreeing(), "|", passed()); diff --git a/tests/harness/corpus-costs/7006-evolving-array-aliases.ts.json b/tests/harness/corpus-costs/7006-evolving-array-aliases.ts.json new file mode 100644 index 000000000..f34efd332 --- /dev/null +++ b/tests/harness/corpus-costs/7006-evolving-array-aliases.ts.json @@ -0,0 +1 @@ +{ "plain": 2292, "san": 3445 } From 51b9bdca11d21e7ee2b670df375c636c2020b286 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Wed, 21 Oct 2026 21:39:25 -0700 Subject: [PATCH 013/197] Compile chalk 5 from a built-in shim with a callable builder class A class whose merged interface carries a call signature stores its call behavior in a '%call' field; calls and function-slot coercions read it. Co-Authored-By: Claude Opus 5.5 --- packages/compiler/shims/chalk/v5.ts | 573 ++++++++++++++++++ packages/compiler/shims/hot-shots/v8.ts | 2 + .../src/frontend/lowering/lower-calls.ts | 25 +- .../compiler/src/frontend/lowering/lowerer.ts | 17 +- .../compiler/src/frontend/package-shims.ts | 49 +- 5 files changed, 657 insertions(+), 9 deletions(-) create mode 100644 packages/compiler/shims/chalk/v5.ts create mode 100644 packages/compiler/shims/hot-shots/v8.ts diff --git a/packages/compiler/shims/chalk/v5.ts b/packages/compiler/shims/chalk/v5.ts new file mode 100644 index 000000000..6599e35f9 --- /dev/null +++ b/packages/compiler/shims/chalk/v5.ts @@ -0,0 +1,573 @@ +// scriptc's built-in implementation of the `chalk` package, 5.x API. +// +// Programs whose installed chalk is 5.x compile this module in its place +// (packages/compiler/src/frontend/package-shims.ts). chalk builds its +// styles as functions whose prototypes it swaps at run time; here a style +// chain is an instance of one class whose style members are cached +// getters, and the instance is callable through its `%call` field (the +// merged interface below gives it chalk's call signature; the lowerer calls +// the field). The output, color-level detection (supports-color 10: +// FORCE_COLOR, the --color/--no-color flags, TTY, TERM, CI variables) and +// the shared, settable `level` follow chalk 5.6 byte for byte. +import * as os from "node:os"; +import * as tty from "node:tty"; + +export type ColorSupportLevel = 0 | 1 | 2 | 3; + +export interface ColorSupport { + level: ColorSupportLevel; + hasBasic: boolean; + has256: boolean; + has16m: boolean; +} + +export type ColorInfo = ColorSupport | false; + +export interface Options { + readonly level?: ColorSupportLevel; +} + +export type ModifierName = + | "reset" + | "bold" + | "dim" + | "italic" + | "underline" + | "overline" + | "inverse" + | "hidden" + | "strikethrough"; + +export type ForegroundColorName = + | "black" + | "red" + | "green" + | "yellow" + | "blue" + | "cyan" + | "magenta" + | "white" + | "gray" + | "grey" + | "blackBright" + | "redBright" + | "greenBright" + | "yellowBright" + | "blueBright" + | "cyanBright" + | "magentaBright" + | "whiteBright"; + +export type BackgroundColorName = + | "bgBlack" + | "bgRed" + | "bgGreen" + | "bgYellow" + | "bgBlue" + | "bgCyan" + | "bgMagenta" + | "bgWhite" + | "bgGray" + | "bgGrey" + | "bgBlackBright" + | "bgRedBright" + | "bgGreenBright" + | "bgYellowBright" + | "bgBlueBright" + | "bgCyanBright" + | "bgMagentaBright" + | "bgWhiteBright"; + +export type ColorName = ForegroundColorName | BackgroundColorName; + +export type Modifiers = ModifierName; +export type ForegroundColor = ForegroundColorName; +export type BackgroundColor = BackgroundColorName; +export type Color = ColorName; + +// ── ansi-styles ────────────────────────────────────────────────────────── + +export const modifierNames: ModifierName[] = [ + "reset", + "bold", + "dim", + "italic", + "underline", + "overline", + "inverse", + "hidden", + "strikethrough", +]; +export const foregroundColorNames: ForegroundColorName[] = [ + "black", + "red", + "green", + "yellow", + "blue", + "magenta", + "cyan", + "white", + "blackBright", + "gray", + "grey", + "redBright", + "greenBright", + "yellowBright", + "blueBright", + "magentaBright", + "cyanBright", + "whiteBright", +]; +export const backgroundColorNames: BackgroundColorName[] = [ + "bgBlack", + "bgRed", + "bgGreen", + "bgYellow", + "bgBlue", + "bgMagenta", + "bgCyan", + "bgWhite", + "bgBlackBright", + "bgGray", + "bgGrey", + "bgRedBright", + "bgGreenBright", + "bgYellowBright", + "bgBlueBright", + "bgMagentaBright", + "bgCyanBright", + "bgWhiteBright", +]; +export const colorNames: ColorName[] = [ + ...foregroundColorNames, + ...backgroundColorNames, +]; +export const modifiers: Modifiers[] = modifierNames; +export const foregroundColors: ForegroundColor[] = foregroundColorNames; +export const backgroundColors: BackgroundColor[] = backgroundColorNames; +export const colors: Color[] = colorNames; + +const ESC = "\u001B["; + +/** Open and close codes, indexed by style id (the order of the getters). */ +const OPEN_CODES: number[] = [ + 0, 1, 2, 3, 4, 53, 7, 8, 9, + 30, 31, 32, 33, 34, 35, 36, 37, 90, 90, 90, 91, 92, 93, 94, 95, 96, 97, + 40, 41, 42, 43, 44, 45, 46, 47, 100, 100, 100, 101, 102, 103, 104, 105, 106, 107, +]; +const CLOSE_CODES: number[] = [ + 0, 22, 22, 23, 24, 55, 27, 28, 29, + 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, +]; +const VISIBLE = 45; + +function rgbToAnsi256(red: number, green: number, blue: number): number { + if (red === green && green === blue) { + if (red < 8) return 16; + if (red > 248) return 231; + return Math.round(((red - 8) / 247) * 24) + 232; + } + return ( + 16 + + 36 * Math.round((red / 255) * 5) + + 6 * Math.round((green / 255) * 5) + + Math.round((blue / 255) * 5) + ); +} + +function hexToRgb(hex: string): number[] { + const matches = /[a-f\d]{6}|[a-f\d]{3}/i.exec(hex); + if (matches === null) return [0, 0, 0]; + let colorString = matches[0]; + if (colorString.length === 3) { + let doubled = ""; + for (const character of colorString) doubled += character + character; + colorString = doubled; + } + const integer = Number.parseInt(colorString, 16); + return [(integer >> 16) & 0xff, (integer >> 8) & 0xff, integer & 0xff]; +} + +function ansi256ToAnsi(code: number): number { + if (code < 8) return 30 + code; + if (code < 16) return 90 + (code - 8); + let red: number; + let green: number; + let blue: number; + if (code >= 232) { + red = ((code - 232) * 10 + 8) / 255; + green = red; + blue = red; + } else { + code -= 16; + const remainder = code % 36; + red = Math.floor(code / 36) / 5; + green = Math.floor(remainder / 6) / 5; + blue = (remainder % 6) / 5; + } + const value = Math.max(red, green, blue) * 2; + if (value === 0) return 30; + let result = 30 + ((Math.round(blue) << 2) | (Math.round(green) << 1) | Math.round(red)); + if (value === 2) result += 60; + return result; +} + +/** The open code of a color model at a level: offset 0 is the foreground, + * 10 the background (ansi-styles' wrapAnsi16/256/16m). */ +function rgbOpen(level: number, offset: number, red: number, green: number, blue: number): string { + if (level === 3) return `${ESC}${38 + offset};2;${red};${green};${blue}m`; + if (level === 2) return `${ESC}${38 + offset};5;${rgbToAnsi256(red, green, blue)}m`; + return `${ESC}${ansi256ToAnsi(rgbToAnsi256(red, green, blue)) + offset}m`; +} + +/** chalk's ansi256 model emits the 256-color code at every level (only + * the rgb and hex models downsample). */ +function ansi256Open(offset: number, code: number): string { + return `${ESC}${38 + offset};5;${code}m`; +} + +// ── supports-color ─────────────────────────────────────────────────────── + +function hasFlag(flag: string): boolean { + const argv = process.argv; + const prefix = flag.startsWith("-") ? "" : flag.length === 1 ? "-" : "--"; + const position = argv.indexOf(prefix + flag); + const terminatorPosition = argv.indexOf("--"); + return position !== -1 && (terminatorPosition === -1 || position < terminatorPosition); +} + +let flagForceColor: number | undefined = undefined; +if (hasFlag("no-color") || hasFlag("no-colors") || hasFlag("color=false") || hasFlag("color=never")) { + flagForceColor = 0; +} else if (hasFlag("color") || hasFlag("colors") || hasFlag("color=true") || hasFlag("color=always")) { + flagForceColor = 1; +} + +function envForceColor(): number | undefined { + const force = process.env["FORCE_COLOR"]; + if (force !== undefined) { + if (force === "true") return 1; + if (force === "false") return 0; + return force.length === 0 ? 1 : Math.min(Number.parseInt(force, 10), 3); + } + return undefined; +} + +function translateLevel(level: number): ColorInfo { + if (level === 0) return false; + return { + level: level as ColorSupportLevel, + hasBasic: true, + has256: level >= 2, + has16m: level >= 3, + }; +} + +function inEnv(name: string): boolean { + return process.env[name] !== undefined; +} + +function detectLevel(streamIsTTY: boolean): number { + const noFlagForceColor = envForceColor(); + if (noFlagForceColor !== undefined) flagForceColor = noFlagForceColor; + const forceColor = flagForceColor; + if (forceColor === 0) return 0; + if (hasFlag("color=16m") || hasFlag("color=full") || hasFlag("color=truecolor")) return 3; + if (hasFlag("color=256")) return 2; + if (inEnv("TF_BUILD") && inEnv("AGENT_NAME")) return 1; + if (!streamIsTTY && forceColor === undefined) return 0; + const min = forceColor === undefined || forceColor === 0 || Number.isNaN(forceColor) ? 0 : forceColor; + const env = process.env; + const term = env["TERM"]; + if (term === "dumb") return min; + if (process.platform === "win32") { + const osRelease = os.release().split("."); + if (Number(osRelease[0]) >= 10 && Number(osRelease[2]) >= 10_586) { + return Number(osRelease[2]) >= 14_931 ? 3 : 2; + } + return 1; + } + if (inEnv("CI")) { + if (inEnv("GITHUB_ACTIONS") || inEnv("GITEA_ACTIONS") || inEnv("CIRCLECI")) return 3; + if ( + inEnv("TRAVIS") || + inEnv("APPVEYOR") || + inEnv("GITLAB_CI") || + inEnv("BUILDKITE") || + inEnv("DRONE") || + env["CI_NAME"] === "codeship" + ) { + return 1; + } + return min; + } + const teamcity = env["TEAMCITY_VERSION"]; + if (teamcity !== undefined) { + return /^(9\.(0*[1-9]\d*)\.|\d{2,}\.)/.test(teamcity) ? 1 : 0; + } + if (env["COLORTERM"] === "truecolor") return 3; + if (term === "xterm-kitty") return 3; + if (term === "xterm-ghostty") return 3; + if (term === "wezterm") return 3; + const termProgram = env["TERM_PROGRAM"]; + if (termProgram !== undefined) { + const version = Number.parseInt((env["TERM_PROGRAM_VERSION"] || "").split(".")[0]!, 10); + if (termProgram === "iTerm.app") return version >= 3 ? 3 : 2; + if (termProgram === "Apple_Terminal") return 2; + } + // supports-color tests these against `env.TERM`, which is the string + // "undefined" when TERM is unset. + const termText = term === undefined ? "undefined" : term; + if (/-256(color)?$/i.test(termText)) return 2; + if (/^screen|^xterm|^vt100|^vt220|^rxvt|color|ansi|cygwin|linux/i.test(termText)) return 1; + if (inEnv("COLORTERM")) return 1; + return min; +} + +const stdoutColor: ColorInfo = translateLevel(detectLevel(tty.isatty(1))); +const stderrColor: ColorInfo = translateLevel(detectLevel(tty.isatty(2))); + +export const supportsColor: ColorInfo = stdoutColor; +export const supportsColorStderr: ColorInfo = stderrColor; + +// ── the builder ────────────────────────────────────────────────────────── + +/** One link of a style chain (chalk's styler). */ +class Styler { + open: string; + close: string; + openAll: string; + closeAll: string; + parent: Styler | undefined; + constructor(open: string, close: string, parent: Styler | undefined) { + this.open = open; + this.close = close; + this.parent = parent; + if (parent === undefined) { + this.openAll = open; + this.closeAll = close; + } else { + this.openAll = parent.openAll + open; + this.closeAll = close + parent.closeAll; + } + } +} + +/** The level every builder of one chalk instance shares. */ +class LevelBox { + level: number; + constructor(level: number) { + this.level = level; + } +} + +function stringReplaceAll(string: string, substring: string, replacer: string): string { + let index = string.indexOf(substring); + if (index === -1) return string; + const substringLength = substring.length; + let endIndex = 0; + let returnValue = ""; + do { + returnValue += string.slice(endIndex, index) + substring + replacer; + endIndex = index + substringLength; + index = string.indexOf(substring, endIndex); + } while (index !== -1); + returnValue += string.slice(endIndex); + return returnValue; +} + +function stringEncaseCRLFWithFirstIndex( + string: string, + prefix: string, + postfix: string, + index: number, +): string { + let endIndex = 0; + let returnValue = ""; + do { + const gotCR = string[index - 1] === "\r"; + returnValue += + string.slice(endIndex, gotCR ? index - 1 : index) + prefix + (gotCR ? "\r\n" : "\n") + postfix; + endIndex = index + 1; + index = string.indexOf("\n", endIndex); + } while (index !== -1); + returnValue += string.slice(endIndex); + return returnValue; +} + +function applyStyle(self: ChalkInstance, string: string): string { + if (self.box.level <= 0 || !string) return self.isEmpty ? "" : string; + let styler = self.styler; + if (styler === undefined) return string; + const openAll = styler.openAll; + const closeAll = styler.closeAll; + if (string.includes("\u001B")) { + while (styler !== undefined) { + string = stringReplaceAll(string, styler.close, styler.open); + styler = styler.parent; + } + } + const lfIndex = string.indexOf("\n"); + if (lfIndex !== -1) string = stringEncaseCRLFWithFirstIndex(string, closeAll, openAll, lfIndex); + return openAll + string + closeAll; +} + +function joinArguments(args: unknown[]): string { + let out = ""; + for (let i = 0; i < args.length; i++) { + if (i > 0) out += " "; + const value = args[i]; + if (value !== undefined && value !== null) out += String(value); + } + return out; +} + +/** A chalk instance or one of its style chains: callable (chalk's call + * signature merges in from the interface), with every style a cached + * getter. */ +export interface ChalkInstance { + (...text: unknown[]): string; +} + +export class ChalkInstance { + "%call": (...text: unknown[]) => string; + box: LevelBox; + styler: Styler | undefined; + isEmpty: boolean; + private readonly cache: (ChalkInstance | undefined)[] = []; + + constructor(box: LevelBox, styler: Styler | undefined, isEmpty: boolean, root: boolean) { + this.box = box; + this.styler = styler; + this.isEmpty = isEmpty; + // One unconditional store: the field's definite assignment. + this["%call"] = (...text: unknown[]): string => + root + ? joinArguments(text) + : applyStyle(this, text.length === 1 ? "" + String(text[0]) : joinArguments(text)); + } + + get level(): ColorSupportLevel { + return this.box.level as ColorSupportLevel; + } + + set level(level: ColorSupportLevel) { + this.box.level = level; + } + + private style(id: number): ChalkInstance { + const cached = this.cache[id]; + if (cached !== undefined) return cached; + let builder: ChalkInstance; + if (id === VISIBLE) { + builder = new ChalkInstance(this.box, this.styler, true, false); + } else { + const styler = new Styler( + `${ESC}${OPEN_CODES[id]!}m`, + `${ESC}${CLOSE_CODES[id]!}m`, + this.styler, + ); + builder = new ChalkInstance(this.box, styler, this.isEmpty, false); + } + while (this.cache.length <= id) this.cache.push(undefined); + this.cache[id] = builder; + return builder; + } + + private model(open: string, background: boolean): ChalkInstance { + const styler = new Styler(open, background ? `${ESC}49m` : `${ESC}39m`, this.styler); + return new ChalkInstance(this.box, styler, this.isEmpty, false); + } + + rgb(red: number, green: number, blue: number): ChalkInstance { + return this.model(rgbOpen(this.box.level, 0, red, green, blue), false); + } + hex(color: string): ChalkInstance { + const [red, green, blue] = hexToRgb(color); + return this.model(rgbOpen(this.box.level, 0, red!, green!, blue!), false); + } + ansi256(index: number): ChalkInstance { + return this.model(ansi256Open(0, index), false); + } + bgRgb(red: number, green: number, blue: number): ChalkInstance { + return this.model(rgbOpen(this.box.level, 10, red, green, blue), true); + } + bgHex(color: string): ChalkInstance { + const [red, green, blue] = hexToRgb(color); + return this.model(rgbOpen(this.box.level, 10, red!, green!, blue!), true); + } + bgAnsi256(index: number): ChalkInstance { + return this.model(ansi256Open(10, index), true); + } + + get reset(): ChalkInstance { return this.style(0); } + get bold(): ChalkInstance { return this.style(1); } + get dim(): ChalkInstance { return this.style(2); } + get italic(): ChalkInstance { return this.style(3); } + get underline(): ChalkInstance { return this.style(4); } + get overline(): ChalkInstance { return this.style(5); } + get inverse(): ChalkInstance { return this.style(6); } + get hidden(): ChalkInstance { return this.style(7); } + get strikethrough(): ChalkInstance { return this.style(8); } + get visible(): ChalkInstance { return this.style(VISIBLE); } + get black(): ChalkInstance { return this.style(9); } + get red(): ChalkInstance { return this.style(10); } + get green(): ChalkInstance { return this.style(11); } + get yellow(): ChalkInstance { return this.style(12); } + get blue(): ChalkInstance { return this.style(13); } + get magenta(): ChalkInstance { return this.style(14); } + get cyan(): ChalkInstance { return this.style(15); } + get white(): ChalkInstance { return this.style(16); } + get blackBright(): ChalkInstance { return this.style(17); } + get gray(): ChalkInstance { return this.style(18); } + get grey(): ChalkInstance { return this.style(19); } + get redBright(): ChalkInstance { return this.style(20); } + get greenBright(): ChalkInstance { return this.style(21); } + get yellowBright(): ChalkInstance { return this.style(22); } + get blueBright(): ChalkInstance { return this.style(23); } + get magentaBright(): ChalkInstance { return this.style(24); } + get cyanBright(): ChalkInstance { return this.style(25); } + get whiteBright(): ChalkInstance { return this.style(26); } + get bgBlack(): ChalkInstance { return this.style(27); } + get bgRed(): ChalkInstance { return this.style(28); } + get bgGreen(): ChalkInstance { return this.style(29); } + get bgYellow(): ChalkInstance { return this.style(30); } + get bgBlue(): ChalkInstance { return this.style(31); } + get bgMagenta(): ChalkInstance { return this.style(32); } + get bgCyan(): ChalkInstance { return this.style(33); } + get bgWhite(): ChalkInstance { return this.style(34); } + get bgBlackBright(): ChalkInstance { return this.style(35); } + get bgGray(): ChalkInstance { return this.style(36); } + get bgGrey(): ChalkInstance { return this.style(37); } + get bgRedBright(): ChalkInstance { return this.style(38); } + get bgGreenBright(): ChalkInstance { return this.style(39); } + get bgYellowBright(): ChalkInstance { return this.style(40); } + get bgBlueBright(): ChalkInstance { return this.style(41); } + get bgMagentaBright(): ChalkInstance { return this.style(42); } + get bgCyanBright(): ChalkInstance { return this.style(43); } + get bgWhiteBright(): ChalkInstance { return this.style(44); } +} + +function optionsLevel(options: Options | undefined): number { + const level = options === undefined ? undefined : options.level; + if (level !== undefined && level !== 0 && !(Number.isInteger(level) && level >= 0 && level <= 3)) { + throw new Error("The `level` option should be an integer from 0 to 3"); + } + if (level === undefined) return stdoutColor === false ? 0 : stdoutColor.level; + return level; +} + +/** `new Chalk(options)`: a fresh chalk instance with its own level. */ +export class Chalk extends ChalkInstance { + constructor(options?: Options) { + super(new LevelBox(optionsLevel(options)), undefined, false, true); + } +} + +const chalk: ChalkInstance = new Chalk(); +export const chalkStderr: ChalkInstance = new Chalk({ + level: stderrColor === false ? 0 : stderrColor.level, +}); + +export default chalk; diff --git a/packages/compiler/shims/hot-shots/v8.ts b/packages/compiler/shims/hot-shots/v8.ts new file mode 100644 index 000000000..2c036408e --- /dev/null +++ b/packages/compiler/shims/hot-shots/v8.ts @@ -0,0 +1,2 @@ +export class StatsD {} +export default StatsD; diff --git a/packages/compiler/src/frontend/lowering/lower-calls.ts b/packages/compiler/src/frontend/lowering/lower-calls.ts index 39b7207c0..53971c7b0 100644 --- a/packages/compiler/src/frontend/lowering/lower-calls.ts +++ b/packages/compiler/src/frontend/lowering/lower-calls.ts @@ -1619,7 +1619,13 @@ export function lowerCall(lowerer: Lowerer, expr: ts.CallExpression): IrExpr { } const accessorTarget = lowerer.fieldTarget(expr.expression); if (accessorTarget?.container === "accessor") { - const callee = lowerer.fieldGetExpr(accessorTarget, locOf(expr.expression), expr.expression); + let callee = lowerer.fieldGetExpr(accessorTarget, locOf(expr.expression), expr.expression); + // A getter returning a callable class instance (chalk's + // `chalk.red("x")`): call through its `%call` field. + if (callee.type.kind === "object") { + callee = lowerer.hybridCallUnwrap(callee); + if (callee.type.kind === "func") return lowerCallThroughValue(lowerer, expr, callee, locOf(expr)); + } if (callee.type.kind === "func") { const args = completeFuncValueArgs(lowerer, expr, callee.type, locOf(expr)); return { kind: "callValue", callee, args, type: callee.type.ret, loc: locOf(expr) }; @@ -1871,7 +1877,19 @@ export function lowerCall(lowerer: Lowerer, expr: ts.CallExpression): IrExpr { // a non-func IR type was already rejected while lowering the callee. // HYBRID (function-with-properties) values call through their %call slot. let callee = lowerer.lowerExpr(expr.expression); - if (callee.type.kind === "record") callee = lowerer.hybridCallUnwrap(callee); + if (callee.type.kind === "record" || callee.type.kind === "object") + callee = lowerer.hybridCallUnwrap(callee); + return lowerCallThroughValue(lowerer, expr, callee, loc); +} + +/** A call through an evaluated callee value: checked-dynamic callees, + * function unions, typed-rest and runtime-arity closures. */ +function lowerCallThroughValue( + lowerer: Lowerer, + expr: ts.CallExpression, + callee: IrExpr, + loc: SrcLoc, +): IrExpr { // A CHECKED-DYNAMIC callee — `fn(a, b)` where fn is an implicit-any // JS binding (the mustCall body's `fn(...args)`), a dyn capture, or a // keyed read off a dyn value: the dynCall boundary. Arguments convert @@ -9181,7 +9199,8 @@ export function lowerRecordFieldCall( // A HYBRID (function-with-properties) field is callable through its // reserved %call slot — `colors.blue("x")` where blue also carries // `.bold` (the chalk shape). - if (callee.type.kind === "record") callee = lowerer.hybridCallUnwrap(callee); + if (callee.type.kind === "record" || callee.type.kind === "object") + callee = lowerer.hybridCallUnwrap(callee); // Inferred JavaScript option records can retain checked callables in // fields even when the checker only describes those members as any. // TypeScript reaches here for callable library types that are checked diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index b02a0a843..b1867be32 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -9294,6 +9294,21 @@ export class Lowerer { * type-mapper.ts's chalk-shape mapping — call paths and func-slot coercions * share it. */ hybridCallUnwrap(expr: IrExpr): IrExpr { + // A callable CLASS instance: a class whose merged interface carries a + // call signature stores its call behavior in a `%call` field (the + // chalk package shim's builder). Calls read the field. + if (expr.type.kind === "object") { + const callType = this.classes.get(expr.type.className)?.fields.get("%call"); + if (callType === undefined || callType.kind !== "func") return expr; + return { + kind: "fieldGet", + obj: expr, + className: expr.type.className, + field: "%call", + type: callType, + loc: expr.loc, + }; + } if (expr.type.kind !== "record") return expr; const shape = this.shapes.get(expr.type.shapeId); const call = shape?.fields.find((f) => f.name === "%call"); @@ -9624,7 +9639,7 @@ export class Lowerer { // A HYBRID (function-with-properties) record flowing into a plain // func slot extracts its reserved %call field — the chalk shape's // "the value IS callable" half (type-mapper.ts's hybrid mapping). - if (expected.kind === "func" && expr.type.kind === "record") { + if (expected.kind === "func" && (expr.type.kind === "record" || expr.type.kind === "object")) { const un = this.hybridCallUnwrap(expr); if (un !== expr && typeEquals(un.type, expected)) return un; } diff --git a/packages/compiler/src/frontend/package-shims.ts b/packages/compiler/src/frontend/package-shims.ts index 86a41d3a6..f9cf1deb6 100644 --- a/packages/compiler/src/frontend/package-shims.ts +++ b/packages/compiler/src/frontend/package-shims.ts @@ -62,6 +62,9 @@ interface ImplementationShim { * of it (a __commonJS chunk whose key names the package, its version * and this file) is the package itself. */ mainFile: string; + /** The package's declaration entry relative to its directory, when it + * is not `index.d.ts`. */ + typesFile?: string; } const IMPLEMENTATION_SHIMS: ReadonlyMap = new Map([ @@ -79,6 +82,26 @@ const IMPLEMENTATION_SHIMS: ReadonlyMap = new Map([ ]), }, ], + [ + "chalk", + { + name: "chalk", + defaultExport: true, + mainFile: "source/index.js", + typesFile: "source/index.d.ts", + versions: new Map([[5, "chalk/v5.ts"]]), + }, + ], + [ + "hot-shots", + { + name: "hot-shots", + defaultExport: true, + mainFile: "index.js", + typesFile: "types.d.ts", + versions: new Map([[8, "hot-shots/v8.ts"]]), + }, + ], [ "undici", { @@ -280,16 +303,32 @@ export function isShimmedImport(fromFile: string, specifier: string): boolean { return isUnloadedPackage(specifier) || packageShimFor(fromFile, specifier) !== null; } +/** The shimmed package whose declaration entry `path` is, with the + * package directory: `/index.d.ts`, or the entry's own `typesFile`. */ +function shimOfDeclarationPath(path: string): { pkgDir: string; shim: ImplementationShim } | null { + if (!path.endsWith(".d.ts")) return null; + if (path.endsWith("/index.d.ts")) { + const pkgDir = dirname(path); + const shim = IMPLEMENTATION_SHIMS.get(pkgDir.slice(pkgDir.lastIndexOf("/") + 1)); + if (shim !== undefined && shim.typesFile === undefined) return { pkgDir, shim }; + } + for (const shim of IMPLEMENTATION_SHIMS.values()) { + if (shim.typesFile === undefined) continue; + const suffix = `/${shim.name}/${shim.typesFile}`; + if (path.endsWith(suffix)) return { pkgDir: path.slice(0, path.length - suffix.length + shim.name.length + 1), shim }; + } + return null; +} + /** The fs-shadow answer for a shimmed package's declaration entry: the * installed `index.d.ts` reads as a re-export of the version's shim module, * so the checker types each importer against its own version. Undefined * for every other path. */ export function packageShimDeclarationText(path: string): string | undefined { - if (!path.endsWith("/index.d.ts")) return undefined; - const pkgDir = dirname(path); - const name = pkgDir.slice(pkgDir.lastIndexOf("/") + 1); - const shim = IMPLEMENTATION_SHIMS.get(name); - if (shim === undefined) return undefined; + const located = shimOfDeclarationPath(path); + if (located === null) return undefined; + const { pkgDir, shim } = located; + const name = shim.name; // A DefinitelyTyped twin (`@types/fs-extra`) types a package that ships no // declarations. Nothing at its path names the installed runtime version, // so it reads as the shim only when every supported version shares one From 750c817649e01371c160e91ee5b5263c671440ef Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Wed, 21 Oct 2026 21:45:50 -0700 Subject: [PATCH 014/197] Compile hot-shots 8 from a built-in typed StatsD shim udp over node:dgram (TypeScript may use the send completion-callback form), tcp over node:net, mock mode, child clients, tags, checks and events; uds reports the missing unix-dgram addon as hot-shots does. Co-Authored-By: Claude Opus 5.5 --- packages/compiler/shims/hot-shots/v8.ts | 952 +++++++++++++++++- .../src/frontend/lowering/lower-dgram.ts | 4 +- 2 files changed, 954 insertions(+), 2 deletions(-) diff --git a/packages/compiler/shims/hot-shots/v8.ts b/packages/compiler/shims/hot-shots/v8.ts index 2c036408e..a8af7a19e 100644 --- a/packages/compiler/shims/hot-shots/v8.ts +++ b/packages/compiler/shims/hot-shots/v8.ts @@ -1,2 +1,952 @@ -export class StatsD {} +// scriptc's built-in implementation of the `hot-shots` package, 8.x API. +// +// Programs whose installed hot-shots is 8.x compile this module in its +// place (packages/compiler/src/frontend/package-shims.ts). It is a typed +// port of hot-shots 8.5's StatsD client: the same datagrams, callbacks, +// error messages and child-client sharing, over node:dgram (udp) and +// node:net (tcp). The uds protocol needs the optional `unix-dgram` addon; +// this entry serves programs where it is not installed, so uds reports +// hot-shots' "not installed" error exactly as Node does. +import * as dgram from "node:dgram"; +import * as dns from "node:dns"; +import * as net from "node:net"; +import * as os from "node:os"; + +export type Tags = { [key: string]: string } | string[]; + +export interface ClientOptions { + bufferFlushInterval?: number; + bufferHolder?: { buffer: string }; + cacheDns?: boolean; + cacheDnsTtl?: number; + errorHandler?: (err: Error) => void; + globalTags?: Tags; + globalize?: boolean; + host?: string; + isChild?: boolean; + maxBufferSize?: number; + mock?: boolean; + path?: string; + port?: number; + prefix?: string; + protocol?: "tcp" | "udp" | "uds"; + sampleRate?: number; + suffix?: string; + telegraf?: boolean; + useDefaultRoute?: boolean; + tagPrefix?: string; + tagSeparator?: string; + udsGracefulErrorHandling?: boolean; + udsGracefulRestartRateLimit?: number; + closingFlushInterval?: number; +} + +export interface ChildClientOptions { + globalTags?: Tags; + prefix?: string; + suffix?: string; + errorHandler?: (err: Error) => void; +} + +export interface CheckOptions { + /** A Unix timestamp; hot-shots also takes a Date, which this shim's + * option records cannot hold yet. */ + date_happened?: number; + hostname?: string; + message?: string; +} + +export interface DatadogChecks { + OK: 0; + WARNING: 1; + CRITICAL: 2; + UNKNOWN: 3; +} + +export type DatadogChecksValues = 0 | 1 | 2 | 3; + +export interface EventOptions { + aggregation_key?: string; + alert_type?: "info" | "warning" | "success" | "error"; + /** A Unix timestamp; hot-shots also takes a Date, which this shim's + * option records cannot hold yet. */ + date_happened?: number; + hostname?: string; + priority?: "low" | "normal"; + source_type_name?: string; +} + +/** A send's completion: hot-shots passes `null` on success. */ +export type StatsCb = (error: Error | null | undefined, bytes?: any) => void; + +type SampleArg = number | Tags | StatsCb | undefined; +type TagsArg = Tags | StatsCb | undefined; + +const PROTOCOL_TCP = "tcp"; +const PROTOCOL_UDS = "uds"; +const PROTOCOL_UDP = "udp"; +const UDS_DEFAULT_GRACEFUL_RESTART_LIMIT = 1000; +const CACHE_DNS_TTL_DEFAULT = 60000; + +function udsErrorCodes(): number[] { + if (process.platform === "linux") { + return [os.constants.errno.ENOTCONN, os.constants.errno.ECONNREFUSED]; + } + if (process.platform === "darwin") { + return [os.constants.errno.EDESTADDRREQ, os.constants.errno.ECONNRESET]; + } + return []; +} +const UDS_ERROR_CODES = udsErrorCodes(); + +// ── helpers ────────────────────────────────────────────────────────────── + +function sanitizeTags(value: string, telegraf: boolean): string { + const blacklist = telegraf ? /:|\||,/g : /:|\||@|,/g; + return String(value).replace(blacklist, "_"); +} + +function formatTags(tags: Tags, telegraf: boolean): string[] { + if (Array.isArray(tags)) return tags; + const out: string[] = []; + for (const key of Object.keys(tags)) { + out.push(`${sanitizeTags(key, telegraf)}:${sanitizeTags(tags[key]!, telegraf)}`); + } + return out; +} + +function isArrayIndexKey(key: string): boolean { + return /^(0|[1-9]\d*)$/.test(key) && Number(key) < 4294967295; +} + +function overrideTags(parent: string[], child: Tags, telegraf: boolean): string[] { + const childKeys: string[] = []; + const childValues = new Map(); + const toAppend: string[] = []; + for (const tag of formatTags(child, telegraf)) { + const idx = tag.indexOf(":"); + if (idx < 1) { + toAppend.push(tag); + } else { + const key = tag.substring(0, idx); + const value = tag.substring(idx + 1); + let values = childValues.get(key); + if (values === undefined) { + values = []; + childValues.set(key, values); + childKeys.push(key); + } + values.push(value); + } + } + const result = parent.filter((tag) => { + const idx = tag.indexOf(":"); + if (idx < 1) return true; + return !childValues.has(tag.substring(0, idx)); + }); + // hot-shots walks an object's keys: array-index keys ascending first, + // then the rest in insertion order. + const indexKeys = childKeys.filter(isArrayIndexKey).sort((a, b) => Number(a) - Number(b)); + const ordered = indexKeys.concat(childKeys.filter((key) => !isArrayIndexKey(key))); + for (const key of ordered) { + for (const value of childValues.get(key)!) result.push(`${key}:${value}`); + } + return result.concat(toAppend); +} + +function formatDate(date: number): number { + return Math.round(date); +} + +function errorWithCode(message: string, code: string | undefined): Error { + const error: NodeJS.ErrnoException = new Error(message); + error.code = code; + return error; +} + +// ── transports ─────────────────────────────────────────────────────────── + +type SendCallback = (err: Error | null, bytes?: number) => void; + +/** The socket a client sends through (hot-shots' transport interface). */ +interface Transport { + createdAt: number; + send(buf: Buffer, callback: SendCallback): void; + close(): void; + onError(listener: (err: Error) => void): void; + offError(listener: (err: Error) => void): void; + onClose(listener: () => void): void; +} + +/** A socket's 'error' listeners, kept by the transport (the typed + * sockets have no removeListener): one socket listener dispatches to a + * snapshot, and an error nobody listens for throws, as an unhandled + * 'error' event does. */ +class ErrorListeners { + private listeners: ((err: Error) => void)[] = []; + add(listener: (err: Error) => void): void { + this.listeners.push(listener); + } + remove(listener: (err: Error) => void): void { + const index = this.listeners.indexOf(listener); + if (index !== -1) this.listeners.splice(index, 1); + } + emit(err: Error): void { + const snapshot = this.listeners.slice(); + if (snapshot.length === 0) throw err; + for (const listener of snapshot) listener(err); + } +} + +function addEol(buf: Buffer): string { + let msg = buf.toString(); + if (msg.length > 0 && msg[msg.length - 1] !== "\n") msg += "\n"; + return msg; +} + +class UdpTransport implements Transport { + createdAt = 0; + private readonly socket: dgram.Socket; + private readonly errors = new ErrorListeners(); + private resolvedAddress: string | undefined = undefined; + private resolvedAt = 0; + private readonly host: string | undefined; + private readonly port: number; + private readonly cacheDns: boolean; + private readonly cacheDnsTtl: number; + constructor(host: string | undefined, port: number, cacheDns: boolean, cacheDnsTtl: number) { + this.host = host; + this.port = port; + this.cacheDns = cacheDns; + this.cacheDnsTtl = cacheDnsTtl; + this.socket = dgram.createSocket("udp4"); + this.socket.unref(); + this.socket.on("error", (err) => this.errors.emit(err)); + } + send(buf: Buffer, callback: SendCallback): void { + if (!this.cacheDns) { + this.socket.send(buf, this.port, this.host || "127.0.0.1", callback); + return; + } + const now = Date.now(); + const cached = this.resolvedAddress; + if (cached === undefined || now - this.resolvedAt > this.cacheDnsTtl) { + dns.lookup(this.host ?? "localhost", (error, address) => { + if (error) { + callback(error); + return; + } + this.resolvedAddress = address; + this.resolvedAt = now; + this.socket.send(buf, this.port, address, callback); + }); + } else { + this.socket.send(buf, this.port, cached, callback); + } + } + close(): void { + this.socket.close(); + } + onError(listener: (err: Error) => void): void { + this.errors.add(listener); + } + offError(listener: (err: Error) => void): void { + this.errors.remove(listener); + } + onClose(listener: () => void): void { + this.socket.on("close", listener); + } +} + +class TcpTransport implements Transport { + createdAt = 0; + private readonly socket: net.Socket; + private readonly errors = new ErrorListeners(); + constructor(host: string | undefined, port: number) { + this.socket = net.connect(port, host ?? "localhost"); + this.socket.setKeepAlive(true); + this.socket.unref(); + this.socket.on("error", (err) => this.errors.emit(err)); + } + send(buf: Buffer, callback: SendCallback): void { + // The typed socket's write takes the data alone; the completion runs + // on the next tick. + this.socket.write(addEol(buf)); + process.nextTick(() => callback(null)); + } + close(): void { + this.socket.destroy(); + } + onError(listener: (err: Error) => void): void { + this.errors.add(listener); + } + offError(listener: (err: Error) => void): void { + this.errors.remove(listener); + } + onClose(listener: () => void): void { + this.socket.on("close", () => listener()); + } +} + +/** hot-shots reaches Unix datagram sockets through the optional + * `unix-dgram` addon; this entry serves programs without it. */ +function createUdsTransport(): Transport { + throw new Error( + "The library `unix_dgram`, needed for the uds protocol to work, is not installed. " + + "You need to pick another protocol to use hot-shots. " + + "See the hot-shots README for additional details.", + ); +} + +function createTransport(client: StatsD): Transport | null { + let transport: Transport | null = null; + const protocol = client.protocol; + try { + if (protocol === PROTOCOL_TCP) { + transport = new TcpTransport(client.host, client.port); + } else if (protocol === PROTOCOL_UDS) { + transport = createUdsTransport(); + } else if (protocol === PROTOCOL_UDP) { + transport = new UdpTransport(client.host, client.port, client.cacheDns, client.cacheDnsTtl); + } else { + throw new Error(`Unsupported protocol '${protocol}'`); + } + transport.createdAt = Date.now(); + } catch (e) { + const handler = client.errorHandler; + if (handler) { + handler(e as Error); + } else { + console.error(e); + } + } + return transport; +} + +// ── the client ─────────────────────────────────────────────────────────── + +class BufferHolder { + buffer = ""; +} + +/** What a child client inherits from its parent. */ +class ChildInit { + socket: Transport | null; + bufferHolder: BufferHolder; + dnsError: Error | undefined; + constructor(socket: Transport | null, bufferHolder: BufferHolder, dnsError: Error | undefined) { + this.socket = socket; + this.bufferHolder = bufferHolder; + this.dnsError = dnsError; + } +} + +export class StatsD { + protocol: string; + cacheDns: boolean; + cacheDnsTtl: number; + host: string | undefined; + port: number; + path: string | undefined; + prefix: string; + suffix: string; + tagPrefix: string; + tagSeparator: string; + mock: boolean; + globalTags: string[]; + telegraf: boolean; + maxBufferSize: number; + sampleRate: number; + bufferFlushInterval: number; + errorHandler: ((err: Error) => void) | undefined; + udsGracefulErrorHandling: boolean; + udsGracefulRestartRateLimit: number; + isChild: boolean; + closingFlushInterval: number; + mockBuffer: string[] | undefined = undefined; + messagesInFlight = 0; + CHECKS: DatadogChecks = { OK: 0, WARNING: 1, CRITICAL: 2, UNKNOWN: 3 }; + private bufferHolder: BufferHolder; + private transport: Transport | null = null; + private dnsError: Error | undefined = undefined; + private stopFlushInterval: (() => void) | undefined = undefined; + + constructor(options?: ClientOptions, child?: ChildInit) { + const o: ClientOptions = options ?? {}; + const protocol = o.protocol ? o.protocol.toLowerCase() : ""; + this.protocol = protocol ? protocol : PROTOCOL_UDP; + this.cacheDns = o.cacheDns === true; + this.cacheDnsTtl = o.cacheDnsTtl || CACHE_DNS_TTL_DEFAULT; + this.host = o.host || process.env["DD_AGENT_HOST"]; + this.port = o.port || parseInt(process.env["DD_DOGSTATSD_PORT"] ?? "", 10) || 8125; + this.path = o.path; + this.prefix = o.prefix || ""; + this.suffix = o.suffix || ""; + this.tagPrefix = o.tagPrefix || "#"; + this.tagSeparator = o.tagSeparator || ","; + this.mock = o.mock === true; + this.telegraf = o.telegraf || false; + const globalTags = o.globalTags; + this.globalTags = globalTags !== undefined ? formatTags(globalTags, this.telegraf) : []; + const entityId = process.env["DD_ENTITY_ID"]; + if (entityId) { + this.globalTags = this.globalTags.filter( + (item) => item.indexOf("dd.internal.entity_id:") !== 0, + ); + this.globalTags.push("dd.internal.entity_id:".concat(sanitizeTags(entityId, false))); + } + this.maxBufferSize = o.maxBufferSize || 0; + this.sampleRate = o.sampleRate || 1; + this.bufferFlushInterval = o.bufferFlushInterval || 1000; + this.bufferHolder = child !== undefined ? child.bufferHolder : new BufferHolder(); + this.errorHandler = o.errorHandler; + const graceful = o.udsGracefulErrorHandling; + this.udsGracefulErrorHandling = graceful === undefined ? true : graceful; + this.udsGracefulRestartRateLimit = + o.udsGracefulRestartRateLimit || UDS_DEFAULT_GRACEFUL_RESTART_LIMIT; + this.isChild = child !== undefined; + this.closingFlushInterval = o.closingFlushInterval || 50; + if (this.mock) this.mockBuffer = []; + if (child === undefined && this.maxBufferSize > 0) { + const handle = setInterval(() => this.onBufferFlushInterval(), this.bufferFlushInterval); + handle.unref(); + this.stopFlushInterval = () => clearInterval(handle); + } + if (child !== undefined) { + if (child.dnsError) this.dnsError = child.dnsError; + this.transport = child.socket; + } + if (!this.transport) this.transport = createTransport(this); + const handler = o.errorHandler; + const transport = this.transport; + if (transport !== null && child === undefined && handler) transport.onError(handler); + if (transport !== null && o.protocol === PROTOCOL_UDS) this.maybeAddUDSErrorHandler(); + } + + private maybeAddUDSErrorHandler(): void { + if (this.isChild || !this.udsGracefulErrorHandling) return; + const transport = this.transport; + if (transport !== null) transport.onError((err) => this.udsErrorHandler(err)); + } + + private udsErrorHandler(err: Error): void { + const code = Number((err as NodeJS.ErrnoException).code); + if (!UDS_ERROR_CODES.includes(-code)) return; + const socket = this.transport; + if (socket === null) return; + if (!socket.createdAt || Date.now() - socket.createdAt < this.udsGracefulRestartRateLimit) { + return; + } + const handler = this.errorHandler; + if (handler) socket.offError(handler); + const newSocket = createTransport(this); + if (newSocket === null) { + const errorMessage = "Could not replace UDS connection with new socket"; + if (handler) { + handler(new Error(errorMessage)); + } else { + console.error(errorMessage); + } + return; + } + socket.close(); + this.transport = newSocket; + this.maybeAddUDSErrorHandler(); + if (handler) { + newSocket.onError(handler); + } else { + newSocket.onError((error) => console.error(`hot-shots UDS error: ${String(error)}`)); + } + } + + // ── stats functions ── + + timing(stat: string | string[], time: number, sampleRate?: number, tags?: Tags, callback?: StatsCb): void; + timing(stat: string | string[], time: number, tags?: Tags, callback?: StatsCb): void; + timing(stat: string | string[], time: number, callback?: StatsCb): void; + timing(stat: string | string[], time: number, sampleRate?: number, callback?: StatsCb): void; + timing( + stat: string | string[], + time: number, + sampleRate?: SampleArg, + tags?: TagsArg, + callback?: StatsCb, + ): void { + this.sendAll(stat, time, "ms", sampleRate, tags, callback); + } + + increment(stat: string, tags?: Tags): void; + increment(stat: string | string[], value: number, sampleRate?: number, tags?: Tags, callback?: StatsCb): void; + increment(stat: string | string[], value: number, tags?: Tags, callback?: StatsCb): void; + increment(stat: string | string[], value: number, callback?: StatsCb): void; + increment(stat: string | string[], value: number, sampleRate?: number, callback?: StatsCb): void; + increment( + stat: string | string[], + value?: number | Tags, + sampleRate?: SampleArg, + tags?: TagsArg, + callback?: StatsCb, + ): void { + // A non-number second argument of a two-argument call is the tags. + let amount = 1; + let tagArg: TagsArg = tags; + if (typeof value === "number") { + amount = value; + } else if (sampleRate === undefined && tags === undefined && callback === undefined) { + tagArg = value; + } + this.sendAll(stat, amount, "c", sampleRate, tagArg, callback); + } + + decrement(stat: string, tags?: Tags): void; + decrement(stat: string | string[], value: number, sampleRate?: number, tags?: Tags, callback?: StatsCb): void; + decrement(stat: string | string[], value: number, tags?: Tags, callback?: StatsCb): void; + decrement(stat: string | string[], value: number, callback?: StatsCb): void; + decrement(stat: string | string[], value: number, sampleRate?: number, callback?: StatsCb): void; + decrement( + stat: string | string[], + value?: number | Tags, + sampleRate?: SampleArg, + tags?: TagsArg, + callback?: StatsCb, + ): void { + let amount = 1; + let tagArg: TagsArg = tags; + if (typeof value === "number") { + amount = value; + } else if (sampleRate === undefined && tags === undefined && callback === undefined) { + tagArg = value; + } + this.sendAll(stat, -amount, "c", sampleRate, tagArg, callback); + } + + histogram(stat: string | string[], value: number, sampleRate?: number, tags?: Tags, callback?: StatsCb): void; + histogram(stat: string | string[], value: number, tags?: Tags, callback?: StatsCb): void; + histogram(stat: string | string[], value: number, callback?: StatsCb): void; + histogram(stat: string | string[], value: number, sampleRate?: number, callback?: StatsCb): void; + histogram( + stat: string | string[], + value: number, + sampleRate?: SampleArg, + tags?: TagsArg, + callback?: StatsCb, + ): void { + this.sendAll(stat, value, "h", sampleRate, tags, callback); + } + + distribution(stat: string | string[], value: number, sampleRate?: number, tags?: Tags, callback?: StatsCb): void; + distribution(stat: string | string[], value: number, tags?: Tags, callback?: StatsCb): void; + distribution(stat: string | string[], value: number, callback?: StatsCb): void; + distribution(stat: string | string[], value: number, sampleRate?: number, callback?: StatsCb): void; + distribution( + stat: string | string[], + value: number, + sampleRate?: SampleArg, + tags?: TagsArg, + callback?: StatsCb, + ): void { + this.sendAll(stat, value, "d", sampleRate, tags, callback); + } + + gauge(stat: string | string[], value: number, sampleRate?: number, tags?: Tags, callback?: StatsCb): void; + gauge(stat: string | string[], value: number, tags?: Tags, callback?: StatsCb): void; + gauge(stat: string | string[], value: number, callback?: StatsCb): void; + gauge(stat: string | string[], value: number, sampleRate?: number, callback?: StatsCb): void; + gauge( + stat: string | string[], + value: number, + sampleRate?: SampleArg, + tags?: TagsArg, + callback?: StatsCb, + ): void { + this.sendAll(stat, value, "g", sampleRate, tags, callback); + } + + set(stat: string | string[], value: number | string, sampleRate?: number, tags?: Tags, callback?: StatsCb): void; + set(stat: string | string[], value: number | string, tags?: Tags, callback?: StatsCb): void; + set(stat: string | string[], value: number | string, callback?: StatsCb): void; + set(stat: string | string[], value: number | string, sampleRate?: number, callback?: StatsCb): void; + set( + stat: string | string[], + value: number | string, + sampleRate?: SampleArg, + tags?: TagsArg, + callback?: StatsCb, + ): void { + this.sendAll(stat, value, "s", sampleRate, tags, callback); + } + + unique(stat: string | string[], value: number | string, sampleRate?: number, tags?: Tags, callback?: StatsCb): void; + unique(stat: string | string[], value: number | string, tags?: Tags, callback?: StatsCb): void; + unique(stat: string | string[], value: number | string, callback?: StatsCb): void; + unique(stat: string | string[], value: number | string, sampleRate?: number, callback?: StatsCb): void; + unique( + stat: string | string[], + value: number | string, + sampleRate?: SampleArg, + tags?: TagsArg, + callback?: StatsCb, + ): void { + this.sendAll(stat, value, "s", sampleRate, tags, callback); + } + + check( + name: string, + status: DatadogChecksValues, + options?: CheckOptions, + tags?: Tags | StatsCb, + callback?: StatsCb, + ): void { + if (this.telegraf) { + const err = new Error("Not supported by Telegraf / InfluxDB"); + if (callback) return callback(err); + const handler = this.errorHandler; + if (handler) return handler(err); + throw err; + } + const check: string[] = ["_sc", this.prefix + name + this.suffix, String(status)]; + const metadata: CheckOptions = options ?? {}; + const happened = metadata.date_happened; + if (happened) { + const timestamp = formatDate(happened); + if (timestamp) check.push(`d:${timestamp}`); + } + if (metadata.hostname) check.push(`h:${metadata.hostname}`); + let mergedTags = this.globalTags; + if (tags !== undefined && typeof tags !== "function") { + mergedTags = overrideTags(mergedTags, tags, this.telegraf); + } + if (mergedTags.length > 0) check.push(`#${mergedTags.join(",")}`); + if (metadata.message) check.push(`m:${metadata.message}`); + let cb = callback; + if (typeof tags === "function" && callback === undefined) cb = tags; + this.sendRaw(check.join("|"), cb); + } + + event( + title: string, + text?: string, + options?: EventOptions, + tags?: Tags | StatsCb, + callback?: StatsCb, + ): void { + if (this.telegraf) { + const err = new Error("Not supported by Telegraf / InfluxDB"); + if (callback) return callback(err); + const handler = this.errorHandler; + if (handler) return handler(err); + throw err; + } + const msgTitle = String(title ? title : ""); + let msgText = String(text ? text : msgTitle); + msgText = msgText.replace(/\n/g, "\\n"); + let message = `_e{${msgTitle.length},${msgText.length}}:${msgTitle}|${msgText}`; + if (options) { + const happened = options.date_happened; + if (happened) { + const timestamp = formatDate(happened); + if (timestamp) message += `|d:${timestamp}`; + } + if (options.hostname) message += `|h:${options.hostname}`; + if (options.aggregation_key) message += `|k:${options.aggregation_key}`; + if (options.priority) message += `|p:${options.priority}`; + if (options.source_type_name) message += `|s:${options.source_type_name}`; + if (options.alert_type) message += `|t:${options.alert_type}`; + } + let cb = callback; + let tagList: Tags | undefined = undefined; + if (typeof tags === "function") { + if (callback === undefined) cb = tags; + } else { + tagList = tags; + } + this.send(message, tagList, cb); + } + + // ── sending ── + + private sendAll( + stat: string | string[], + value: number | string, + type: string, + sampleRateArg: SampleArg, + tagsArg: TagsArg, + callbackArg: StatsCb | undefined, + ): void { + let sampleRate: number | undefined = undefined; + let tags: Tags | undefined = undefined; + let callback: StatsCb | undefined = callbackArg; + if (typeof sampleRateArg === "number") { + sampleRate = sampleRateArg; + if (typeof tagsArg === "function") { + callback = tagsArg; + } else { + tags = tagsArg; + } + } else if (sampleRateArg !== undefined) { + // A non-number sample rate shifts into the tags, and the tags + // argument into the callback. + if (typeof sampleRateArg === "function") { + callback = sampleRateArg; + } else { + tags = sampleRateArg; + callback = typeof tagsArg === "function" ? tagsArg : undefined; + } + } else if (typeof tagsArg === "function") { + callback = tagsArg; + } else { + tags = tagsArg; + } + + if (!Array.isArray(stat)) { + this.sendStat(stat, value, type, sampleRate, tags, callback); + return; + } + let completed = 0; + let calledback = false; + let sentBytes = 0; + const count = stat.length; + const onSend: StatsCb = (error, bytes) => { + completed += 1; + if (calledback) return; + if (error) { + if (callback !== undefined) { + calledback = true; + callback(error); + } else { + const handler = this.errorHandler; + if (handler) { + calledback = true; + handler(error); + } + } + return; + } + if (bytes) sentBytes += bytes as number; + if (completed === count && callback !== undefined) callback(null, sentBytes); + }; + for (const item of stat) this.sendStat(item, value, type, sampleRate, tags, onSend); + } + + private sendStat( + stat: string, + value: number | string, + type: string, + sampleRateArg: number | undefined, + tags: Tags | undefined, + callback: StatsCb | undefined, + ): void { + let message = `${this.prefix + stat + this.suffix}:${value}|${type}`; + const sampleRate = sampleRateArg || this.sampleRate; + if (sampleRate && sampleRate < 1) { + if (Math.random() < sampleRate) { + message += `|@${sampleRate}`; + } else { + if (callback) callback(undefined); + return; + } + } + this.send(message, tags, callback); + } + + private send(message: string, tags: Tags | undefined, callback: StatsCb | undefined): void { + let mergedTags = this.globalTags; + if (tags !== undefined) mergedTags = overrideTags(mergedTags, tags, this.telegraf); + if (mergedTags.length > 0) { + if (this.telegraf) { + const parts = message.split(":"); + message = `${parts[0]!},${mergedTags.join(",").replace(/:/g, "=")}:${parts.slice(1).join(":")}`; + } else { + message += `|${this.tagPrefix}${mergedTags.join(this.tagSeparator)}`; + } + } + this.sendRaw(message, callback); + } + + /** hot-shots' `_send`: the message with its tags. */ + private sendRaw(message: string, callback: StatsCb | undefined): void { + const dnsError = this.dnsError; + if (dnsError) { + if (callback) return callback(dnsError); + const handler = this.errorHandler; + if (handler) return handler(dnsError); + throw dnsError; + } + if (!this.mock) { + if (this.maxBufferSize === 0) { + this.sendMessage(message, callback); + } else { + this.enqueue(message, callback); + } + } else { + const buffer = this.mockBuffer; + if (buffer !== undefined) buffer.push(message); + if (callback) callback(null, 0); + } + } + + private enqueue(message: string, callback: StatsCb | undefined): void { + message += "\n"; + if (this.bufferHolder.buffer.length + message.length > this.maxBufferSize) { + this.flushQueue(callback); + this.bufferHolder.buffer += message; + } else { + this.bufferHolder.buffer += message; + if (callback) callback(null); + } + } + + private flushQueue(callback?: StatsCb): void { + this.sendMessage(this.bufferHolder.buffer, callback); + this.bufferHolder.buffer = ""; + } + + private sendMessage(message: string, callback: StatsCb | undefined): void { + if (message === "") { + if (callback) callback(null); + return; + } + const socketWasMissing = this.transport === null; + if (socketWasMissing && this.protocol === PROTOCOL_UDS) { + this.transport = createTransport(this); + if (this.transport !== null) this.maybeAddUDSErrorHandler(); + } + const socket = this.transport; + if (socketWasMissing || socket === null) { + const error = new Error("Socket not created properly. Check previous errors for details."); + if (callback) return callback(error); + const handler = this.errorHandler; + if (handler) return handler(error); + console.error(String(error)); + return; + } + const handleCallback = (err: Error | null, bytes?: number): void => { + this.messagesInFlight--; + let errFormatted: Error | null = null; + if (err) { + errFormatted = errorWithCode( + `Error sending hot-shots message: ${String(err)}`, + (err as NodeJS.ErrnoException).code, + ); + if (this.protocol === PROTOCOL_UDS && (callback || this.errorHandler)) { + this.udsErrorHandler(err); + } + } + if (callback) { + callback(errFormatted, bytes); + } else if (errFormatted) { + const handler = this.errorHandler; + if (handler) { + handler(errFormatted); + } else { + console.error(String(errFormatted)); + } + } + }; + try { + this.messagesInFlight++; + socket.send(Buffer.from(message), handleCallback); + } catch (err) { + handleCallback(err as Error); + } + } + + private onBufferFlushInterval(): void { + this.flushQueue(); + } + + close(callback?: (error?: Error) => void): void { + const stop = this.stopFlushInterval; + if (stop !== undefined) stop(); + this.flushQueue((err) => { + if (err) { + if (callback) return callback(err); + console.error(err); + return; + } + let intervalAttempts = 0; + const waitForMessages = setInterval(() => { + intervalAttempts++; + if (intervalAttempts > 10) { + console.log("hot-shots could not clear out messages in flight but closing anyways"); + this.messagesInFlight = 0; + } + if (this.messagesInFlight <= 0) { + clearInterval(waitForMessages); + this.closeSocket(callback); + } + }, this.closingFlushInterval); + }); + } + + private closeSocket(callback: ((error?: Error) => void) | undefined): void { + let handledError = false; + const handleErr = (err: unknown): void => { + const errMessage = `Error closing hot-shots socket: ${String(err)}`; + if (handledError) { + console.error(errMessage); + } else { + handledError = true; + if (callback) { + callback(new Error(errMessage)); + } else { + const handler = this.errorHandler; + if (handler) { + handler(new Error(errMessage)); + } else { + console.error(errMessage); + } + } + } + }; + const socket = this.transport; + if (socket === null) { + handleErr(new TypeError("Cannot read properties of null (reading 'removeListener')")); + return; + } + const handler = this.errorHandler; + if (handler) socket.offError(handler); + socket.onError(handleErr); + if (callback) { + socket.onClose(() => { + if (!handledError) callback(undefined); + }); + } + try { + socket.close(); + } catch (err) { + handleErr(err); + } + } + + childClient(options?: ChildClientOptions): StatsD { + const o: ChildClientOptions = options ?? {}; + const childTags = o.globalTags; + return new StatsD( + { + isChild: true, + errorHandler: o.errorHandler || this.errorHandler, + host: this.host, + port: this.port, + tagPrefix: this.tagPrefix, + tagSeparator: this.tagSeparator, + prefix: (o.prefix || "") + this.prefix, + suffix: this.suffix + (o.suffix || ""), + globalize: false, + mock: this.mock, + globalTags: + childTags !== undefined + ? overrideTags(this.globalTags, childTags, this.telegraf) + : this.globalTags, + maxBufferSize: this.maxBufferSize, + bufferFlushInterval: this.bufferFlushInterval, + telegraf: this.telegraf, + protocol: this.protocol === "tcp" ? "tcp" : this.protocol === "uds" ? "uds" : "udp", + closingFlushInterval: this.closingFlushInterval, + }, + new ChildInit(this.transport, this.bufferHolder, this.dnsError), + ); + } +} + export default StatsD; diff --git a/packages/compiler/src/frontend/lowering/lower-dgram.ts b/packages/compiler/src/frontend/lowering/lower-dgram.ts index aa90fc7c7..0e0ddf975 100644 --- a/packages/compiler/src/frontend/lowering/lower-dgram.ts +++ b/packages/compiler/src/frontend/lowering/lower-dgram.ts @@ -612,8 +612,10 @@ export function lowerDgramMethodCall( const fn: IrLibFn = data.type.kind === "string" ? "dgram.sendStr" : "dgram.sendBytes"; return { kind: "libCall", fn, args: [receiver, data, port, host], type: VOID, loc }; } + // TypeScript reaches the ladder for the completion-callback form + // send(msg, port, address, callback) (the hot-shots package shim). if ( - isJsSourceFile(ts.getSourceFileOfNode(call)) && + (isJsSourceFile(ts.getSourceFileOfNode(call)) || args.length === 4) && args.length <= 6 && !args.some(ts.isSpreadElement) ) { From cab61904fd43bfa9d4736e2c9a72b435fd9b9aef Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Wed, 21 Oct 2026 21:55:38 -0700 Subject: [PATCH 015/197] Send hot-shots uds metrics through a native Unix datagram socket Where unix-dgram is installed beside hot-shots, the shim's entry sends through runtime AF_UNIX datagram calls with the addon's semantics; without it, uds reports the addon missing as hot-shots does. Co-Authored-By: Claude Opus 5.5 --- packages/compiler/shims/hot-shots/client.ts | 1033 +++++++++++++++++ .../compiler/shims/hot-shots/v8-unix-dgram.ts | 8 + packages/compiler/shims/hot-shots/v8.ts | 956 +-------------- .../compiler/src/backend/llvm/lib-datagram.ts | 4 + .../src/frontend/lowering/lower-calls.ts | 43 + .../src/frontend/lowering/lower-namespaces.ts | 8 +- .../compiler/src/frontend/package-shims.ts | 27 +- .../compiler/src/ir/builtin-signatures.ts | 7 + packages/runtime/src/scr_dgram.c | 70 ++ 9 files changed, 1202 insertions(+), 954 deletions(-) create mode 100644 packages/compiler/shims/hot-shots/client.ts create mode 100644 packages/compiler/shims/hot-shots/v8-unix-dgram.ts diff --git a/packages/compiler/shims/hot-shots/client.ts b/packages/compiler/shims/hot-shots/client.ts new file mode 100644 index 000000000..d0b5c8924 --- /dev/null +++ b/packages/compiler/shims/hot-shots/client.ts @@ -0,0 +1,1033 @@ +// scriptc's built-in implementation of the `hot-shots` package, 8.x API: +// a typed port of hot-shots 8.5's StatsD client with the same datagrams, +// callbacks, error messages and child-client sharing, over node:dgram +// (udp), node:net (tcp) and a native Unix datagram socket (uds). The +// entry modules (v8.ts, v8-unix-dgram.ts) say whether the optional +// unix-dgram addon that hot-shots' uds protocol loads is installed. +import * as dgram from "node:dgram"; +import * as dns from "node:dns"; +import * as net from "node:net"; +import * as os from "node:os"; + +export type Tags = { [key: string]: string } | string[]; + +export interface ClientOptions { + bufferFlushInterval?: number; + bufferHolder?: { buffer: string }; + cacheDns?: boolean; + cacheDnsTtl?: number; + errorHandler?: (err: Error) => void; + globalTags?: Tags; + globalize?: boolean; + host?: string; + isChild?: boolean; + maxBufferSize?: number; + mock?: boolean; + path?: string; + port?: number; + prefix?: string; + protocol?: "tcp" | "udp" | "uds"; + sampleRate?: number; + suffix?: string; + telegraf?: boolean; + useDefaultRoute?: boolean; + tagPrefix?: string; + tagSeparator?: string; + udsGracefulErrorHandling?: boolean; + udsGracefulRestartRateLimit?: number; + closingFlushInterval?: number; +} + +export interface ChildClientOptions { + globalTags?: Tags; + prefix?: string; + suffix?: string; + errorHandler?: (err: Error) => void; +} + +export interface CheckOptions { + /** A Unix timestamp; hot-shots also takes a Date, which this shim's + * option records cannot hold yet. */ + date_happened?: number; + hostname?: string; + message?: string; +} + +export interface DatadogChecks { + OK: 0; + WARNING: 1; + CRITICAL: 2; + UNKNOWN: 3; +} + +export type DatadogChecksValues = 0 | 1 | 2 | 3; + +export interface EventOptions { + aggregation_key?: string; + alert_type?: "info" | "warning" | "success" | "error"; + /** A Unix timestamp; hot-shots also takes a Date, which this shim's + * option records cannot hold yet. */ + date_happened?: number; + hostname?: string; + priority?: "low" | "normal"; + source_type_name?: string; +} + +/** A send's completion: hot-shots passes `null` on success. */ +export type StatsCb = (error: Error | null | undefined, bytes?: any) => void; + +type SampleArg = number | Tags | StatsCb | undefined; +type TagsArg = Tags | StatsCb | undefined; + +const PROTOCOL_TCP = "tcp"; +const PROTOCOL_UDS = "uds"; +const PROTOCOL_UDP = "udp"; +const UDS_DEFAULT_GRACEFUL_RESTART_LIMIT = 1000; +const CACHE_DNS_TTL_DEFAULT = 60000; + +function udsErrorCodes(): number[] { + if (process.platform === "linux") { + return [os.constants.errno.ENOTCONN, os.constants.errno.ECONNREFUSED]; + } + if (process.platform === "darwin") { + return [os.constants.errno.EDESTADDRREQ, os.constants.errno.ECONNRESET]; + } + return []; +} +const UDS_ERROR_CODES = udsErrorCodes(); + +// ── helpers ────────────────────────────────────────────────────────────── + +function sanitizeTags(value: string, telegraf: boolean): string { + const blacklist = telegraf ? /:|\||,/g : /:|\||@|,/g; + return String(value).replace(blacklist, "_"); +} + +function formatTags(tags: Tags, telegraf: boolean): string[] { + if (Array.isArray(tags)) return tags; + const out: string[] = []; + for (const key of Object.keys(tags)) { + out.push(`${sanitizeTags(key, telegraf)}:${sanitizeTags(tags[key]!, telegraf)}`); + } + return out; +} + +function isArrayIndexKey(key: string): boolean { + return /^(0|[1-9]\d*)$/.test(key) && Number(key) < 4294967295; +} + +function overrideTags(parent: string[], child: Tags, telegraf: boolean): string[] { + const childKeys: string[] = []; + const childValues = new Map(); + const toAppend: string[] = []; + for (const tag of formatTags(child, telegraf)) { + const idx = tag.indexOf(":"); + if (idx < 1) { + toAppend.push(tag); + } else { + const key = tag.substring(0, idx); + const value = tag.substring(idx + 1); + let values = childValues.get(key); + if (values === undefined) { + values = []; + childValues.set(key, values); + childKeys.push(key); + } + values.push(value); + } + } + const result = parent.filter((tag) => { + const idx = tag.indexOf(":"); + if (idx < 1) return true; + return !childValues.has(tag.substring(0, idx)); + }); + // hot-shots walks an object's keys: array-index keys ascending first, + // then the rest in insertion order. + const indexKeys = childKeys.filter(isArrayIndexKey).sort((a, b) => Number(a) - Number(b)); + const ordered = indexKeys.concat(childKeys.filter((key) => !isArrayIndexKey(key))); + for (const key of ordered) { + for (const value of childValues.get(key)!) result.push(`${key}:${value}`); + } + return result.concat(toAppend); +} + +function formatDate(date: number): number { + return Math.round(date); +} + +function errorWithCode(message: string, code: string | undefined): Error { + const error: NodeJS.ErrnoException = new Error(message); + error.code = code; + return error; +} + +// ── transports ─────────────────────────────────────────────────────────── + +type SendCallback = (err: Error | null, bytes?: number) => void; + +/** The socket a client sends through (hot-shots' transport interface). */ +interface Transport { + createdAt: number; + send(buf: Buffer, callback: SendCallback): void; + close(): void; + onError(listener: (err: Error) => void): void; + offError(listener: (err: Error) => void): void; + onClose(listener: () => void): void; +} + +/** A socket's 'error' listeners, kept by the transport (the typed + * sockets have no removeListener): one socket listener dispatches to a + * snapshot, and an error nobody listens for throws, as an unhandled + * 'error' event does. */ +class ErrorListeners { + private listeners: ((err: Error) => void)[] = []; + add(listener: (err: Error) => void): void { + this.listeners.push(listener); + } + remove(listener: (err: Error) => void): void { + const index = this.listeners.indexOf(listener); + if (index !== -1) this.listeners.splice(index, 1); + } + emit(err: Error): void { + const snapshot = this.listeners.slice(); + if (snapshot.length === 0) throw err; + for (const listener of snapshot) listener(err); + } +} + +function addEol(buf: Buffer): string { + let msg = buf.toString(); + if (msg.length > 0 && msg[msg.length - 1] !== "\n") msg += "\n"; + return msg; +} + +class UdpTransport implements Transport { + createdAt = 0; + private readonly socket: dgram.Socket; + private readonly errors = new ErrorListeners(); + private resolvedAddress: string | undefined = undefined; + private resolvedAt = 0; + private readonly host: string | undefined; + private readonly port: number; + private readonly cacheDns: boolean; + private readonly cacheDnsTtl: number; + constructor(host: string | undefined, port: number, cacheDns: boolean, cacheDnsTtl: number) { + this.host = host; + this.port = port; + this.cacheDns = cacheDns; + this.cacheDnsTtl = cacheDnsTtl; + this.socket = dgram.createSocket("udp4"); + this.socket.unref(); + this.socket.on("error", (err) => this.errors.emit(err)); + } + send(buf: Buffer, callback: SendCallback): void { + if (!this.cacheDns) { + this.socket.send(buf, this.port, this.host || "127.0.0.1", callback); + return; + } + const now = Date.now(); + const cached = this.resolvedAddress; + if (cached === undefined || now - this.resolvedAt > this.cacheDnsTtl) { + dns.lookup(this.host ?? "localhost", (error, address) => { + if (error) { + callback(error); + return; + } + this.resolvedAddress = address; + this.resolvedAt = now; + this.socket.send(buf, this.port, address, callback); + }); + } else { + this.socket.send(buf, this.port, cached, callback); + } + } + close(): void { + this.socket.close(); + } + onError(listener: (err: Error) => void): void { + this.errors.add(listener); + } + offError(listener: (err: Error) => void): void { + this.errors.remove(listener); + } + onClose(listener: () => void): void { + this.socket.on("close", listener); + } +} + +class TcpTransport implements Transport { + createdAt = 0; + private readonly socket: net.Socket; + private readonly errors = new ErrorListeners(); + constructor(host: string | undefined, port: number) { + this.socket = net.connect(port, host ?? "localhost"); + this.socket.setKeepAlive(true); + this.socket.unref(); + this.socket.on("error", (err) => this.errors.emit(err)); + } + send(buf: Buffer, callback: SendCallback): void { + // The typed socket's write takes the data alone; the completion runs + // on the next tick. + this.socket.write(addEol(buf)); + process.nextTick(() => callback(null)); + } + close(): void { + this.socket.destroy(); + } + onError(listener: (err: Error) => void): void { + this.errors.add(listener); + } + offError(listener: (err: Error) => void): void { + this.errors.remove(listener); + } + onClose(listener: () => void): void { + this.socket.on("close", () => listener()); + } +} + +// The runtime's AF_UNIX datagram calls (the unix-dgram addon's binding): +// results >= 0, or a negative errno; send answers 1 when it would block. +declare function scriptcUnixDgramSocket(): number; +declare function scriptcUnixDgramConnect(fd: number, path: string): number; +declare function scriptcUnixDgramSend(fd: number, data: Buffer): number; +declare function scriptcUnixDgramClose(fd: number): number; + +let unixDgramInstalled = false; + +/** Whether the unix-dgram addon is installed beside hot-shots (set by the + * entry module). */ +export function configureUnixDgram(installed: boolean): void { + unixDgramInstalled = installed; +} + +const UDS_PATH_DEFAULT = "/var/run/datadog/dsd.socket"; + +/** unix-dgram's errnoException: " " (the code is the + * negative errno; this Error's code field holds its decimal string). */ +function errnoException(errorno: number, syscall: string): Error { + const error: NodeJS.ErrnoException = new Error(`${syscall} ${errorno}`); + error.errno = errorno; + error.code = String(errorno); + error.syscall = syscall; + return error; +} + +/** hot-shots' uds transport over unix-dgram: a connected, non-blocking + * AF_UNIX datagram socket. Sends complete synchronously, as the addon's + * do. The addon polls the socket for reads, which holds the event loop + * until close; a ref'd timer stands in for that poll. */ +class UdsTransport implements Transport { + createdAt = 0; + private fd: number; + private readonly errors = new ErrorListeners(); + private readonly closeListeners: (() => void)[] = []; + private readonly stopHold: () => void; + constructor(path: string) { + const fd = scriptcUnixDgramSocket(); + if (fd < 0) throw errnoException(fd, "socket"); + this.fd = fd; + const hold = setInterval(() => {}, 2147483647); + this.stopHold = () => clearInterval(hold); + const err = scriptcUnixDgramConnect(fd, path); + if (err < 0) { + // The addon emits the connect failure as 'error' before anyone + // listens, which throws; hot-shots closes the socket and rethrows. + this.close(); + throw errnoException(err, "connect"); + } + } + send(buf: Buffer, callback: SendCallback): void { + const result = scriptcUnixDgramSend(this.fd, buf); + if (result < 0) { + callback(errnoException(result, "send")); + } else if (result === 1) { + const congestion: NodeJS.ErrnoException = new Error("congestion"); + congestion.code = "1"; + callback(congestion); + } else { + callback(null); + } + } + close(): void { + const result = scriptcUnixDgramClose(this.fd); + if (result < 0) throw errnoException(result, "close"); + this.fd = -1; + this.stopHold(); + for (const listener of this.closeListeners.slice()) listener(); + } + onError(listener: (err: Error) => void): void { + this.errors.add(listener); + } + offError(listener: (err: Error) => void): void { + this.errors.remove(listener); + } + onClose(listener: () => void): void { + this.closeListeners.push(listener); + } +} + +/** hot-shots' createUdsTransport: without the unix-dgram addon the + * protocol reports it missing. */ +function createUdsTransport(path: string | undefined): Transport { + if (!unixDgramInstalled) { + throw new Error( + "The library `unix_dgram`, needed for the uds protocol to work, is not installed. " + + "You need to pick another protocol to use hot-shots. " + + "See the hot-shots README for additional details.", + ); + } + return new UdsTransport(path ? path : UDS_PATH_DEFAULT); +} + +function createTransport(client: StatsD): Transport | null { + let transport: Transport | null = null; + const protocol = client.protocol; + try { + if (protocol === PROTOCOL_TCP) { + transport = new TcpTransport(client.host, client.port); + } else if (protocol === PROTOCOL_UDS) { + transport = createUdsTransport(client.path); + } else if (protocol === PROTOCOL_UDP) { + transport = new UdpTransport(client.host, client.port, client.cacheDns, client.cacheDnsTtl); + } else { + throw new Error(`Unsupported protocol '${protocol}'`); + } + transport.createdAt = Date.now(); + } catch (e) { + const handler = client.errorHandler; + if (handler) { + handler(e as Error); + } else { + console.error(e); + } + } + return transport; +} + +// ── the client ─────────────────────────────────────────────────────────── + +class BufferHolder { + buffer = ""; +} + +/** What a child client inherits from its parent. */ +class ChildInit { + socket: Transport | null; + bufferHolder: BufferHolder; + dnsError: Error | undefined; + constructor(socket: Transport | null, bufferHolder: BufferHolder, dnsError: Error | undefined) { + this.socket = socket; + this.bufferHolder = bufferHolder; + this.dnsError = dnsError; + } +} + +export class StatsD { + protocol: string; + cacheDns: boolean; + cacheDnsTtl: number; + host: string | undefined; + port: number; + path: string | undefined; + prefix: string; + suffix: string; + tagPrefix: string; + tagSeparator: string; + mock: boolean; + globalTags: string[]; + telegraf: boolean; + maxBufferSize: number; + sampleRate: number; + bufferFlushInterval: number; + errorHandler: ((err: Error) => void) | undefined; + udsGracefulErrorHandling: boolean; + udsGracefulRestartRateLimit: number; + isChild: boolean; + closingFlushInterval: number; + mockBuffer: string[] | undefined = undefined; + messagesInFlight = 0; + CHECKS: DatadogChecks = { OK: 0, WARNING: 1, CRITICAL: 2, UNKNOWN: 3 }; + private bufferHolder: BufferHolder; + private transport: Transport | null = null; + private dnsError: Error | undefined = undefined; + private stopFlushInterval: (() => void) | undefined = undefined; + + constructor(options?: ClientOptions, child?: ChildInit) { + const o: ClientOptions = options ?? {}; + const protocol = o.protocol ? o.protocol.toLowerCase() : ""; + this.protocol = protocol ? protocol : PROTOCOL_UDP; + this.cacheDns = o.cacheDns === true; + this.cacheDnsTtl = o.cacheDnsTtl || CACHE_DNS_TTL_DEFAULT; + this.host = o.host || process.env["DD_AGENT_HOST"]; + this.port = o.port || parseInt(process.env["DD_DOGSTATSD_PORT"] ?? "", 10) || 8125; + this.path = o.path; + this.prefix = o.prefix || ""; + this.suffix = o.suffix || ""; + this.tagPrefix = o.tagPrefix || "#"; + this.tagSeparator = o.tagSeparator || ","; + this.mock = o.mock === true; + this.telegraf = o.telegraf || false; + const globalTags = o.globalTags; + this.globalTags = globalTags !== undefined ? formatTags(globalTags, this.telegraf) : []; + const entityId = process.env["DD_ENTITY_ID"]; + if (entityId) { + this.globalTags = this.globalTags.filter( + (item) => item.indexOf("dd.internal.entity_id:") !== 0, + ); + this.globalTags.push("dd.internal.entity_id:".concat(sanitizeTags(entityId, false))); + } + this.maxBufferSize = o.maxBufferSize || 0; + this.sampleRate = o.sampleRate || 1; + this.bufferFlushInterval = o.bufferFlushInterval || 1000; + this.bufferHolder = child !== undefined ? child.bufferHolder : new BufferHolder(); + this.errorHandler = o.errorHandler; + const graceful = o.udsGracefulErrorHandling; + this.udsGracefulErrorHandling = graceful === undefined ? true : graceful; + this.udsGracefulRestartRateLimit = + o.udsGracefulRestartRateLimit || UDS_DEFAULT_GRACEFUL_RESTART_LIMIT; + this.isChild = child !== undefined; + this.closingFlushInterval = o.closingFlushInterval || 50; + if (this.mock) this.mockBuffer = []; + if (child === undefined && this.maxBufferSize > 0) { + const handle = setInterval(() => this.onBufferFlushInterval(), this.bufferFlushInterval); + handle.unref(); + this.stopFlushInterval = () => clearInterval(handle); + } + if (child !== undefined) { + if (child.dnsError) this.dnsError = child.dnsError; + this.transport = child.socket; + } + if (!this.transport) this.transport = createTransport(this); + const handler = o.errorHandler; + const transport = this.transport; + if (transport !== null && child === undefined && handler) transport.onError(handler); + if (transport !== null && o.protocol === PROTOCOL_UDS) this.maybeAddUDSErrorHandler(); + } + + private maybeAddUDSErrorHandler(): void { + if (this.isChild || !this.udsGracefulErrorHandling) return; + const transport = this.transport; + if (transport !== null) transport.onError((err) => this.udsErrorHandler(err)); + } + + private udsErrorHandler(err: Error): void { + const code = Number((err as NodeJS.ErrnoException).code); + if (!UDS_ERROR_CODES.includes(-code)) return; + const socket = this.transport; + if (socket === null) return; + if (!socket.createdAt || Date.now() - socket.createdAt < this.udsGracefulRestartRateLimit) { + return; + } + const handler = this.errorHandler; + if (handler) socket.offError(handler); + const newSocket = createTransport(this); + if (newSocket === null) { + const errorMessage = "Could not replace UDS connection with new socket"; + if (handler) { + handler(new Error(errorMessage)); + } else { + console.error(errorMessage); + } + return; + } + socket.close(); + this.transport = newSocket; + this.maybeAddUDSErrorHandler(); + if (handler) { + newSocket.onError(handler); + } else { + newSocket.onError((error) => console.error(`hot-shots UDS error: ${String(error)}`)); + } + } + + // ── stats functions ── + + timing(stat: string | string[], time: number, sampleRate?: number, tags?: Tags, callback?: StatsCb): void; + timing(stat: string | string[], time: number, tags?: Tags, callback?: StatsCb): void; + timing(stat: string | string[], time: number, callback?: StatsCb): void; + timing(stat: string | string[], time: number, sampleRate?: number, callback?: StatsCb): void; + timing( + stat: string | string[], + time: number, + sampleRate?: SampleArg, + tags?: TagsArg, + callback?: StatsCb, + ): void { + this.sendAll(stat, time, "ms", sampleRate, tags, callback); + } + + increment(stat: string, tags?: Tags): void; + increment(stat: string | string[], value: number, sampleRate?: number, tags?: Tags, callback?: StatsCb): void; + increment(stat: string | string[], value: number, tags?: Tags, callback?: StatsCb): void; + increment(stat: string | string[], value: number, callback?: StatsCb): void; + increment(stat: string | string[], value: number, sampleRate?: number, callback?: StatsCb): void; + increment( + stat: string | string[], + value?: number | Tags, + sampleRate?: SampleArg, + tags?: TagsArg, + callback?: StatsCb, + ): void { + // A non-number second argument of a two-argument call is the tags. + let amount = 1; + let tagArg: TagsArg = tags; + if (typeof value === "number") { + amount = value; + } else if (sampleRate === undefined && tags === undefined && callback === undefined) { + tagArg = value; + } + this.sendAll(stat, amount, "c", sampleRate, tagArg, callback); + } + + decrement(stat: string, tags?: Tags): void; + decrement(stat: string | string[], value: number, sampleRate?: number, tags?: Tags, callback?: StatsCb): void; + decrement(stat: string | string[], value: number, tags?: Tags, callback?: StatsCb): void; + decrement(stat: string | string[], value: number, callback?: StatsCb): void; + decrement(stat: string | string[], value: number, sampleRate?: number, callback?: StatsCb): void; + decrement( + stat: string | string[], + value?: number | Tags, + sampleRate?: SampleArg, + tags?: TagsArg, + callback?: StatsCb, + ): void { + let amount = 1; + let tagArg: TagsArg = tags; + if (typeof value === "number") { + amount = value; + } else if (sampleRate === undefined && tags === undefined && callback === undefined) { + tagArg = value; + } + this.sendAll(stat, -amount, "c", sampleRate, tagArg, callback); + } + + histogram(stat: string | string[], value: number, sampleRate?: number, tags?: Tags, callback?: StatsCb): void; + histogram(stat: string | string[], value: number, tags?: Tags, callback?: StatsCb): void; + histogram(stat: string | string[], value: number, callback?: StatsCb): void; + histogram(stat: string | string[], value: number, sampleRate?: number, callback?: StatsCb): void; + histogram( + stat: string | string[], + value: number, + sampleRate?: SampleArg, + tags?: TagsArg, + callback?: StatsCb, + ): void { + this.sendAll(stat, value, "h", sampleRate, tags, callback); + } + + distribution(stat: string | string[], value: number, sampleRate?: number, tags?: Tags, callback?: StatsCb): void; + distribution(stat: string | string[], value: number, tags?: Tags, callback?: StatsCb): void; + distribution(stat: string | string[], value: number, callback?: StatsCb): void; + distribution(stat: string | string[], value: number, sampleRate?: number, callback?: StatsCb): void; + distribution( + stat: string | string[], + value: number, + sampleRate?: SampleArg, + tags?: TagsArg, + callback?: StatsCb, + ): void { + this.sendAll(stat, value, "d", sampleRate, tags, callback); + } + + gauge(stat: string | string[], value: number, sampleRate?: number, tags?: Tags, callback?: StatsCb): void; + gauge(stat: string | string[], value: number, tags?: Tags, callback?: StatsCb): void; + gauge(stat: string | string[], value: number, callback?: StatsCb): void; + gauge(stat: string | string[], value: number, sampleRate?: number, callback?: StatsCb): void; + gauge( + stat: string | string[], + value: number, + sampleRate?: SampleArg, + tags?: TagsArg, + callback?: StatsCb, + ): void { + this.sendAll(stat, value, "g", sampleRate, tags, callback); + } + + set(stat: string | string[], value: number | string, sampleRate?: number, tags?: Tags, callback?: StatsCb): void; + set(stat: string | string[], value: number | string, tags?: Tags, callback?: StatsCb): void; + set(stat: string | string[], value: number | string, callback?: StatsCb): void; + set(stat: string | string[], value: number | string, sampleRate?: number, callback?: StatsCb): void; + set( + stat: string | string[], + value: number | string, + sampleRate?: SampleArg, + tags?: TagsArg, + callback?: StatsCb, + ): void { + this.sendAll(stat, value, "s", sampleRate, tags, callback); + } + + unique(stat: string | string[], value: number | string, sampleRate?: number, tags?: Tags, callback?: StatsCb): void; + unique(stat: string | string[], value: number | string, tags?: Tags, callback?: StatsCb): void; + unique(stat: string | string[], value: number | string, callback?: StatsCb): void; + unique(stat: string | string[], value: number | string, sampleRate?: number, callback?: StatsCb): void; + unique( + stat: string | string[], + value: number | string, + sampleRate?: SampleArg, + tags?: TagsArg, + callback?: StatsCb, + ): void { + this.sendAll(stat, value, "s", sampleRate, tags, callback); + } + + check( + name: string, + status: DatadogChecksValues, + options?: CheckOptions, + tags?: Tags | StatsCb, + callback?: StatsCb, + ): void { + if (this.telegraf) { + const err = new Error("Not supported by Telegraf / InfluxDB"); + if (callback) return callback(err); + const handler = this.errorHandler; + if (handler) return handler(err); + throw err; + } + const check: string[] = ["_sc", this.prefix + name + this.suffix, String(status)]; + const metadata: CheckOptions = options ?? {}; + const happened = metadata.date_happened; + if (happened) { + const timestamp = formatDate(happened); + if (timestamp) check.push(`d:${timestamp}`); + } + if (metadata.hostname) check.push(`h:${metadata.hostname}`); + let mergedTags = this.globalTags; + if (tags !== undefined && typeof tags !== "function") { + mergedTags = overrideTags(mergedTags, tags, this.telegraf); + } + if (mergedTags.length > 0) check.push(`#${mergedTags.join(",")}`); + if (metadata.message) check.push(`m:${metadata.message}`); + let cb = callback; + if (typeof tags === "function" && callback === undefined) cb = tags; + this.sendRaw(check.join("|"), cb); + } + + event( + title: string, + text?: string, + options?: EventOptions, + tags?: Tags | StatsCb, + callback?: StatsCb, + ): void { + if (this.telegraf) { + const err = new Error("Not supported by Telegraf / InfluxDB"); + if (callback) return callback(err); + const handler = this.errorHandler; + if (handler) return handler(err); + throw err; + } + const msgTitle = String(title ? title : ""); + let msgText = String(text ? text : msgTitle); + msgText = msgText.replace(/\n/g, "\\n"); + let message = `_e{${msgTitle.length},${msgText.length}}:${msgTitle}|${msgText}`; + if (options) { + const happened = options.date_happened; + if (happened) { + const timestamp = formatDate(happened); + if (timestamp) message += `|d:${timestamp}`; + } + if (options.hostname) message += `|h:${options.hostname}`; + if (options.aggregation_key) message += `|k:${options.aggregation_key}`; + if (options.priority) message += `|p:${options.priority}`; + if (options.source_type_name) message += `|s:${options.source_type_name}`; + if (options.alert_type) message += `|t:${options.alert_type}`; + } + let cb = callback; + let tagList: Tags | undefined = undefined; + if (typeof tags === "function") { + if (callback === undefined) cb = tags; + } else { + tagList = tags; + } + this.send(message, tagList, cb); + } + + // ── sending ── + + private sendAll( + stat: string | string[], + value: number | string, + type: string, + sampleRateArg: SampleArg, + tagsArg: TagsArg, + callbackArg: StatsCb | undefined, + ): void { + let sampleRate: number | undefined = undefined; + let tags: Tags | undefined = undefined; + let callback: StatsCb | undefined = callbackArg; + if (typeof sampleRateArg === "number") { + sampleRate = sampleRateArg; + if (typeof tagsArg === "function") { + callback = tagsArg; + } else { + tags = tagsArg; + } + } else if (sampleRateArg !== undefined) { + // A non-number sample rate shifts into the tags, and the tags + // argument into the callback. + if (typeof sampleRateArg === "function") { + callback = sampleRateArg; + } else { + tags = sampleRateArg; + callback = typeof tagsArg === "function" ? tagsArg : undefined; + } + } else if (typeof tagsArg === "function") { + callback = tagsArg; + } else { + tags = tagsArg; + } + + if (!Array.isArray(stat)) { + this.sendStat(stat, value, type, sampleRate, tags, callback); + return; + } + let completed = 0; + let calledback = false; + let sentBytes = 0; + const count = stat.length; + const onSend: StatsCb = (error, bytes) => { + completed += 1; + if (calledback) return; + if (error) { + if (callback !== undefined) { + calledback = true; + callback(error); + } else { + const handler = this.errorHandler; + if (handler) { + calledback = true; + handler(error); + } + } + return; + } + if (bytes) sentBytes += bytes as number; + if (completed === count && callback !== undefined) callback(null, sentBytes); + }; + for (const item of stat) this.sendStat(item, value, type, sampleRate, tags, onSend); + } + + private sendStat( + stat: string, + value: number | string, + type: string, + sampleRateArg: number | undefined, + tags: Tags | undefined, + callback: StatsCb | undefined, + ): void { + let message = `${this.prefix + stat + this.suffix}:${value}|${type}`; + const sampleRate = sampleRateArg || this.sampleRate; + if (sampleRate && sampleRate < 1) { + if (Math.random() < sampleRate) { + message += `|@${sampleRate}`; + } else { + if (callback) callback(undefined); + return; + } + } + this.send(message, tags, callback); + } + + private send(message: string, tags: Tags | undefined, callback: StatsCb | undefined): void { + let mergedTags = this.globalTags; + if (tags !== undefined) mergedTags = overrideTags(mergedTags, tags, this.telegraf); + if (mergedTags.length > 0) { + if (this.telegraf) { + const parts = message.split(":"); + message = `${parts[0]!},${mergedTags.join(",").replace(/:/g, "=")}:${parts.slice(1).join(":")}`; + } else { + message += `|${this.tagPrefix}${mergedTags.join(this.tagSeparator)}`; + } + } + this.sendRaw(message, callback); + } + + /** hot-shots' `_send`: the message with its tags. */ + private sendRaw(message: string, callback: StatsCb | undefined): void { + const dnsError = this.dnsError; + if (dnsError) { + if (callback) return callback(dnsError); + const handler = this.errorHandler; + if (handler) return handler(dnsError); + throw dnsError; + } + if (!this.mock) { + if (this.maxBufferSize === 0) { + this.sendMessage(message, callback); + } else { + this.enqueue(message, callback); + } + } else { + const buffer = this.mockBuffer; + if (buffer !== undefined) buffer.push(message); + if (callback) callback(null, 0); + } + } + + private enqueue(message: string, callback: StatsCb | undefined): void { + message += "\n"; + if (this.bufferHolder.buffer.length + message.length > this.maxBufferSize) { + this.flushQueue(callback); + this.bufferHolder.buffer += message; + } else { + this.bufferHolder.buffer += message; + if (callback) callback(null); + } + } + + private flushQueue(callback?: StatsCb): void { + this.sendMessage(this.bufferHolder.buffer, callback); + this.bufferHolder.buffer = ""; + } + + private sendMessage(message: string, callback: StatsCb | undefined): void { + if (message === "") { + if (callback) callback(null); + return; + } + const socketWasMissing = this.transport === null; + if (socketWasMissing && this.protocol === PROTOCOL_UDS) { + this.transport = createTransport(this); + if (this.transport !== null) this.maybeAddUDSErrorHandler(); + } + const socket = this.transport; + if (socketWasMissing || socket === null) { + const error = new Error("Socket not created properly. Check previous errors for details."); + if (callback) return callback(error); + const handler = this.errorHandler; + if (handler) return handler(error); + console.error(String(error)); + return; + } + const handleCallback = (err: Error | null, bytes?: number): void => { + this.messagesInFlight--; + let errFormatted: Error | null = null; + if (err) { + errFormatted = errorWithCode( + `Error sending hot-shots message: ${String(err)}`, + (err as NodeJS.ErrnoException).code, + ); + if (this.protocol === PROTOCOL_UDS && (callback || this.errorHandler)) { + this.udsErrorHandler(err); + } + } + if (callback) { + callback(errFormatted, bytes); + } else if (errFormatted) { + const handler = this.errorHandler; + if (handler) { + handler(errFormatted); + } else { + console.error(String(errFormatted)); + } + } + }; + try { + this.messagesInFlight++; + socket.send(Buffer.from(message), handleCallback); + } catch (err) { + handleCallback(err as Error); + } + } + + private onBufferFlushInterval(): void { + this.flushQueue(); + } + + close(callback?: (error?: Error) => void): void { + const stop = this.stopFlushInterval; + if (stop !== undefined) stop(); + this.flushQueue((err) => { + if (err) { + if (callback) return callback(err); + console.error(err); + return; + } + let intervalAttempts = 0; + const waitForMessages = setInterval(() => { + intervalAttempts++; + if (intervalAttempts > 10) { + console.log("hot-shots could not clear out messages in flight but closing anyways"); + this.messagesInFlight = 0; + } + if (this.messagesInFlight <= 0) { + clearInterval(waitForMessages); + this.closeSocket(callback); + } + }, this.closingFlushInterval); + }); + } + + private closeSocket(callback: ((error?: Error) => void) | undefined): void { + let handledError = false; + const handleErr = (err: unknown): void => { + const errMessage = `Error closing hot-shots socket: ${String(err)}`; + if (handledError) { + console.error(errMessage); + } else { + handledError = true; + if (callback) { + callback(new Error(errMessage)); + } else { + const handler = this.errorHandler; + if (handler) { + handler(new Error(errMessage)); + } else { + console.error(errMessage); + } + } + } + }; + const socket = this.transport; + if (socket === null) { + handleErr(new TypeError("Cannot read properties of null (reading 'removeListener')")); + return; + } + const handler = this.errorHandler; + if (handler) socket.offError(handler); + socket.onError(handleErr); + if (callback) { + socket.onClose(() => { + if (!handledError) callback(undefined); + }); + } + try { + socket.close(); + } catch (err) { + handleErr(err); + } + } + + childClient(options?: ChildClientOptions): StatsD { + const o: ChildClientOptions = options ?? {}; + const childTags = o.globalTags; + return new StatsD( + { + isChild: true, + errorHandler: o.errorHandler || this.errorHandler, + host: this.host, + port: this.port, + tagPrefix: this.tagPrefix, + tagSeparator: this.tagSeparator, + prefix: (o.prefix || "") + this.prefix, + suffix: this.suffix + (o.suffix || ""), + globalize: false, + mock: this.mock, + globalTags: + childTags !== undefined + ? overrideTags(this.globalTags, childTags, this.telegraf) + : this.globalTags, + maxBufferSize: this.maxBufferSize, + bufferFlushInterval: this.bufferFlushInterval, + telegraf: this.telegraf, + protocol: this.protocol === "tcp" ? "tcp" : this.protocol === "uds" ? "uds" : "udp", + closingFlushInterval: this.closingFlushInterval, + }, + new ChildInit(this.transport, this.bufferHolder, this.dnsError), + ); + } +} + +export default StatsD; diff --git a/packages/compiler/shims/hot-shots/v8-unix-dgram.ts b/packages/compiler/shims/hot-shots/v8-unix-dgram.ts new file mode 100644 index 000000000..f4bfb180e --- /dev/null +++ b/packages/compiler/shims/hot-shots/v8-unix-dgram.ts @@ -0,0 +1,8 @@ +// The hot-shots 8.x shim for installations with the optional unix-dgram +// addon: the uds protocol sends through a native Unix datagram socket. +import { configureUnixDgram } from "./client.ts"; + +configureUnixDgram(true); + +export * from "./client.ts"; +export { default } from "./client.ts"; diff --git a/packages/compiler/shims/hot-shots/v8.ts b/packages/compiler/shims/hot-shots/v8.ts index a8af7a19e..0d1871170 100644 --- a/packages/compiler/shims/hot-shots/v8.ts +++ b/packages/compiler/shims/hot-shots/v8.ts @@ -1,952 +1,8 @@ -// scriptc's built-in implementation of the `hot-shots` package, 8.x API. -// -// Programs whose installed hot-shots is 8.x compile this module in its -// place (packages/compiler/src/frontend/package-shims.ts). It is a typed -// port of hot-shots 8.5's StatsD client: the same datagrams, callbacks, -// error messages and child-client sharing, over node:dgram (udp) and -// node:net (tcp). The uds protocol needs the optional `unix-dgram` addon; -// this entry serves programs where it is not installed, so uds reports -// hot-shots' "not installed" error exactly as Node does. -import * as dgram from "node:dgram"; -import * as dns from "node:dns"; -import * as net from "node:net"; -import * as os from "node:os"; +// The hot-shots 8.x shim for installations without the optional unix-dgram +// addon: the uds protocol reports it missing, as hot-shots does under Node. +import { configureUnixDgram } from "./client.ts"; -export type Tags = { [key: string]: string } | string[]; +configureUnixDgram(false); -export interface ClientOptions { - bufferFlushInterval?: number; - bufferHolder?: { buffer: string }; - cacheDns?: boolean; - cacheDnsTtl?: number; - errorHandler?: (err: Error) => void; - globalTags?: Tags; - globalize?: boolean; - host?: string; - isChild?: boolean; - maxBufferSize?: number; - mock?: boolean; - path?: string; - port?: number; - prefix?: string; - protocol?: "tcp" | "udp" | "uds"; - sampleRate?: number; - suffix?: string; - telegraf?: boolean; - useDefaultRoute?: boolean; - tagPrefix?: string; - tagSeparator?: string; - udsGracefulErrorHandling?: boolean; - udsGracefulRestartRateLimit?: number; - closingFlushInterval?: number; -} - -export interface ChildClientOptions { - globalTags?: Tags; - prefix?: string; - suffix?: string; - errorHandler?: (err: Error) => void; -} - -export interface CheckOptions { - /** A Unix timestamp; hot-shots also takes a Date, which this shim's - * option records cannot hold yet. */ - date_happened?: number; - hostname?: string; - message?: string; -} - -export interface DatadogChecks { - OK: 0; - WARNING: 1; - CRITICAL: 2; - UNKNOWN: 3; -} - -export type DatadogChecksValues = 0 | 1 | 2 | 3; - -export interface EventOptions { - aggregation_key?: string; - alert_type?: "info" | "warning" | "success" | "error"; - /** A Unix timestamp; hot-shots also takes a Date, which this shim's - * option records cannot hold yet. */ - date_happened?: number; - hostname?: string; - priority?: "low" | "normal"; - source_type_name?: string; -} - -/** A send's completion: hot-shots passes `null` on success. */ -export type StatsCb = (error: Error | null | undefined, bytes?: any) => void; - -type SampleArg = number | Tags | StatsCb | undefined; -type TagsArg = Tags | StatsCb | undefined; - -const PROTOCOL_TCP = "tcp"; -const PROTOCOL_UDS = "uds"; -const PROTOCOL_UDP = "udp"; -const UDS_DEFAULT_GRACEFUL_RESTART_LIMIT = 1000; -const CACHE_DNS_TTL_DEFAULT = 60000; - -function udsErrorCodes(): number[] { - if (process.platform === "linux") { - return [os.constants.errno.ENOTCONN, os.constants.errno.ECONNREFUSED]; - } - if (process.platform === "darwin") { - return [os.constants.errno.EDESTADDRREQ, os.constants.errno.ECONNRESET]; - } - return []; -} -const UDS_ERROR_CODES = udsErrorCodes(); - -// ── helpers ────────────────────────────────────────────────────────────── - -function sanitizeTags(value: string, telegraf: boolean): string { - const blacklist = telegraf ? /:|\||,/g : /:|\||@|,/g; - return String(value).replace(blacklist, "_"); -} - -function formatTags(tags: Tags, telegraf: boolean): string[] { - if (Array.isArray(tags)) return tags; - const out: string[] = []; - for (const key of Object.keys(tags)) { - out.push(`${sanitizeTags(key, telegraf)}:${sanitizeTags(tags[key]!, telegraf)}`); - } - return out; -} - -function isArrayIndexKey(key: string): boolean { - return /^(0|[1-9]\d*)$/.test(key) && Number(key) < 4294967295; -} - -function overrideTags(parent: string[], child: Tags, telegraf: boolean): string[] { - const childKeys: string[] = []; - const childValues = new Map(); - const toAppend: string[] = []; - for (const tag of formatTags(child, telegraf)) { - const idx = tag.indexOf(":"); - if (idx < 1) { - toAppend.push(tag); - } else { - const key = tag.substring(0, idx); - const value = tag.substring(idx + 1); - let values = childValues.get(key); - if (values === undefined) { - values = []; - childValues.set(key, values); - childKeys.push(key); - } - values.push(value); - } - } - const result = parent.filter((tag) => { - const idx = tag.indexOf(":"); - if (idx < 1) return true; - return !childValues.has(tag.substring(0, idx)); - }); - // hot-shots walks an object's keys: array-index keys ascending first, - // then the rest in insertion order. - const indexKeys = childKeys.filter(isArrayIndexKey).sort((a, b) => Number(a) - Number(b)); - const ordered = indexKeys.concat(childKeys.filter((key) => !isArrayIndexKey(key))); - for (const key of ordered) { - for (const value of childValues.get(key)!) result.push(`${key}:${value}`); - } - return result.concat(toAppend); -} - -function formatDate(date: number): number { - return Math.round(date); -} - -function errorWithCode(message: string, code: string | undefined): Error { - const error: NodeJS.ErrnoException = new Error(message); - error.code = code; - return error; -} - -// ── transports ─────────────────────────────────────────────────────────── - -type SendCallback = (err: Error | null, bytes?: number) => void; - -/** The socket a client sends through (hot-shots' transport interface). */ -interface Transport { - createdAt: number; - send(buf: Buffer, callback: SendCallback): void; - close(): void; - onError(listener: (err: Error) => void): void; - offError(listener: (err: Error) => void): void; - onClose(listener: () => void): void; -} - -/** A socket's 'error' listeners, kept by the transport (the typed - * sockets have no removeListener): one socket listener dispatches to a - * snapshot, and an error nobody listens for throws, as an unhandled - * 'error' event does. */ -class ErrorListeners { - private listeners: ((err: Error) => void)[] = []; - add(listener: (err: Error) => void): void { - this.listeners.push(listener); - } - remove(listener: (err: Error) => void): void { - const index = this.listeners.indexOf(listener); - if (index !== -1) this.listeners.splice(index, 1); - } - emit(err: Error): void { - const snapshot = this.listeners.slice(); - if (snapshot.length === 0) throw err; - for (const listener of snapshot) listener(err); - } -} - -function addEol(buf: Buffer): string { - let msg = buf.toString(); - if (msg.length > 0 && msg[msg.length - 1] !== "\n") msg += "\n"; - return msg; -} - -class UdpTransport implements Transport { - createdAt = 0; - private readonly socket: dgram.Socket; - private readonly errors = new ErrorListeners(); - private resolvedAddress: string | undefined = undefined; - private resolvedAt = 0; - private readonly host: string | undefined; - private readonly port: number; - private readonly cacheDns: boolean; - private readonly cacheDnsTtl: number; - constructor(host: string | undefined, port: number, cacheDns: boolean, cacheDnsTtl: number) { - this.host = host; - this.port = port; - this.cacheDns = cacheDns; - this.cacheDnsTtl = cacheDnsTtl; - this.socket = dgram.createSocket("udp4"); - this.socket.unref(); - this.socket.on("error", (err) => this.errors.emit(err)); - } - send(buf: Buffer, callback: SendCallback): void { - if (!this.cacheDns) { - this.socket.send(buf, this.port, this.host || "127.0.0.1", callback); - return; - } - const now = Date.now(); - const cached = this.resolvedAddress; - if (cached === undefined || now - this.resolvedAt > this.cacheDnsTtl) { - dns.lookup(this.host ?? "localhost", (error, address) => { - if (error) { - callback(error); - return; - } - this.resolvedAddress = address; - this.resolvedAt = now; - this.socket.send(buf, this.port, address, callback); - }); - } else { - this.socket.send(buf, this.port, cached, callback); - } - } - close(): void { - this.socket.close(); - } - onError(listener: (err: Error) => void): void { - this.errors.add(listener); - } - offError(listener: (err: Error) => void): void { - this.errors.remove(listener); - } - onClose(listener: () => void): void { - this.socket.on("close", listener); - } -} - -class TcpTransport implements Transport { - createdAt = 0; - private readonly socket: net.Socket; - private readonly errors = new ErrorListeners(); - constructor(host: string | undefined, port: number) { - this.socket = net.connect(port, host ?? "localhost"); - this.socket.setKeepAlive(true); - this.socket.unref(); - this.socket.on("error", (err) => this.errors.emit(err)); - } - send(buf: Buffer, callback: SendCallback): void { - // The typed socket's write takes the data alone; the completion runs - // on the next tick. - this.socket.write(addEol(buf)); - process.nextTick(() => callback(null)); - } - close(): void { - this.socket.destroy(); - } - onError(listener: (err: Error) => void): void { - this.errors.add(listener); - } - offError(listener: (err: Error) => void): void { - this.errors.remove(listener); - } - onClose(listener: () => void): void { - this.socket.on("close", () => listener()); - } -} - -/** hot-shots reaches Unix datagram sockets through the optional - * `unix-dgram` addon; this entry serves programs without it. */ -function createUdsTransport(): Transport { - throw new Error( - "The library `unix_dgram`, needed for the uds protocol to work, is not installed. " + - "You need to pick another protocol to use hot-shots. " + - "See the hot-shots README for additional details.", - ); -} - -function createTransport(client: StatsD): Transport | null { - let transport: Transport | null = null; - const protocol = client.protocol; - try { - if (protocol === PROTOCOL_TCP) { - transport = new TcpTransport(client.host, client.port); - } else if (protocol === PROTOCOL_UDS) { - transport = createUdsTransport(); - } else if (protocol === PROTOCOL_UDP) { - transport = new UdpTransport(client.host, client.port, client.cacheDns, client.cacheDnsTtl); - } else { - throw new Error(`Unsupported protocol '${protocol}'`); - } - transport.createdAt = Date.now(); - } catch (e) { - const handler = client.errorHandler; - if (handler) { - handler(e as Error); - } else { - console.error(e); - } - } - return transport; -} - -// ── the client ─────────────────────────────────────────────────────────── - -class BufferHolder { - buffer = ""; -} - -/** What a child client inherits from its parent. */ -class ChildInit { - socket: Transport | null; - bufferHolder: BufferHolder; - dnsError: Error | undefined; - constructor(socket: Transport | null, bufferHolder: BufferHolder, dnsError: Error | undefined) { - this.socket = socket; - this.bufferHolder = bufferHolder; - this.dnsError = dnsError; - } -} - -export class StatsD { - protocol: string; - cacheDns: boolean; - cacheDnsTtl: number; - host: string | undefined; - port: number; - path: string | undefined; - prefix: string; - suffix: string; - tagPrefix: string; - tagSeparator: string; - mock: boolean; - globalTags: string[]; - telegraf: boolean; - maxBufferSize: number; - sampleRate: number; - bufferFlushInterval: number; - errorHandler: ((err: Error) => void) | undefined; - udsGracefulErrorHandling: boolean; - udsGracefulRestartRateLimit: number; - isChild: boolean; - closingFlushInterval: number; - mockBuffer: string[] | undefined = undefined; - messagesInFlight = 0; - CHECKS: DatadogChecks = { OK: 0, WARNING: 1, CRITICAL: 2, UNKNOWN: 3 }; - private bufferHolder: BufferHolder; - private transport: Transport | null = null; - private dnsError: Error | undefined = undefined; - private stopFlushInterval: (() => void) | undefined = undefined; - - constructor(options?: ClientOptions, child?: ChildInit) { - const o: ClientOptions = options ?? {}; - const protocol = o.protocol ? o.protocol.toLowerCase() : ""; - this.protocol = protocol ? protocol : PROTOCOL_UDP; - this.cacheDns = o.cacheDns === true; - this.cacheDnsTtl = o.cacheDnsTtl || CACHE_DNS_TTL_DEFAULT; - this.host = o.host || process.env["DD_AGENT_HOST"]; - this.port = o.port || parseInt(process.env["DD_DOGSTATSD_PORT"] ?? "", 10) || 8125; - this.path = o.path; - this.prefix = o.prefix || ""; - this.suffix = o.suffix || ""; - this.tagPrefix = o.tagPrefix || "#"; - this.tagSeparator = o.tagSeparator || ","; - this.mock = o.mock === true; - this.telegraf = o.telegraf || false; - const globalTags = o.globalTags; - this.globalTags = globalTags !== undefined ? formatTags(globalTags, this.telegraf) : []; - const entityId = process.env["DD_ENTITY_ID"]; - if (entityId) { - this.globalTags = this.globalTags.filter( - (item) => item.indexOf("dd.internal.entity_id:") !== 0, - ); - this.globalTags.push("dd.internal.entity_id:".concat(sanitizeTags(entityId, false))); - } - this.maxBufferSize = o.maxBufferSize || 0; - this.sampleRate = o.sampleRate || 1; - this.bufferFlushInterval = o.bufferFlushInterval || 1000; - this.bufferHolder = child !== undefined ? child.bufferHolder : new BufferHolder(); - this.errorHandler = o.errorHandler; - const graceful = o.udsGracefulErrorHandling; - this.udsGracefulErrorHandling = graceful === undefined ? true : graceful; - this.udsGracefulRestartRateLimit = - o.udsGracefulRestartRateLimit || UDS_DEFAULT_GRACEFUL_RESTART_LIMIT; - this.isChild = child !== undefined; - this.closingFlushInterval = o.closingFlushInterval || 50; - if (this.mock) this.mockBuffer = []; - if (child === undefined && this.maxBufferSize > 0) { - const handle = setInterval(() => this.onBufferFlushInterval(), this.bufferFlushInterval); - handle.unref(); - this.stopFlushInterval = () => clearInterval(handle); - } - if (child !== undefined) { - if (child.dnsError) this.dnsError = child.dnsError; - this.transport = child.socket; - } - if (!this.transport) this.transport = createTransport(this); - const handler = o.errorHandler; - const transport = this.transport; - if (transport !== null && child === undefined && handler) transport.onError(handler); - if (transport !== null && o.protocol === PROTOCOL_UDS) this.maybeAddUDSErrorHandler(); - } - - private maybeAddUDSErrorHandler(): void { - if (this.isChild || !this.udsGracefulErrorHandling) return; - const transport = this.transport; - if (transport !== null) transport.onError((err) => this.udsErrorHandler(err)); - } - - private udsErrorHandler(err: Error): void { - const code = Number((err as NodeJS.ErrnoException).code); - if (!UDS_ERROR_CODES.includes(-code)) return; - const socket = this.transport; - if (socket === null) return; - if (!socket.createdAt || Date.now() - socket.createdAt < this.udsGracefulRestartRateLimit) { - return; - } - const handler = this.errorHandler; - if (handler) socket.offError(handler); - const newSocket = createTransport(this); - if (newSocket === null) { - const errorMessage = "Could not replace UDS connection with new socket"; - if (handler) { - handler(new Error(errorMessage)); - } else { - console.error(errorMessage); - } - return; - } - socket.close(); - this.transport = newSocket; - this.maybeAddUDSErrorHandler(); - if (handler) { - newSocket.onError(handler); - } else { - newSocket.onError((error) => console.error(`hot-shots UDS error: ${String(error)}`)); - } - } - - // ── stats functions ── - - timing(stat: string | string[], time: number, sampleRate?: number, tags?: Tags, callback?: StatsCb): void; - timing(stat: string | string[], time: number, tags?: Tags, callback?: StatsCb): void; - timing(stat: string | string[], time: number, callback?: StatsCb): void; - timing(stat: string | string[], time: number, sampleRate?: number, callback?: StatsCb): void; - timing( - stat: string | string[], - time: number, - sampleRate?: SampleArg, - tags?: TagsArg, - callback?: StatsCb, - ): void { - this.sendAll(stat, time, "ms", sampleRate, tags, callback); - } - - increment(stat: string, tags?: Tags): void; - increment(stat: string | string[], value: number, sampleRate?: number, tags?: Tags, callback?: StatsCb): void; - increment(stat: string | string[], value: number, tags?: Tags, callback?: StatsCb): void; - increment(stat: string | string[], value: number, callback?: StatsCb): void; - increment(stat: string | string[], value: number, sampleRate?: number, callback?: StatsCb): void; - increment( - stat: string | string[], - value?: number | Tags, - sampleRate?: SampleArg, - tags?: TagsArg, - callback?: StatsCb, - ): void { - // A non-number second argument of a two-argument call is the tags. - let amount = 1; - let tagArg: TagsArg = tags; - if (typeof value === "number") { - amount = value; - } else if (sampleRate === undefined && tags === undefined && callback === undefined) { - tagArg = value; - } - this.sendAll(stat, amount, "c", sampleRate, tagArg, callback); - } - - decrement(stat: string, tags?: Tags): void; - decrement(stat: string | string[], value: number, sampleRate?: number, tags?: Tags, callback?: StatsCb): void; - decrement(stat: string | string[], value: number, tags?: Tags, callback?: StatsCb): void; - decrement(stat: string | string[], value: number, callback?: StatsCb): void; - decrement(stat: string | string[], value: number, sampleRate?: number, callback?: StatsCb): void; - decrement( - stat: string | string[], - value?: number | Tags, - sampleRate?: SampleArg, - tags?: TagsArg, - callback?: StatsCb, - ): void { - let amount = 1; - let tagArg: TagsArg = tags; - if (typeof value === "number") { - amount = value; - } else if (sampleRate === undefined && tags === undefined && callback === undefined) { - tagArg = value; - } - this.sendAll(stat, -amount, "c", sampleRate, tagArg, callback); - } - - histogram(stat: string | string[], value: number, sampleRate?: number, tags?: Tags, callback?: StatsCb): void; - histogram(stat: string | string[], value: number, tags?: Tags, callback?: StatsCb): void; - histogram(stat: string | string[], value: number, callback?: StatsCb): void; - histogram(stat: string | string[], value: number, sampleRate?: number, callback?: StatsCb): void; - histogram( - stat: string | string[], - value: number, - sampleRate?: SampleArg, - tags?: TagsArg, - callback?: StatsCb, - ): void { - this.sendAll(stat, value, "h", sampleRate, tags, callback); - } - - distribution(stat: string | string[], value: number, sampleRate?: number, tags?: Tags, callback?: StatsCb): void; - distribution(stat: string | string[], value: number, tags?: Tags, callback?: StatsCb): void; - distribution(stat: string | string[], value: number, callback?: StatsCb): void; - distribution(stat: string | string[], value: number, sampleRate?: number, callback?: StatsCb): void; - distribution( - stat: string | string[], - value: number, - sampleRate?: SampleArg, - tags?: TagsArg, - callback?: StatsCb, - ): void { - this.sendAll(stat, value, "d", sampleRate, tags, callback); - } - - gauge(stat: string | string[], value: number, sampleRate?: number, tags?: Tags, callback?: StatsCb): void; - gauge(stat: string | string[], value: number, tags?: Tags, callback?: StatsCb): void; - gauge(stat: string | string[], value: number, callback?: StatsCb): void; - gauge(stat: string | string[], value: number, sampleRate?: number, callback?: StatsCb): void; - gauge( - stat: string | string[], - value: number, - sampleRate?: SampleArg, - tags?: TagsArg, - callback?: StatsCb, - ): void { - this.sendAll(stat, value, "g", sampleRate, tags, callback); - } - - set(stat: string | string[], value: number | string, sampleRate?: number, tags?: Tags, callback?: StatsCb): void; - set(stat: string | string[], value: number | string, tags?: Tags, callback?: StatsCb): void; - set(stat: string | string[], value: number | string, callback?: StatsCb): void; - set(stat: string | string[], value: number | string, sampleRate?: number, callback?: StatsCb): void; - set( - stat: string | string[], - value: number | string, - sampleRate?: SampleArg, - tags?: TagsArg, - callback?: StatsCb, - ): void { - this.sendAll(stat, value, "s", sampleRate, tags, callback); - } - - unique(stat: string | string[], value: number | string, sampleRate?: number, tags?: Tags, callback?: StatsCb): void; - unique(stat: string | string[], value: number | string, tags?: Tags, callback?: StatsCb): void; - unique(stat: string | string[], value: number | string, callback?: StatsCb): void; - unique(stat: string | string[], value: number | string, sampleRate?: number, callback?: StatsCb): void; - unique( - stat: string | string[], - value: number | string, - sampleRate?: SampleArg, - tags?: TagsArg, - callback?: StatsCb, - ): void { - this.sendAll(stat, value, "s", sampleRate, tags, callback); - } - - check( - name: string, - status: DatadogChecksValues, - options?: CheckOptions, - tags?: Tags | StatsCb, - callback?: StatsCb, - ): void { - if (this.telegraf) { - const err = new Error("Not supported by Telegraf / InfluxDB"); - if (callback) return callback(err); - const handler = this.errorHandler; - if (handler) return handler(err); - throw err; - } - const check: string[] = ["_sc", this.prefix + name + this.suffix, String(status)]; - const metadata: CheckOptions = options ?? {}; - const happened = metadata.date_happened; - if (happened) { - const timestamp = formatDate(happened); - if (timestamp) check.push(`d:${timestamp}`); - } - if (metadata.hostname) check.push(`h:${metadata.hostname}`); - let mergedTags = this.globalTags; - if (tags !== undefined && typeof tags !== "function") { - mergedTags = overrideTags(mergedTags, tags, this.telegraf); - } - if (mergedTags.length > 0) check.push(`#${mergedTags.join(",")}`); - if (metadata.message) check.push(`m:${metadata.message}`); - let cb = callback; - if (typeof tags === "function" && callback === undefined) cb = tags; - this.sendRaw(check.join("|"), cb); - } - - event( - title: string, - text?: string, - options?: EventOptions, - tags?: Tags | StatsCb, - callback?: StatsCb, - ): void { - if (this.telegraf) { - const err = new Error("Not supported by Telegraf / InfluxDB"); - if (callback) return callback(err); - const handler = this.errorHandler; - if (handler) return handler(err); - throw err; - } - const msgTitle = String(title ? title : ""); - let msgText = String(text ? text : msgTitle); - msgText = msgText.replace(/\n/g, "\\n"); - let message = `_e{${msgTitle.length},${msgText.length}}:${msgTitle}|${msgText}`; - if (options) { - const happened = options.date_happened; - if (happened) { - const timestamp = formatDate(happened); - if (timestamp) message += `|d:${timestamp}`; - } - if (options.hostname) message += `|h:${options.hostname}`; - if (options.aggregation_key) message += `|k:${options.aggregation_key}`; - if (options.priority) message += `|p:${options.priority}`; - if (options.source_type_name) message += `|s:${options.source_type_name}`; - if (options.alert_type) message += `|t:${options.alert_type}`; - } - let cb = callback; - let tagList: Tags | undefined = undefined; - if (typeof tags === "function") { - if (callback === undefined) cb = tags; - } else { - tagList = tags; - } - this.send(message, tagList, cb); - } - - // ── sending ── - - private sendAll( - stat: string | string[], - value: number | string, - type: string, - sampleRateArg: SampleArg, - tagsArg: TagsArg, - callbackArg: StatsCb | undefined, - ): void { - let sampleRate: number | undefined = undefined; - let tags: Tags | undefined = undefined; - let callback: StatsCb | undefined = callbackArg; - if (typeof sampleRateArg === "number") { - sampleRate = sampleRateArg; - if (typeof tagsArg === "function") { - callback = tagsArg; - } else { - tags = tagsArg; - } - } else if (sampleRateArg !== undefined) { - // A non-number sample rate shifts into the tags, and the tags - // argument into the callback. - if (typeof sampleRateArg === "function") { - callback = sampleRateArg; - } else { - tags = sampleRateArg; - callback = typeof tagsArg === "function" ? tagsArg : undefined; - } - } else if (typeof tagsArg === "function") { - callback = tagsArg; - } else { - tags = tagsArg; - } - - if (!Array.isArray(stat)) { - this.sendStat(stat, value, type, sampleRate, tags, callback); - return; - } - let completed = 0; - let calledback = false; - let sentBytes = 0; - const count = stat.length; - const onSend: StatsCb = (error, bytes) => { - completed += 1; - if (calledback) return; - if (error) { - if (callback !== undefined) { - calledback = true; - callback(error); - } else { - const handler = this.errorHandler; - if (handler) { - calledback = true; - handler(error); - } - } - return; - } - if (bytes) sentBytes += bytes as number; - if (completed === count && callback !== undefined) callback(null, sentBytes); - }; - for (const item of stat) this.sendStat(item, value, type, sampleRate, tags, onSend); - } - - private sendStat( - stat: string, - value: number | string, - type: string, - sampleRateArg: number | undefined, - tags: Tags | undefined, - callback: StatsCb | undefined, - ): void { - let message = `${this.prefix + stat + this.suffix}:${value}|${type}`; - const sampleRate = sampleRateArg || this.sampleRate; - if (sampleRate && sampleRate < 1) { - if (Math.random() < sampleRate) { - message += `|@${sampleRate}`; - } else { - if (callback) callback(undefined); - return; - } - } - this.send(message, tags, callback); - } - - private send(message: string, tags: Tags | undefined, callback: StatsCb | undefined): void { - let mergedTags = this.globalTags; - if (tags !== undefined) mergedTags = overrideTags(mergedTags, tags, this.telegraf); - if (mergedTags.length > 0) { - if (this.telegraf) { - const parts = message.split(":"); - message = `${parts[0]!},${mergedTags.join(",").replace(/:/g, "=")}:${parts.slice(1).join(":")}`; - } else { - message += `|${this.tagPrefix}${mergedTags.join(this.tagSeparator)}`; - } - } - this.sendRaw(message, callback); - } - - /** hot-shots' `_send`: the message with its tags. */ - private sendRaw(message: string, callback: StatsCb | undefined): void { - const dnsError = this.dnsError; - if (dnsError) { - if (callback) return callback(dnsError); - const handler = this.errorHandler; - if (handler) return handler(dnsError); - throw dnsError; - } - if (!this.mock) { - if (this.maxBufferSize === 0) { - this.sendMessage(message, callback); - } else { - this.enqueue(message, callback); - } - } else { - const buffer = this.mockBuffer; - if (buffer !== undefined) buffer.push(message); - if (callback) callback(null, 0); - } - } - - private enqueue(message: string, callback: StatsCb | undefined): void { - message += "\n"; - if (this.bufferHolder.buffer.length + message.length > this.maxBufferSize) { - this.flushQueue(callback); - this.bufferHolder.buffer += message; - } else { - this.bufferHolder.buffer += message; - if (callback) callback(null); - } - } - - private flushQueue(callback?: StatsCb): void { - this.sendMessage(this.bufferHolder.buffer, callback); - this.bufferHolder.buffer = ""; - } - - private sendMessage(message: string, callback: StatsCb | undefined): void { - if (message === "") { - if (callback) callback(null); - return; - } - const socketWasMissing = this.transport === null; - if (socketWasMissing && this.protocol === PROTOCOL_UDS) { - this.transport = createTransport(this); - if (this.transport !== null) this.maybeAddUDSErrorHandler(); - } - const socket = this.transport; - if (socketWasMissing || socket === null) { - const error = new Error("Socket not created properly. Check previous errors for details."); - if (callback) return callback(error); - const handler = this.errorHandler; - if (handler) return handler(error); - console.error(String(error)); - return; - } - const handleCallback = (err: Error | null, bytes?: number): void => { - this.messagesInFlight--; - let errFormatted: Error | null = null; - if (err) { - errFormatted = errorWithCode( - `Error sending hot-shots message: ${String(err)}`, - (err as NodeJS.ErrnoException).code, - ); - if (this.protocol === PROTOCOL_UDS && (callback || this.errorHandler)) { - this.udsErrorHandler(err); - } - } - if (callback) { - callback(errFormatted, bytes); - } else if (errFormatted) { - const handler = this.errorHandler; - if (handler) { - handler(errFormatted); - } else { - console.error(String(errFormatted)); - } - } - }; - try { - this.messagesInFlight++; - socket.send(Buffer.from(message), handleCallback); - } catch (err) { - handleCallback(err as Error); - } - } - - private onBufferFlushInterval(): void { - this.flushQueue(); - } - - close(callback?: (error?: Error) => void): void { - const stop = this.stopFlushInterval; - if (stop !== undefined) stop(); - this.flushQueue((err) => { - if (err) { - if (callback) return callback(err); - console.error(err); - return; - } - let intervalAttempts = 0; - const waitForMessages = setInterval(() => { - intervalAttempts++; - if (intervalAttempts > 10) { - console.log("hot-shots could not clear out messages in flight but closing anyways"); - this.messagesInFlight = 0; - } - if (this.messagesInFlight <= 0) { - clearInterval(waitForMessages); - this.closeSocket(callback); - } - }, this.closingFlushInterval); - }); - } - - private closeSocket(callback: ((error?: Error) => void) | undefined): void { - let handledError = false; - const handleErr = (err: unknown): void => { - const errMessage = `Error closing hot-shots socket: ${String(err)}`; - if (handledError) { - console.error(errMessage); - } else { - handledError = true; - if (callback) { - callback(new Error(errMessage)); - } else { - const handler = this.errorHandler; - if (handler) { - handler(new Error(errMessage)); - } else { - console.error(errMessage); - } - } - } - }; - const socket = this.transport; - if (socket === null) { - handleErr(new TypeError("Cannot read properties of null (reading 'removeListener')")); - return; - } - const handler = this.errorHandler; - if (handler) socket.offError(handler); - socket.onError(handleErr); - if (callback) { - socket.onClose(() => { - if (!handledError) callback(undefined); - }); - } - try { - socket.close(); - } catch (err) { - handleErr(err); - } - } - - childClient(options?: ChildClientOptions): StatsD { - const o: ChildClientOptions = options ?? {}; - const childTags = o.globalTags; - return new StatsD( - { - isChild: true, - errorHandler: o.errorHandler || this.errorHandler, - host: this.host, - port: this.port, - tagPrefix: this.tagPrefix, - tagSeparator: this.tagSeparator, - prefix: (o.prefix || "") + this.prefix, - suffix: this.suffix + (o.suffix || ""), - globalize: false, - mock: this.mock, - globalTags: - childTags !== undefined - ? overrideTags(this.globalTags, childTags, this.telegraf) - : this.globalTags, - maxBufferSize: this.maxBufferSize, - bufferFlushInterval: this.bufferFlushInterval, - telegraf: this.telegraf, - protocol: this.protocol === "tcp" ? "tcp" : this.protocol === "uds" ? "uds" : "udp", - closingFlushInterval: this.closingFlushInterval, - }, - new ChildInit(this.transport, this.bufferHolder, this.dnsError), - ); - } -} - -export default StatsD; +export * from "./client.ts"; +export { default } from "./client.ts"; diff --git a/packages/compiler/src/backend/llvm/lib-datagram.ts b/packages/compiler/src/backend/llvm/lib-datagram.ts index 184fdb99e..6e10e9c62 100644 --- a/packages/compiler/src/backend/llvm/lib-datagram.ts +++ b/packages/compiler/src/backend/llvm/lib-datagram.ts @@ -105,6 +105,10 @@ export function emitDatagramLibCall(host: LlvmEmitterContext, e: LibCallExpr): L "dgram.sendStr": "scr_dgram_send_str", "dgram.sendBytes": "scr_dgram_send_bytes", "dgram.sendChk": "scr_dgram_send_chk", + "dgram.unixSocket": "scr_dgram_unix_socket", + "dgram.unixConnect": "scr_dgram_unix_connect", + "dgram.unixSend": "scr_dgram_unix_send", + "dgram.unixClose": "scr_dgram_unix_close", "dgram.ref": "scr_dgram_ref", "dgram.unref": "scr_dgram_unref", "dgram.setBroadcast": "scr_dgram_set_broadcast", diff --git a/packages/compiler/src/frontend/lowering/lower-calls.ts b/packages/compiler/src/frontend/lowering/lower-calls.ts index 53971c7b0..3c04924c0 100644 --- a/packages/compiler/src/frontend/lowering/lower-calls.ts +++ b/packages/compiler/src/frontend/lowering/lower-calls.ts @@ -45,6 +45,7 @@ import { errorToStringCall } from "./error-methods.js"; import { lowerClassPrototypeAssign, lowerClassPrototypeDescriptors } from "./class-prototypes.js"; import { BYTES_ELEMENT_NAME, + BYTES_U8, BIGINT_T, BOOL, CAUGHT, @@ -53,6 +54,7 @@ import { F64, type IrExpr, type IrFunction, + type IrLibFn, type IrLocal, type IrParam, type IrStmt, @@ -76,6 +78,7 @@ import { } from "../../ir/ir.js"; import { lowerProtocolMethodCall } from "./class-protocol-dispatch.js"; import { isJsSourceFile, locOf, requireSpecOf } from "../program.js"; +import { isPackageShimFile, isShimIntrinsicName } from "../package-shims.js"; import { lowerJsonRequireCall } from "./json-require.js"; import { lowerAssertInstanceCall } from "./assert-instances.js"; import { @@ -292,6 +295,12 @@ export function lowerCall(lowerer: Lowerer, expr: ts.CallExpression): IrExpr { const codeGeneration = codeGenerationCalleeOf(lowerer, expr.expression); if (codeGeneration !== null) return lowerCodeGenerationCall(lowerer, expr, codeGeneration); } + // A package shim's runtime intrinsic (a bodiless `declare function` + // named scriptcUnixDgram* in a shipped shim): a direct runtime call. + if (ts.isIdentifier(expr.expression) && isShimIntrinsicName(expr.expression.text)) { + const intrinsic = lowerShimIntrinsicCall(lowerer, expr, expr.expression, loc); + if (intrinsic) return intrinsic; + } // Inline require() of a program module (cjs-inline-require.ts): a // member call over its export table resolves like a binding's member // call after the module's %init at this position; the require itself @@ -10958,3 +10967,37 @@ function fenceClassCopyIntoFreshObject( `Object.assign copying a '${lowerer.fmt(rawSource.type)}' instance into a fresh object typed as the class (the copy is a plain object holding the instance's own fields, not an instance — build the object literal from the fields you need)`, ); } + +/** The runtime calls package shims declare as bodiless functions. */ +const SHIM_INTRINSICS: ReadonlyMap = new Map< + string, + { fn: IrLibFn; args: IrType[]; ret: IrType } +>([ + ["scriptcUnixDgramSocket", { fn: "dgram.unixSocket", args: [], ret: F64 }], + ["scriptcUnixDgramConnect", { fn: "dgram.unixConnect", args: [F64, STRING], ret: F64 }], + ["scriptcUnixDgramSend", { fn: "dgram.unixSend", args: [F64, BYTES_U8], ret: F64 }], + ["scriptcUnixDgramClose", { fn: "dgram.unixClose", args: [F64], ret: F64 }], +]); + +function lowerShimIntrinsicCall( + lowerer: Lowerer, + expr: ts.CallExpression, + callee: ts.Identifier, + loc: SrcLoc, +): IrExpr | null { + const intrinsic = SHIM_INTRINSICS.get(callee.text); + if (intrinsic === undefined) return null; + const decl = lowerer.checker.getSymbolAtLocation(callee)?.valueDeclaration; + if ( + decl === undefined || + !ts.isFunctionDeclaration(decl) || + decl.body || + !isPackageShimFile(ts.getSourceFileOfNode(decl).fileName) || + expr.arguments.nodes.length !== intrinsic.args.length + ) + return null; + const args = expr.arguments.nodes.map((arg, i) => + lowerer.lowerExprExpecting(arg, intrinsic.args[i]!), + ); + return { kind: "libCall", fn: intrinsic.fn, args, type: intrinsic.ret, loc }; +} diff --git a/packages/compiler/src/frontend/lowering/lower-namespaces.ts b/packages/compiler/src/frontend/lowering/lower-namespaces.ts index 19123dae3..ea844ab1e 100644 --- a/packages/compiler/src/frontend/lowering/lower-namespaces.ts +++ b/packages/compiler/src/frontend/lowering/lower-namespaces.ts @@ -38,7 +38,7 @@ * calls). */ import { positionTag, type PositionTagOwner } from "./structural-positions.js"; import * as ts from "../ts7/adapter.js"; -import { packageShimFor } from "../package-shims.js"; +import { isPackageShimFile, isShimIntrinsicName, packageShimFor } from "../package-shims.js"; import type { Lowerer } from "./lowerer.js"; import { boundIdentifiersOf } from "./lowerer.js"; import type { FileParts } from "./lower-modules.js"; @@ -654,6 +654,12 @@ export function ambientUndefinedFnSymbolOf( const sym = lowerer.resolveValueSymbol(ident); if (!sym || !(sym.flags & ts.SymbolFlags.Function)) return null; if (lowerer.isStdlibSymbol(sym)) return null; + // A package shim's runtime intrinsic (lower-calls.ts) is defined. + if ( + isShimIntrinsicName(ident.text) && + sym.declarations.some((d) => isPackageShimFile(ts.getSourceFileOfNode(d).fileName)) + ) + return null; let sawFn = false; for (const d of sym.declarations) { if (ts.isInterfaceDeclaration(d) || ts.isTypeAliasDeclaration(d)) continue; // type-world merge partners diff --git a/packages/compiler/src/frontend/package-shims.ts b/packages/compiler/src/frontend/package-shims.ts index f9cf1deb6..3af7ebf59 100644 --- a/packages/compiler/src/frontend/package-shims.ts +++ b/packages/compiler/src/frontend/package-shims.ts @@ -65,9 +65,14 @@ interface ImplementationShim { /** The package's declaration entry relative to its directory, when it * is not `index.d.ts`. */ typesFile?: string; + /** An alternative entry for this installation (relative to the shim + * root), or undefined for the version's own entry. */ + variant?: (pkgDir: string) => string | undefined; + /** Every variant entry, for the guard test that compiles each one. */ + variants?: readonly string[]; } -const IMPLEMENTATION_SHIMS: ReadonlyMap = new Map([ +const IMPLEMENTATION_SHIMS: ReadonlyMap = new Map([ [ "fs-extra", { @@ -100,6 +105,13 @@ const IMPLEMENTATION_SHIMS: ReadonlyMap = new Map([ mainFile: "index.js", typesFile: "types.d.ts", versions: new Map([[8, "hot-shots/v8.ts"]]), + // hot-shots' uds protocol loads the optional unix-dgram addon: where + // it is installed, the entry with the native Unix datagram transport. + variant: (pkgDir: string) => + installedPackageDir(join(pkgDir, "index.js"), "unix-dgram") === null + ? undefined + : "hot-shots/v8-unix-dgram.ts", + variants: ["hot-shots/v8-unix-dgram.ts"], }, ], [ @@ -164,8 +176,10 @@ export function throughBundledShimForwarder(resolved: string): string { * the guard test that compiles each one. */ export function packageShimEntries(): string[] { const entries = new Set(); - for (const shim of IMPLEMENTATION_SHIMS.values()) + for (const shim of IMPLEMENTATION_SHIMS.values()) { for (const entry of shim.versions.values()) entries.add(join(packageShimRoot(), entry)); + for (const entry of shim.variants ?? []) entries.add(join(packageShimRoot(), entry)); + } return [...entries].sort(); } @@ -263,7 +277,8 @@ function shimEntryForPackageDir(pkgDir: string, expectedName: string | null): st if (shim === undefined) return null; const major = Number(/^(\d+)\./.exec(pkg.version)?.[1] ?? NaN); const entry = shim.versions.get(major); - return entry === undefined ? null : join(packageShimRoot(), entry); + if (entry === undefined) return null; + return join(packageShimRoot(), shim.variant?.(pkgDir) ?? entry); } /** The shim module `specifier` resolves to from `fromFile`, or null. Only @@ -355,6 +370,12 @@ export function isPackageShimFile(fileName: string): boolean { return fileName.startsWith(root.endsWith("/") ? root : `${root}/`); } +/** True for a name a shipped shim may declare as a bodiless runtime + * intrinsic (lower-calls.ts lowers its calls to runtime calls). */ +export function isShimIntrinsicName(name: string): boolean { + return name.startsWith("scriptcUnixDgram"); +} + /** Resets per-compile caches. */ export function clearPackageShimCaches(): void { installedCache.clear(); diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index fcdbe8694..29385db37 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -1528,6 +1528,13 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ }, "dgram.sendStr": { argTypes: [DGRAMSOCK_T, STRING, F64, STRING], result: VOID }, "dgram.sendBytes": { argTypes: [DGRAMSOCK_T, BYTES_U8, F64, STRING], result: VOID }, + /** Connected AF_UNIX datagram sockets for the hot-shots package shim + * (the unix-dgram addon's surface): fd or result >= 0, -errno on + * failure; send answers 1 when the socket would block. */ + "dgram.unixSocket": { argTypes: [], result: F64 }, + "dgram.unixConnect": { argTypes: [F64, STRING], result: F64 }, + "dgram.unixSend": { argTypes: [F64, BYTES_U8], result: F64 }, + "dgram.unixClose": { argTypes: [F64], result: F64 }, /** The send argument-validation ladder over dyn arguments (Node's * signature shuffle: slice bounds, list/type contracts, port/address * validation, connected-state errors), then the send: lists, connected diff --git a/packages/runtime/src/scr_dgram.c b/packages/runtime/src/scr_dgram.c index bb14d063e..8d1c7b058 100644 --- a/packages/runtime/src/scr_dgram.c +++ b/packages/runtime/src/scr_dgram.c @@ -2158,3 +2158,73 @@ void scr_dgram_send_chk(ScrDgramSocket *s, const ScrDyn *buffer, const ScrDyn *a * it removes a 'message' listener. Defined last so the functions above * keep their emission order. */ void scr_dgram_rearm(ScrDgramSocket *s) { scr_dgram_update_read(s); } + +/* ── Unix datagram sockets for the hot-shots package shim ────────────── + * The unix-dgram addon's surface, which hot-shots' uds transport uses: a + * non-blocking AF_UNIX datagram socket connected to a path. Each call + * answers its result (fd, 0) or a negative errno, as the addon's binding + * does; send answers 1 when the socket would block (the addon's + * "congestion"). Defined after the functions above so their emission + * order is unchanged. */ +#ifndef _WIN32 +#include +#endif + +double scr_dgram_unix_socket(void) { +#ifdef _WIN32 + return -(double)ENOSYS; +#else + int fd = socket(AF_UNIX, SOCK_DGRAM, 0); + if (fd < 0) return -(double)errno; + int flags = fcntl(fd, F_GETFL, 0); + if (flags >= 0) (void)fcntl(fd, F_SETFL, flags | O_NONBLOCK); + (void)fcntl(fd, F_SETFD, FD_CLOEXEC); + return (double)fd; +#endif +} + +double scr_dgram_unix_connect(double fd, ScrStr *path) { +#ifdef _WIN32 + (void)fd; + (void)path; + return -(double)ENOSYS; +#else + struct sockaddr_un addr; + memset(&addr, 0, sizeof addr); + addr.sun_family = AF_UNIX; + char *owned; + const char *bytes = scr_str_utf8(path, &owned); + size_t n = strlen(bytes); + if (n > sizeof addr.sun_path - 1) n = sizeof addr.sun_path - 1; + memcpy(addr.sun_path, bytes, n); + scr_str_utf8_end(owned); + if (connect((int)fd, (struct sockaddr *)&addr, sizeof addr) != 0) return -(double)errno; + return 0; +#endif +} + +double scr_dgram_unix_send(double fd, ScrBytes *data) { +#ifdef _WIN32 + (void)fd; + (void)data; + return -(double)ENOSYS; +#else + SCR_BYTES_SNAPSHOT(data); + ssize_t sent; + do { + sent = send((int)fd, data->data, data->len, 0); + } while (sent < 0 && errno == EINTR); + if (sent >= 0) return 0; + if (errno == EAGAIN || errno == EWOULDBLOCK) return 1; + return -(double)errno; +#endif +} + +double scr_dgram_unix_close(double fd) { +#ifdef _WIN32 + (void)fd; + return -(double)ENOSYS; +#else + return close((int)fd) != 0 ? -(double)errno : 0; +#endif +} From eef238eb9473ac937c5cf9ed1d1af73cc8930f8c Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Wed, 21 Oct 2026 21:59:20 -0700 Subject: [PATCH 016/197] Pin chalk builder values and buffered hot-shots against Node A callable class instance in a narrower function slot crosses the checked function boundary. New npm cases: chalk5-builder, hot-shots-buffered. Co-Authored-By: Claude Opus 5.5 --- .../compiler/src/frontend/lowering/lowerer.ts | 16 + .../compiler/test/package-shim-guard.test.ts | 3 + .../fixtures/npm/cases/chalk5-builder/main.ts | 45 +++ .../chalk5-builder/node_modules/chalk/license | 9 + .../node_modules/chalk/package.json | 83 +++++ .../node_modules/chalk/source/index.d.ts | 325 ++++++++++++++++++ .../node_modules/chalk/source/index.js | 225 ++++++++++++ .../node_modules/chalk/source/utilities.js | 33 ++ .../source/vendor/ansi-styles/index.d.ts | 236 +++++++++++++ .../chalk/source/vendor/ansi-styles/index.js | 223 ++++++++++++ .../source/vendor/supports-color/browser.d.ts | 1 + .../source/vendor/supports-color/browser.js | 34 ++ .../source/vendor/supports-color/index.d.ts | 55 +++ .../source/vendor/supports-color/index.js | 190 ++++++++++ .../npm/cases/chalk5-builder/package.json | 1 + .../npm/cases/hot-shots-buffered/main.ts | 80 +++++ .../npm/cases/hot-shots-buffered/package.json | 1 + 17 files changed, 1560 insertions(+) create mode 100644 tests/fixtures/npm/cases/chalk5-builder/main.ts create mode 100644 tests/fixtures/npm/cases/chalk5-builder/node_modules/chalk/license create mode 100644 tests/fixtures/npm/cases/chalk5-builder/node_modules/chalk/package.json create mode 100644 tests/fixtures/npm/cases/chalk5-builder/node_modules/chalk/source/index.d.ts create mode 100644 tests/fixtures/npm/cases/chalk5-builder/node_modules/chalk/source/index.js create mode 100644 tests/fixtures/npm/cases/chalk5-builder/node_modules/chalk/source/utilities.js create mode 100644 tests/fixtures/npm/cases/chalk5-builder/node_modules/chalk/source/vendor/ansi-styles/index.d.ts create mode 100644 tests/fixtures/npm/cases/chalk5-builder/node_modules/chalk/source/vendor/ansi-styles/index.js create mode 100644 tests/fixtures/npm/cases/chalk5-builder/node_modules/chalk/source/vendor/supports-color/browser.d.ts create mode 100644 tests/fixtures/npm/cases/chalk5-builder/node_modules/chalk/source/vendor/supports-color/browser.js create mode 100644 tests/fixtures/npm/cases/chalk5-builder/node_modules/chalk/source/vendor/supports-color/index.d.ts create mode 100644 tests/fixtures/npm/cases/chalk5-builder/node_modules/chalk/source/vendor/supports-color/index.js create mode 100644 tests/fixtures/npm/cases/chalk5-builder/package.json create mode 100644 tests/fixtures/npm/cases/hot-shots-buffered/main.ts create mode 100644 tests/fixtures/npm/cases/hot-shots-buffered/package.json diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index b1867be32..ac1a778cf 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -230,6 +230,7 @@ import { arrayOf, BOOL, canAdaptDynFuncTo, + canBoxFuncIntoDyn, canDynCheckTo, canConvertToDyn, DYN, @@ -9642,6 +9643,21 @@ export class Lowerer { if (expected.kind === "func" && (expr.type.kind === "record" || expr.type.kind === "object")) { const un = this.hybridCallUnwrap(expr); if (un !== expr && typeEquals(un.type, expected)) return un; + // A callable class instance (the chalk shim's variadic builder) in a + // narrower function slot: its call closure crosses the checked + // function boundary, which adapts the arity. + if ( + un !== expr && + expr.type.kind === "object" && + un.type.kind === "func" && + canBoxFuncIntoDyn( + un.type, + (id) => this.shapes.get(id), + (id) => this.unions.get(id), + ) + ) { + return this.coerceToExpected({ kind: "dynFrom", value: un, type: DYN, loc: un.loc }, expected); + } } // A zero-param function whose RETURN is a wider record array than the // slot's (`getRoutes: () => cachedRoutes` against `() => Narrow[]`): diff --git a/packages/compiler/test/package-shim-guard.test.ts b/packages/compiler/test/package-shim-guard.test.ts index ff01acbb8..a8620c1ce 100644 --- a/packages/compiler/test/package-shim-guard.test.ts +++ b/packages/compiler/test/package-shim-guard.test.ts @@ -49,7 +49,10 @@ describe("package shims compile without refusals", () => { const entries = packageShimEntries(); test("the registry lists its shim modules", () => { expect(entries.map((e) => e.slice(shimRoot.length + 1))).toEqual([ + "chalk/v5.ts", "fs-extra/index.ts", + "hot-shots/v8-unix-dgram.ts", + "hot-shots/v8.ts", "undici/v5.ts", "undici/v7.ts", ]); diff --git a/tests/fixtures/npm/cases/chalk5-builder/main.ts b/tests/fixtures/npm/cases/chalk5-builder/main.ts new file mode 100644 index 000000000..8f2623717 --- /dev/null +++ b/tests/fixtures/npm/cases/chalk5-builder/main.ts @@ -0,0 +1,45 @@ +// chalk 5 builders used as values (the package compiles from scriptc's +// built-in shim): builders in function slots and arrays, styles picked at +// run time, visible at level 0, chalkStderr, and the root's own call. +import chalk, { Chalk, chalkStderr, type ChalkInstance, type ForegroundColorName } from "chalk"; + +const show = (label: string, value: string): void => { + console.log(label + " " + JSON.stringify(value)); +}; + +const c = new Chalk({ level: 1 }); +const paint: (text: string) => string = c.green; +show("slot", paint("go")); +const styles: ChalkInstance[] = [c.red, c.bold, c.underline.blue]; +show("array", styles.map((style) => style("x")).join("|")); +const byName = (name: ForegroundColorName): ChalkInstance => { + switch (name) { + case "red": + return c.red; + case "green": + return c.green; + default: + return c.white; + } +}; +show("picked", byName("red")("r") + byName("green")("g") + byName("cyan")("w")); +let chain: ChalkInstance = c; +for (const step of ["bold", "italic", "yellow"]) { + chain = step === "bold" ? chain.bold : step === "italic" ? chain.italic : chain.yellow; +} +show("built", chain("chain")); +show("cached", c.red === c.red ? "same" : "different"); +show("nested close", c.red("a" + c.blue("b") + "c" + c.red("d") + "e")); +show("bg nested", c.bgRed("a" + c.bgBlue("b") + "c")); + +const off = new Chalk({ level: 0 }); +show("visible off", off.visible("v") + "|" + off.red.visible("v") + "|" + off.red("plain")); +show("root call", c("a", 1, true) + "|" + c() + "|" + c(undefined) + "|" + c(null, "n")); +show("builder args", c.red(undefined) + "|" + c.red(null) + "|" + c.red(1, 2) + "|" + c.red()); +show("multiline", c.cyan("one\r\ntwo\nthree")); +show("stderr level", String(chalkStderr.level >= 0)); +show("rgb levels", [1, 2, 3].map((level) => new Chalk({ level: level as 1 | 2 | 3 }).rgb(255, 136, 0)("o")).join("|")); +show("hex bg", new Chalk({ level: 2 }).bgHex("#abcdef")("h") + new Chalk({ level: 1 }).bgHex("123")("i")); +c.level = 0; +show("after level 0", paint("go") + "|" + c.red("r")); +show("default red", chalk.level >= 0 ? "ok" : "bad"); diff --git a/tests/fixtures/npm/cases/chalk5-builder/node_modules/chalk/license b/tests/fixtures/npm/cases/chalk5-builder/node_modules/chalk/license new file mode 100644 index 000000000..fa7ceba3e --- /dev/null +++ b/tests/fixtures/npm/cases/chalk5-builder/node_modules/chalk/license @@ -0,0 +1,9 @@ +MIT License + +Copyright (c) Sindre Sorhus (https://sindresorhus.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/tests/fixtures/npm/cases/chalk5-builder/node_modules/chalk/package.json b/tests/fixtures/npm/cases/chalk5-builder/node_modules/chalk/package.json new file mode 100644 index 000000000..c9e0dc52b --- /dev/null +++ b/tests/fixtures/npm/cases/chalk5-builder/node_modules/chalk/package.json @@ -0,0 +1,83 @@ +{ + "name": "chalk", + "version": "5.6.2", + "description": "Terminal string styling done right", + "license": "MIT", + "repository": "chalk/chalk", + "funding": "https://github.com/chalk/chalk?sponsor=1", + "type": "module", + "main": "./source/index.js", + "exports": "./source/index.js", + "imports": { + "#ansi-styles": "./source/vendor/ansi-styles/index.js", + "#supports-color": { + "node": "./source/vendor/supports-color/index.js", + "default": "./source/vendor/supports-color/browser.js" + } + }, + "types": "./source/index.d.ts", + "sideEffects": false, + "engines": { + "node": "^12.17.0 || ^14.13 || >=16.0.0" + }, + "scripts": { + "test": "xo && c8 ava && tsd", + "bench": "matcha benchmark.js" + }, + "files": [ + "source", + "!source/index.test-d.ts" + ], + "keywords": [ + "color", + "colour", + "colors", + "terminal", + "console", + "cli", + "string", + "ansi", + "style", + "styles", + "tty", + "formatting", + "rgb", + "256", + "shell", + "xterm", + "log", + "logging", + "command-line", + "text" + ], + "devDependencies": { + "@types/node": "^16.11.10", + "ava": "^3.15.0", + "c8": "^7.10.0", + "color-convert": "^2.0.1", + "execa": "^6.0.0", + "log-update": "^5.0.0", + "matcha": "^0.7.0", + "tsd": "^0.19.0", + "xo": "^0.57.0", + "yoctodelay": "^2.0.0" + }, + "xo": { + "rules": { + "unicorn/prefer-string-slice": "off", + "@typescript-eslint/consistent-type-imports": "off", + "@typescript-eslint/consistent-type-exports": "off", + "@typescript-eslint/consistent-type-definitions": "off", + "unicorn/expiring-todo-comments": "off" + } + }, + "c8": { + "reporter": [ + "text", + "lcov" + ], + "exclude": [ + "source/vendor" + ] + } +} diff --git a/tests/fixtures/npm/cases/chalk5-builder/node_modules/chalk/source/index.d.ts b/tests/fixtures/npm/cases/chalk5-builder/node_modules/chalk/source/index.d.ts new file mode 100644 index 000000000..8295d9252 --- /dev/null +++ b/tests/fixtures/npm/cases/chalk5-builder/node_modules/chalk/source/index.d.ts @@ -0,0 +1,325 @@ +// TODO: Make it this when TS suports that. +// import {ModifierName, ForegroundColor, BackgroundColor, ColorName} from '#ansi-styles'; +// import {ColorInfo, ColorSupportLevel} from '#supports-color'; +import { + ModifierName, + ForegroundColorName, + BackgroundColorName, + ColorName, +} from './vendor/ansi-styles/index.js'; +import {ColorInfo, ColorSupportLevel} from './vendor/supports-color/index.js'; + +export interface Options { + /** + Specify the color support for Chalk. + + By default, color support is automatically detected based on the environment. + + Levels: + - `0` - All colors disabled. + - `1` - Basic 16 colors support. + - `2` - ANSI 256 colors support. + - `3` - Truecolor 16 million colors support. + */ + readonly level?: ColorSupportLevel; +} + +/** +Return a new Chalk instance. +*/ +export const Chalk: new (options?: Options) => ChalkInstance; // eslint-disable-line @typescript-eslint/naming-convention + +export interface ChalkInstance { + (...text: unknown[]): string; + + /** + The color support for Chalk. + + By default, color support is automatically detected based on the environment. + + Levels: + - `0` - All colors disabled. + - `1` - Basic 16 colors support. + - `2` - ANSI 256 colors support. + - `3` - Truecolor 16 million colors support. + */ + level: ColorSupportLevel; + + /** + Use RGB values to set text color. + + @example + ``` + import chalk from 'chalk'; + + chalk.rgb(222, 173, 237); + ``` + */ + rgb: (red: number, green: number, blue: number) => this; + + /** + Use HEX value to set text color. + + @param color - Hexadecimal value representing the desired color. + + @example + ``` + import chalk from 'chalk'; + + chalk.hex('#DEADED'); + ``` + */ + hex: (color: string) => this; + + /** + Use an [8-bit unsigned number](https://en.wikipedia.org/wiki/ANSI_escape_code#8-bit) to set text color. + + @example + ``` + import chalk from 'chalk'; + + chalk.ansi256(201); + ``` + */ + ansi256: (index: number) => this; + + /** + Use RGB values to set background color. + + @example + ``` + import chalk from 'chalk'; + + chalk.bgRgb(222, 173, 237); + ``` + */ + bgRgb: (red: number, green: number, blue: number) => this; + + /** + Use HEX value to set background color. + + @param color - Hexadecimal value representing the desired color. + + @example + ``` + import chalk from 'chalk'; + + chalk.bgHex('#DEADED'); + ``` + */ + bgHex: (color: string) => this; + + /** + Use a [8-bit unsigned number](https://en.wikipedia.org/wiki/ANSI_escape_code#8-bit) to set background color. + + @example + ``` + import chalk from 'chalk'; + + chalk.bgAnsi256(201); + ``` + */ + bgAnsi256: (index: number) => this; + + /** + Modifier: Reset the current style. + */ + readonly reset: this; + + /** + Modifier: Make the text bold. + */ + readonly bold: this; + + /** + Modifier: Make the text have lower opacity. + */ + readonly dim: this; + + /** + Modifier: Make the text italic. *(Not widely supported)* + */ + readonly italic: this; + + /** + Modifier: Put a horizontal line below the text. *(Not widely supported)* + */ + readonly underline: this; + + /** + Modifier: Put a horizontal line above the text. *(Not widely supported)* + */ + readonly overline: this; + + /** + Modifier: Invert background and foreground colors. + */ + readonly inverse: this; + + /** + Modifier: Print the text but make it invisible. + */ + readonly hidden: this; + + /** + Modifier: Puts a horizontal line through the center of the text. *(Not widely supported)* + */ + readonly strikethrough: this; + + /** + Modifier: Print the text only when Chalk has a color level above zero. + + Can be useful for things that are purely cosmetic. + */ + readonly visible: this; + + readonly black: this; + readonly red: this; + readonly green: this; + readonly yellow: this; + readonly blue: this; + readonly magenta: this; + readonly cyan: this; + readonly white: this; + + /* + Alias for `blackBright`. + */ + readonly gray: this; + + /* + Alias for `blackBright`. + */ + readonly grey: this; + + readonly blackBright: this; + readonly redBright: this; + readonly greenBright: this; + readonly yellowBright: this; + readonly blueBright: this; + readonly magentaBright: this; + readonly cyanBright: this; + readonly whiteBright: this; + + readonly bgBlack: this; + readonly bgRed: this; + readonly bgGreen: this; + readonly bgYellow: this; + readonly bgBlue: this; + readonly bgMagenta: this; + readonly bgCyan: this; + readonly bgWhite: this; + + /* + Alias for `bgBlackBright`. + */ + readonly bgGray: this; + + /* + Alias for `bgBlackBright`. + */ + readonly bgGrey: this; + + readonly bgBlackBright: this; + readonly bgRedBright: this; + readonly bgGreenBright: this; + readonly bgYellowBright: this; + readonly bgBlueBright: this; + readonly bgMagentaBright: this; + readonly bgCyanBright: this; + readonly bgWhiteBright: this; +} + +/** +Main Chalk object that allows to chain styles together. + +Call the last one as a method with a string argument. + +Order doesn't matter, and later styles take precedent in case of a conflict. + +This simply means that `chalk.red.yellow.green` is equivalent to `chalk.green`. +*/ +declare const chalk: ChalkInstance; + +export const supportsColor: ColorInfo; + +export const chalkStderr: typeof chalk; +export const supportsColorStderr: typeof supportsColor; + +export { + ModifierName, ForegroundColorName, BackgroundColorName, ColorName, + modifierNames, foregroundColorNames, backgroundColorNames, colorNames, +// } from '#ansi-styles'; +} from './vendor/ansi-styles/index.js'; + +export { + ColorInfo, + ColorSupport, + ColorSupportLevel, +// } from '#supports-color'; +} from './vendor/supports-color/index.js'; + +// TODO: Remove these aliases in the next major version +/** +@deprecated Use `ModifierName` instead. + +Basic modifier names. +*/ +export type Modifiers = ModifierName; + +/** +@deprecated Use `ForegroundColorName` instead. + +Basic foreground color names. + +[More colors here.](https://github.com/chalk/chalk/blob/main/readme.md#256-and-truecolor-color-support) +*/ +export type ForegroundColor = ForegroundColorName; + +/** +@deprecated Use `BackgroundColorName` instead. + +Basic background color names. + +[More colors here.](https://github.com/chalk/chalk/blob/main/readme.md#256-and-truecolor-color-support) +*/ +export type BackgroundColor = BackgroundColorName; + +/** +@deprecated Use `ColorName` instead. + +Basic color names. The combination of foreground and background color names. + +[More colors here.](https://github.com/chalk/chalk/blob/main/readme.md#256-and-truecolor-color-support) +*/ +export type Color = ColorName; + +/** +@deprecated Use `modifierNames` instead. + +Basic modifier names. +*/ +export const modifiers: readonly Modifiers[]; + +/** +@deprecated Use `foregroundColorNames` instead. + +Basic foreground color names. +*/ +export const foregroundColors: readonly ForegroundColor[]; + +/** +@deprecated Use `backgroundColorNames` instead. + +Basic background color names. +*/ +export const backgroundColors: readonly BackgroundColor[]; + +/** +@deprecated Use `colorNames` instead. + +Basic color names. The combination of foreground and background color names. +*/ +export const colors: readonly Color[]; + +export default chalk; diff --git a/tests/fixtures/npm/cases/chalk5-builder/node_modules/chalk/source/index.js b/tests/fixtures/npm/cases/chalk5-builder/node_modules/chalk/source/index.js new file mode 100644 index 000000000..8bc993da5 --- /dev/null +++ b/tests/fixtures/npm/cases/chalk5-builder/node_modules/chalk/source/index.js @@ -0,0 +1,225 @@ +import ansiStyles from '#ansi-styles'; +import supportsColor from '#supports-color'; +import { // eslint-disable-line import/order + stringReplaceAll, + stringEncaseCRLFWithFirstIndex, +} from './utilities.js'; + +const {stdout: stdoutColor, stderr: stderrColor} = supportsColor; + +const GENERATOR = Symbol('GENERATOR'); +const STYLER = Symbol('STYLER'); +const IS_EMPTY = Symbol('IS_EMPTY'); + +// `supportsColor.level` → `ansiStyles.color[name]` mapping +const levelMapping = [ + 'ansi', + 'ansi', + 'ansi256', + 'ansi16m', +]; + +const styles = Object.create(null); + +const applyOptions = (object, options = {}) => { + if (options.level && !(Number.isInteger(options.level) && options.level >= 0 && options.level <= 3)) { + throw new Error('The `level` option should be an integer from 0 to 3'); + } + + // Detect level if not set manually + const colorLevel = stdoutColor ? stdoutColor.level : 0; + object.level = options.level === undefined ? colorLevel : options.level; +}; + +export class Chalk { + constructor(options) { + // eslint-disable-next-line no-constructor-return + return chalkFactory(options); + } +} + +const chalkFactory = options => { + const chalk = (...strings) => strings.join(' '); + applyOptions(chalk, options); + + Object.setPrototypeOf(chalk, createChalk.prototype); + + return chalk; +}; + +function createChalk(options) { + return chalkFactory(options); +} + +Object.setPrototypeOf(createChalk.prototype, Function.prototype); + +for (const [styleName, style] of Object.entries(ansiStyles)) { + styles[styleName] = { + get() { + const builder = createBuilder(this, createStyler(style.open, style.close, this[STYLER]), this[IS_EMPTY]); + Object.defineProperty(this, styleName, {value: builder}); + return builder; + }, + }; +} + +styles.visible = { + get() { + const builder = createBuilder(this, this[STYLER], true); + Object.defineProperty(this, 'visible', {value: builder}); + return builder; + }, +}; + +const getModelAnsi = (model, level, type, ...arguments_) => { + if (model === 'rgb') { + if (level === 'ansi16m') { + return ansiStyles[type].ansi16m(...arguments_); + } + + if (level === 'ansi256') { + return ansiStyles[type].ansi256(ansiStyles.rgbToAnsi256(...arguments_)); + } + + return ansiStyles[type].ansi(ansiStyles.rgbToAnsi(...arguments_)); + } + + if (model === 'hex') { + return getModelAnsi('rgb', level, type, ...ansiStyles.hexToRgb(...arguments_)); + } + + return ansiStyles[type][model](...arguments_); +}; + +const usedModels = ['rgb', 'hex', 'ansi256']; + +for (const model of usedModels) { + styles[model] = { + get() { + const {level} = this; + return function (...arguments_) { + const styler = createStyler(getModelAnsi(model, levelMapping[level], 'color', ...arguments_), ansiStyles.color.close, this[STYLER]); + return createBuilder(this, styler, this[IS_EMPTY]); + }; + }, + }; + + const bgModel = 'bg' + model[0].toUpperCase() + model.slice(1); + styles[bgModel] = { + get() { + const {level} = this; + return function (...arguments_) { + const styler = createStyler(getModelAnsi(model, levelMapping[level], 'bgColor', ...arguments_), ansiStyles.bgColor.close, this[STYLER]); + return createBuilder(this, styler, this[IS_EMPTY]); + }; + }, + }; +} + +const proto = Object.defineProperties(() => {}, { + ...styles, + level: { + enumerable: true, + get() { + return this[GENERATOR].level; + }, + set(level) { + this[GENERATOR].level = level; + }, + }, +}); + +const createStyler = (open, close, parent) => { + let openAll; + let closeAll; + if (parent === undefined) { + openAll = open; + closeAll = close; + } else { + openAll = parent.openAll + open; + closeAll = close + parent.closeAll; + } + + return { + open, + close, + openAll, + closeAll, + parent, + }; +}; + +const createBuilder = (self, _styler, _isEmpty) => { + // Single argument is hot path, implicit coercion is faster than anything + // eslint-disable-next-line no-implicit-coercion + const builder = (...arguments_) => applyStyle(builder, (arguments_.length === 1) ? ('' + arguments_[0]) : arguments_.join(' ')); + + // We alter the prototype because we must return a function, but there is + // no way to create a function with a different prototype + Object.setPrototypeOf(builder, proto); + + builder[GENERATOR] = self; + builder[STYLER] = _styler; + builder[IS_EMPTY] = _isEmpty; + + return builder; +}; + +const applyStyle = (self, string) => { + if (self.level <= 0 || !string) { + return self[IS_EMPTY] ? '' : string; + } + + let styler = self[STYLER]; + + if (styler === undefined) { + return string; + } + + const {openAll, closeAll} = styler; + if (string.includes('\u001B')) { + while (styler !== undefined) { + // Replace any instances already present with a re-opening code + // otherwise only the part of the string until said closing code + // will be colored, and the rest will simply be 'plain'. + string = stringReplaceAll(string, styler.close, styler.open); + + styler = styler.parent; + } + } + + // We can move both next actions out of loop, because remaining actions in loop won't have + // any/visible effect on parts we add here. Close the styling before a linebreak and reopen + // after next line to fix a bleed issue on macOS: https://github.com/chalk/chalk/pull/92 + const lfIndex = string.indexOf('\n'); + if (lfIndex !== -1) { + string = stringEncaseCRLFWithFirstIndex(string, closeAll, openAll, lfIndex); + } + + return openAll + string + closeAll; +}; + +Object.defineProperties(createChalk.prototype, styles); + +const chalk = createChalk(); +export const chalkStderr = createChalk({level: stderrColor ? stderrColor.level : 0}); + +export { + modifierNames, + foregroundColorNames, + backgroundColorNames, + colorNames, + + // TODO: Remove these aliases in the next major version + modifierNames as modifiers, + foregroundColorNames as foregroundColors, + backgroundColorNames as backgroundColors, + colorNames as colors, +} from './vendor/ansi-styles/index.js'; + +export { + stdoutColor as supportsColor, + stderrColor as supportsColorStderr, +}; + +export default chalk; diff --git a/tests/fixtures/npm/cases/chalk5-builder/node_modules/chalk/source/utilities.js b/tests/fixtures/npm/cases/chalk5-builder/node_modules/chalk/source/utilities.js new file mode 100644 index 000000000..4366dee0d --- /dev/null +++ b/tests/fixtures/npm/cases/chalk5-builder/node_modules/chalk/source/utilities.js @@ -0,0 +1,33 @@ +// TODO: When targeting Node.js 16, use `String.prototype.replaceAll`. +export function stringReplaceAll(string, substring, replacer) { + let index = string.indexOf(substring); + if (index === -1) { + return string; + } + + const substringLength = substring.length; + let endIndex = 0; + let returnValue = ''; + do { + returnValue += string.slice(endIndex, index) + substring + replacer; + endIndex = index + substringLength; + index = string.indexOf(substring, endIndex); + } while (index !== -1); + + returnValue += string.slice(endIndex); + return returnValue; +} + +export function stringEncaseCRLFWithFirstIndex(string, prefix, postfix, index) { + let endIndex = 0; + let returnValue = ''; + do { + const gotCR = string[index - 1] === '\r'; + returnValue += string.slice(endIndex, (gotCR ? index - 1 : index)) + prefix + (gotCR ? '\r\n' : '\n') + postfix; + endIndex = index + 1; + index = string.indexOf('\n', endIndex); + } while (index !== -1); + + returnValue += string.slice(endIndex); + return returnValue; +} diff --git a/tests/fixtures/npm/cases/chalk5-builder/node_modules/chalk/source/vendor/ansi-styles/index.d.ts b/tests/fixtures/npm/cases/chalk5-builder/node_modules/chalk/source/vendor/ansi-styles/index.d.ts new file mode 100644 index 000000000..58f133abe --- /dev/null +++ b/tests/fixtures/npm/cases/chalk5-builder/node_modules/chalk/source/vendor/ansi-styles/index.d.ts @@ -0,0 +1,236 @@ +export interface CSPair { // eslint-disable-line @typescript-eslint/naming-convention + /** + The ANSI terminal control sequence for starting this style. + */ + readonly open: string; + + /** + The ANSI terminal control sequence for ending this style. + */ + readonly close: string; +} + +export interface ColorBase { + /** + The ANSI terminal control sequence for ending this color. + */ + readonly close: string; + + ansi(code: number): string; + + ansi256(code: number): string; + + ansi16m(red: number, green: number, blue: number): string; +} + +export interface Modifier { + /** + Resets the current color chain. + */ + readonly reset: CSPair; + + /** + Make text bold. + */ + readonly bold: CSPair; + + /** + Emitting only a small amount of light. + */ + readonly dim: CSPair; + + /** + Make text italic. (Not widely supported) + */ + readonly italic: CSPair; + + /** + Make text underline. (Not widely supported) + */ + readonly underline: CSPair; + + /** + Make text overline. + + Supported on VTE-based terminals, the GNOME terminal, mintty, and Git Bash. + */ + readonly overline: CSPair; + + /** + Inverse background and foreground colors. + */ + readonly inverse: CSPair; + + /** + Prints the text, but makes it invisible. + */ + readonly hidden: CSPair; + + /** + Puts a horizontal line through the center of the text. (Not widely supported) + */ + readonly strikethrough: CSPair; +} + +export interface ForegroundColor { + readonly black: CSPair; + readonly red: CSPair; + readonly green: CSPair; + readonly yellow: CSPair; + readonly blue: CSPair; + readonly cyan: CSPair; + readonly magenta: CSPair; + readonly white: CSPair; + + /** + Alias for `blackBright`. + */ + readonly gray: CSPair; + + /** + Alias for `blackBright`. + */ + readonly grey: CSPair; + + readonly blackBright: CSPair; + readonly redBright: CSPair; + readonly greenBright: CSPair; + readonly yellowBright: CSPair; + readonly blueBright: CSPair; + readonly cyanBright: CSPair; + readonly magentaBright: CSPair; + readonly whiteBright: CSPair; +} + +export interface BackgroundColor { + readonly bgBlack: CSPair; + readonly bgRed: CSPair; + readonly bgGreen: CSPair; + readonly bgYellow: CSPair; + readonly bgBlue: CSPair; + readonly bgCyan: CSPair; + readonly bgMagenta: CSPair; + readonly bgWhite: CSPair; + + /** + Alias for `bgBlackBright`. + */ + readonly bgGray: CSPair; + + /** + Alias for `bgBlackBright`. + */ + readonly bgGrey: CSPair; + + readonly bgBlackBright: CSPair; + readonly bgRedBright: CSPair; + readonly bgGreenBright: CSPair; + readonly bgYellowBright: CSPair; + readonly bgBlueBright: CSPair; + readonly bgCyanBright: CSPair; + readonly bgMagentaBright: CSPair; + readonly bgWhiteBright: CSPair; +} + +export interface ConvertColor { + /** + Convert from the RGB color space to the ANSI 256 color space. + + @param red - (`0...255`) + @param green - (`0...255`) + @param blue - (`0...255`) + */ + rgbToAnsi256(red: number, green: number, blue: number): number; + + /** + Convert from the RGB HEX color space to the RGB color space. + + @param hex - A hexadecimal string containing RGB data. + */ + hexToRgb(hex: string): [red: number, green: number, blue: number]; + + /** + Convert from the RGB HEX color space to the ANSI 256 color space. + + @param hex - A hexadecimal string containing RGB data. + */ + hexToAnsi256(hex: string): number; + + /** + Convert from the ANSI 256 color space to the ANSI 16 color space. + + @param code - A number representing the ANSI 256 color. + */ + ansi256ToAnsi(code: number): number; + + /** + Convert from the RGB color space to the ANSI 16 color space. + + @param red - (`0...255`) + @param green - (`0...255`) + @param blue - (`0...255`) + */ + rgbToAnsi(red: number, green: number, blue: number): number; + + /** + Convert from the RGB HEX color space to the ANSI 16 color space. + + @param hex - A hexadecimal string containing RGB data. + */ + hexToAnsi(hex: string): number; +} + +/** +Basic modifier names. +*/ +export type ModifierName = keyof Modifier; + +/** +Basic foreground color names. + +[More colors here.](https://github.com/chalk/chalk/blob/main/readme.md#256-and-truecolor-color-support) +*/ +export type ForegroundColorName = keyof ForegroundColor; + +/** +Basic background color names. + +[More colors here.](https://github.com/chalk/chalk/blob/main/readme.md#256-and-truecolor-color-support) +*/ +export type BackgroundColorName = keyof BackgroundColor; + +/** +Basic color names. The combination of foreground and background color names. + +[More colors here.](https://github.com/chalk/chalk/blob/main/readme.md#256-and-truecolor-color-support) +*/ +export type ColorName = ForegroundColorName | BackgroundColorName; + +/** +Basic modifier names. +*/ +export const modifierNames: readonly ModifierName[]; + +/** +Basic foreground color names. +*/ +export const foregroundColorNames: readonly ForegroundColorName[]; + +/** +Basic background color names. +*/ +export const backgroundColorNames: readonly BackgroundColorName[]; + +/* +Basic color names. The combination of foreground and background color names. +*/ +export const colorNames: readonly ColorName[]; + +declare const ansiStyles: { + readonly modifier: Modifier; + readonly color: ColorBase & ForegroundColor; + readonly bgColor: ColorBase & BackgroundColor; + readonly codes: ReadonlyMap; +} & ForegroundColor & BackgroundColor & Modifier & ConvertColor; + +export default ansiStyles; diff --git a/tests/fixtures/npm/cases/chalk5-builder/node_modules/chalk/source/vendor/ansi-styles/index.js b/tests/fixtures/npm/cases/chalk5-builder/node_modules/chalk/source/vendor/ansi-styles/index.js new file mode 100644 index 000000000..eaa7bed6c --- /dev/null +++ b/tests/fixtures/npm/cases/chalk5-builder/node_modules/chalk/source/vendor/ansi-styles/index.js @@ -0,0 +1,223 @@ +const ANSI_BACKGROUND_OFFSET = 10; + +const wrapAnsi16 = (offset = 0) => code => `\u001B[${code + offset}m`; + +const wrapAnsi256 = (offset = 0) => code => `\u001B[${38 + offset};5;${code}m`; + +const wrapAnsi16m = (offset = 0) => (red, green, blue) => `\u001B[${38 + offset};2;${red};${green};${blue}m`; + +const styles = { + modifier: { + reset: [0, 0], + // 21 isn't widely supported and 22 does the same thing + bold: [1, 22], + dim: [2, 22], + italic: [3, 23], + underline: [4, 24], + overline: [53, 55], + inverse: [7, 27], + hidden: [8, 28], + strikethrough: [9, 29], + }, + color: { + black: [30, 39], + red: [31, 39], + green: [32, 39], + yellow: [33, 39], + blue: [34, 39], + magenta: [35, 39], + cyan: [36, 39], + white: [37, 39], + + // Bright color + blackBright: [90, 39], + gray: [90, 39], // Alias of `blackBright` + grey: [90, 39], // Alias of `blackBright` + redBright: [91, 39], + greenBright: [92, 39], + yellowBright: [93, 39], + blueBright: [94, 39], + magentaBright: [95, 39], + cyanBright: [96, 39], + whiteBright: [97, 39], + }, + bgColor: { + bgBlack: [40, 49], + bgRed: [41, 49], + bgGreen: [42, 49], + bgYellow: [43, 49], + bgBlue: [44, 49], + bgMagenta: [45, 49], + bgCyan: [46, 49], + bgWhite: [47, 49], + + // Bright color + bgBlackBright: [100, 49], + bgGray: [100, 49], // Alias of `bgBlackBright` + bgGrey: [100, 49], // Alias of `bgBlackBright` + bgRedBright: [101, 49], + bgGreenBright: [102, 49], + bgYellowBright: [103, 49], + bgBlueBright: [104, 49], + bgMagentaBright: [105, 49], + bgCyanBright: [106, 49], + bgWhiteBright: [107, 49], + }, +}; + +export const modifierNames = Object.keys(styles.modifier); +export const foregroundColorNames = Object.keys(styles.color); +export const backgroundColorNames = Object.keys(styles.bgColor); +export const colorNames = [...foregroundColorNames, ...backgroundColorNames]; + +function assembleStyles() { + const codes = new Map(); + + for (const [groupName, group] of Object.entries(styles)) { + for (const [styleName, style] of Object.entries(group)) { + styles[styleName] = { + open: `\u001B[${style[0]}m`, + close: `\u001B[${style[1]}m`, + }; + + group[styleName] = styles[styleName]; + + codes.set(style[0], style[1]); + } + + Object.defineProperty(styles, groupName, { + value: group, + enumerable: false, + }); + } + + Object.defineProperty(styles, 'codes', { + value: codes, + enumerable: false, + }); + + styles.color.close = '\u001B[39m'; + styles.bgColor.close = '\u001B[49m'; + + styles.color.ansi = wrapAnsi16(); + styles.color.ansi256 = wrapAnsi256(); + styles.color.ansi16m = wrapAnsi16m(); + styles.bgColor.ansi = wrapAnsi16(ANSI_BACKGROUND_OFFSET); + styles.bgColor.ansi256 = wrapAnsi256(ANSI_BACKGROUND_OFFSET); + styles.bgColor.ansi16m = wrapAnsi16m(ANSI_BACKGROUND_OFFSET); + + // From https://github.com/Qix-/color-convert/blob/3f0e0d4e92e235796ccb17f6e85c72094a651f49/conversions.js + Object.defineProperties(styles, { + rgbToAnsi256: { + value(red, green, blue) { + // We use the extended greyscale palette here, with the exception of + // black and white. normal palette only has 4 greyscale shades. + if (red === green && green === blue) { + if (red < 8) { + return 16; + } + + if (red > 248) { + return 231; + } + + return Math.round(((red - 8) / 247) * 24) + 232; + } + + return 16 + + (36 * Math.round(red / 255 * 5)) + + (6 * Math.round(green / 255 * 5)) + + Math.round(blue / 255 * 5); + }, + enumerable: false, + }, + hexToRgb: { + value(hex) { + const matches = /[a-f\d]{6}|[a-f\d]{3}/i.exec(hex.toString(16)); + if (!matches) { + return [0, 0, 0]; + } + + let [colorString] = matches; + + if (colorString.length === 3) { + colorString = [...colorString].map(character => character + character).join(''); + } + + const integer = Number.parseInt(colorString, 16); + + return [ + /* eslint-disable no-bitwise */ + (integer >> 16) & 0xFF, + (integer >> 8) & 0xFF, + integer & 0xFF, + /* eslint-enable no-bitwise */ + ]; + }, + enumerable: false, + }, + hexToAnsi256: { + value: hex => styles.rgbToAnsi256(...styles.hexToRgb(hex)), + enumerable: false, + }, + ansi256ToAnsi: { + value(code) { + if (code < 8) { + return 30 + code; + } + + if (code < 16) { + return 90 + (code - 8); + } + + let red; + let green; + let blue; + + if (code >= 232) { + red = (((code - 232) * 10) + 8) / 255; + green = red; + blue = red; + } else { + code -= 16; + + const remainder = code % 36; + + red = Math.floor(code / 36) / 5; + green = Math.floor(remainder / 6) / 5; + blue = (remainder % 6) / 5; + } + + const value = Math.max(red, green, blue) * 2; + + if (value === 0) { + return 30; + } + + // eslint-disable-next-line no-bitwise + let result = 30 + ((Math.round(blue) << 2) | (Math.round(green) << 1) | Math.round(red)); + + if (value === 2) { + result += 60; + } + + return result; + }, + enumerable: false, + }, + rgbToAnsi: { + value: (red, green, blue) => styles.ansi256ToAnsi(styles.rgbToAnsi256(red, green, blue)), + enumerable: false, + }, + hexToAnsi: { + value: hex => styles.ansi256ToAnsi(styles.hexToAnsi256(hex)), + enumerable: false, + }, + }); + + return styles; +} + +const ansiStyles = assembleStyles(); + +export default ansiStyles; diff --git a/tests/fixtures/npm/cases/chalk5-builder/node_modules/chalk/source/vendor/supports-color/browser.d.ts b/tests/fixtures/npm/cases/chalk5-builder/node_modules/chalk/source/vendor/supports-color/browser.d.ts new file mode 100644 index 000000000..0e8a9ccb8 --- /dev/null +++ b/tests/fixtures/npm/cases/chalk5-builder/node_modules/chalk/source/vendor/supports-color/browser.d.ts @@ -0,0 +1 @@ +export {default} from './index.js'; diff --git a/tests/fixtures/npm/cases/chalk5-builder/node_modules/chalk/source/vendor/supports-color/browser.js b/tests/fixtures/npm/cases/chalk5-builder/node_modules/chalk/source/vendor/supports-color/browser.js new file mode 100644 index 000000000..fbb6ce0fc --- /dev/null +++ b/tests/fixtures/npm/cases/chalk5-builder/node_modules/chalk/source/vendor/supports-color/browser.js @@ -0,0 +1,34 @@ +/* eslint-env browser */ + +const level = (() => { + if (!('navigator' in globalThis)) { + return 0; + } + + if (globalThis.navigator.userAgentData) { + const brand = navigator.userAgentData.brands.find(({brand}) => brand === 'Chromium'); + if (brand && brand.version > 93) { + return 3; + } + } + + if (/\b(Chrome|Chromium)\//.test(globalThis.navigator.userAgent)) { + return 1; + } + + return 0; +})(); + +const colorSupport = level !== 0 && { + level, + hasBasic: true, + has256: level >= 2, + has16m: level >= 3, +}; + +const supportsColor = { + stdout: colorSupport, + stderr: colorSupport, +}; + +export default supportsColor; diff --git a/tests/fixtures/npm/cases/chalk5-builder/node_modules/chalk/source/vendor/supports-color/index.d.ts b/tests/fixtures/npm/cases/chalk5-builder/node_modules/chalk/source/vendor/supports-color/index.d.ts new file mode 100644 index 000000000..db44a7851 --- /dev/null +++ b/tests/fixtures/npm/cases/chalk5-builder/node_modules/chalk/source/vendor/supports-color/index.d.ts @@ -0,0 +1,55 @@ +import type {WriteStream} from 'node:tty'; + +export type Options = { + /** + Whether `process.argv` should be sniffed for `--color` and `--no-color` flags. + + @default true + */ + readonly sniffFlags?: boolean; +}; + +/** +Levels: +- `0` - All colors disabled. +- `1` - Basic 16 colors support. +- `2` - ANSI 256 colors support. +- `3` - Truecolor 16 million colors support. +*/ +export type ColorSupportLevel = 0 | 1 | 2 | 3; + +/** +Detect whether the terminal supports color. +*/ +export type ColorSupport = { + /** + The color level. + */ + level: ColorSupportLevel; + + /** + Whether basic 16 colors are supported. + */ + hasBasic: boolean; + + /** + Whether ANSI 256 colors are supported. + */ + has256: boolean; + + /** + Whether Truecolor 16 million colors are supported. + */ + has16m: boolean; +}; + +export type ColorInfo = ColorSupport | false; + +export function createSupportsColor(stream?: WriteStream, options?: Options): ColorInfo; + +declare const supportsColor: { + stdout: ColorInfo; + stderr: ColorInfo; +}; + +export default supportsColor; diff --git a/tests/fixtures/npm/cases/chalk5-builder/node_modules/chalk/source/vendor/supports-color/index.js b/tests/fixtures/npm/cases/chalk5-builder/node_modules/chalk/source/vendor/supports-color/index.js new file mode 100644 index 000000000..265d7f858 --- /dev/null +++ b/tests/fixtures/npm/cases/chalk5-builder/node_modules/chalk/source/vendor/supports-color/index.js @@ -0,0 +1,190 @@ +import process from 'node:process'; +import os from 'node:os'; +import tty from 'node:tty'; + +// From: https://github.com/sindresorhus/has-flag/blob/main/index.js +/// function hasFlag(flag, argv = globalThis.Deno?.args ?? process.argv) { +function hasFlag(flag, argv = globalThis.Deno ? globalThis.Deno.args : process.argv) { + const prefix = flag.startsWith('-') ? '' : (flag.length === 1 ? '-' : '--'); + const position = argv.indexOf(prefix + flag); + const terminatorPosition = argv.indexOf('--'); + return position !== -1 && (terminatorPosition === -1 || position < terminatorPosition); +} + +const {env} = process; + +let flagForceColor; +if ( + hasFlag('no-color') + || hasFlag('no-colors') + || hasFlag('color=false') + || hasFlag('color=never') +) { + flagForceColor = 0; +} else if ( + hasFlag('color') + || hasFlag('colors') + || hasFlag('color=true') + || hasFlag('color=always') +) { + flagForceColor = 1; +} + +function envForceColor() { + if ('FORCE_COLOR' in env) { + if (env.FORCE_COLOR === 'true') { + return 1; + } + + if (env.FORCE_COLOR === 'false') { + return 0; + } + + return env.FORCE_COLOR.length === 0 ? 1 : Math.min(Number.parseInt(env.FORCE_COLOR, 10), 3); + } +} + +function translateLevel(level) { + if (level === 0) { + return false; + } + + return { + level, + hasBasic: true, + has256: level >= 2, + has16m: level >= 3, + }; +} + +function _supportsColor(haveStream, {streamIsTTY, sniffFlags = true} = {}) { + const noFlagForceColor = envForceColor(); + if (noFlagForceColor !== undefined) { + flagForceColor = noFlagForceColor; + } + + const forceColor = sniffFlags ? flagForceColor : noFlagForceColor; + + if (forceColor === 0) { + return 0; + } + + if (sniffFlags) { + if (hasFlag('color=16m') + || hasFlag('color=full') + || hasFlag('color=truecolor')) { + return 3; + } + + if (hasFlag('color=256')) { + return 2; + } + } + + // Check for Azure DevOps pipelines. + // Has to be above the `!streamIsTTY` check. + if ('TF_BUILD' in env && 'AGENT_NAME' in env) { + return 1; + } + + if (haveStream && !streamIsTTY && forceColor === undefined) { + return 0; + } + + const min = forceColor || 0; + + if (env.TERM === 'dumb') { + return min; + } + + if (process.platform === 'win32') { + // Windows 10 build 10586 is the first Windows release that supports 256 colors. + // Windows 10 build 14931 is the first release that supports 16m/TrueColor. + const osRelease = os.release().split('.'); + if ( + Number(osRelease[0]) >= 10 + && Number(osRelease[2]) >= 10_586 + ) { + return Number(osRelease[2]) >= 14_931 ? 3 : 2; + } + + return 1; + } + + if ('CI' in env) { + if (['GITHUB_ACTIONS', 'GITEA_ACTIONS', 'CIRCLECI'].some(key => key in env)) { + return 3; + } + + if (['TRAVIS', 'APPVEYOR', 'GITLAB_CI', 'BUILDKITE', 'DRONE'].some(sign => sign in env) || env.CI_NAME === 'codeship') { + return 1; + } + + return min; + } + + if ('TEAMCITY_VERSION' in env) { + return /^(9\.(0*[1-9]\d*)\.|\d{2,}\.)/.test(env.TEAMCITY_VERSION) ? 1 : 0; + } + + if (env.COLORTERM === 'truecolor') { + return 3; + } + + if (env.TERM === 'xterm-kitty') { + return 3; + } + + if (env.TERM === 'xterm-ghostty') { + return 3; + } + + if (env.TERM === 'wezterm') { + return 3; + } + + if ('TERM_PROGRAM' in env) { + const version = Number.parseInt((env.TERM_PROGRAM_VERSION || '').split('.')[0], 10); + + switch (env.TERM_PROGRAM) { + case 'iTerm.app': { + return version >= 3 ? 3 : 2; + } + + case 'Apple_Terminal': { + return 2; + } + // No default + } + } + + if (/-256(color)?$/i.test(env.TERM)) { + return 2; + } + + if (/^screen|^xterm|^vt100|^vt220|^rxvt|color|ansi|cygwin|linux/i.test(env.TERM)) { + return 1; + } + + if ('COLORTERM' in env) { + return 1; + } + + return min; +} + +export function createSupportsColor(stream, options = {}) { + const level = _supportsColor(stream, { + streamIsTTY: stream && stream.isTTY, + ...options, + }); + + return translateLevel(level); +} + +const supportsColor = { + stdout: createSupportsColor({isTTY: tty.isatty(1)}), + stderr: createSupportsColor({isTTY: tty.isatty(2)}), +}; + +export default supportsColor; diff --git a/tests/fixtures/npm/cases/chalk5-builder/package.json b/tests/fixtures/npm/cases/chalk5-builder/package.json new file mode 100644 index 000000000..3804daa8d --- /dev/null +++ b/tests/fixtures/npm/cases/chalk5-builder/package.json @@ -0,0 +1 @@ +{ "name": "chalk5-case", "private": true, "type": "module" } diff --git a/tests/fixtures/npm/cases/hot-shots-buffered/main.ts b/tests/fixtures/npm/cases/hot-shots-buffered/main.ts new file mode 100644 index 000000000..d17ce1167 --- /dev/null +++ b/tests/fixtures/npm/cases/hot-shots-buffered/main.ts @@ -0,0 +1,80 @@ +// The hot-shots StatsD client (compiled from scriptc's built-in shim) in +// buffered mode: metrics join one datagram until maxBufferSize, close +// flushes the rest, child clients share the buffer. Also explicit sample +// rates of 1, the telegraf line format, stat arrays with a callback, and +// suffixes. +import * as dgram from "node:dgram"; +import { StatsD } from "hot-shots"; + +const server = dgram.createSocket("udp4"); +const received: string[] = []; +let waiting: (() => void) | null = null; +let want = 0; +server.on("message", (msg) => { + received.push(msg.toString()); + if (waiting !== null && received.length >= want) { + const done = waiting; + waiting = null; + done(); + } +}); + +function datagrams(count: number): Promise { + want = count; + return new Promise((resolve) => { + if (received.length >= want) resolve(); + else waiting = resolve; + }); +} + +function flush(label: string): void { + console.log("--", label); + for (const line of received.splice(0)) console.log(JSON.stringify(line)); +} + +async function run(port: number): Promise { + const buffered = new StatsD({ + host: "127.0.0.1", + port, + maxBufferSize: 64, + bufferFlushInterval: 60000, + suffix: ".s", + globalTags: ["team:core"], + }); + buffered.increment("one"); + buffered.increment("two", 2); + buffered.gauge("three", 3); + buffered.histogram("four", 4, ["h:1"]); + const child = buffered.childClient({ prefix: "kid.", suffix: ".x" }); + child.increment("five"); + await new Promise((resolve) => buffered.close(() => resolve())); + await datagrams(2); + flush("buffered"); + + const telegraf = new StatsD({ host: "127.0.0.1", port, telegraf: true, globalTags: { region: "eu" } }); + telegraf.increment("hits", 1, 1, { route: "/x" }); + telegraf.timing("lat", 7, ["kind:a"]); + await datagrams(2); + flush("telegraf"); + + const plain = new StatsD({ host: "127.0.0.1", port, prefix: "p." }); + await new Promise((resolve) => { + plain.increment(["a", "b", "c"], 1, 1, ["t:1"], (err: unknown, bytes: unknown) => { + console.log("array sent", err, bytes); + resolve(); + }); + }); + plain.set("users", 42, 1); + plain.unique("visitors", "v1"); + plain.decrement("down", 3, ["d:1"]); + plain.decrement("gone"); + await datagrams(7); + flush("plain"); + await new Promise((resolve) => { + telegraf.close(() => plain.close(() => resolve())); + }); +} + +server.bind(0, "127.0.0.1", () => { + void run(server.address().port).then(() => server.close(() => console.log("server closed"))); +}); diff --git a/tests/fixtures/npm/cases/hot-shots-buffered/package.json b/tests/fixtures/npm/cases/hot-shots-buffered/package.json new file mode 100644 index 000000000..5ffd9800b --- /dev/null +++ b/tests/fixtures/npm/cases/hot-shots-buffered/package.json @@ -0,0 +1 @@ +{ "type": "module" } From 6de7864697c5b49f90be08694b6104b2987d0b58 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Wed, 21 Oct 2026 22:00:42 -0700 Subject: [PATCH 017/197] Record order baselines for the new chalk and hot-shots npm cases Co-Authored-By: Claude Opus 5.5 --- .../tests/fixtures/npm/cases/chalk5-builder/main.ts.json | 6 ++++++ .../fixtures/npm/cases/hot-shots-buffered/main.ts.json | 6 ++++++ 2 files changed, 12 insertions(+) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/chalk5-builder/main.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/hot-shots-buffered/main.ts.json diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/chalk5-builder/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/chalk5-builder/main.ts.json new file mode 100644 index 000000000..545c4d701 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/chalk5-builder/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/chalk5-builder/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/hot-shots-buffered/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/hot-shots-buffered/main.ts.json new file mode 100644 index 000000000..e60660c09 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/hot-shots-buffered/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/hot-shots-buffered/main.ts" + ], + "diags": [] +} From 284517accbfd37cd06c87a5a352fe8b43acf10f8 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Wed, 21 Oct 2026 22:04:27 -0700 Subject: [PATCH 018/197] Call chalk builders with one string through a typed method A callable class's %call1 method takes a single string argument directly, skipping the variadic closure's checked-dynamic call. Co-Authored-By: Claude Opus 5.5 --- packages/compiler/shims/chalk/v5.ts | 8 ++++ .../src/frontend/lowering/lower-calls.ts | 42 +++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/packages/compiler/shims/chalk/v5.ts b/packages/compiler/shims/chalk/v5.ts index 6599e35f9..494a9c69f 100644 --- a/packages/compiler/shims/chalk/v5.ts +++ b/packages/compiler/shims/chalk/v5.ts @@ -435,12 +435,14 @@ export class ChalkInstance { box: LevelBox; styler: Styler | undefined; isEmpty: boolean; + private readonly root: boolean; private readonly cache: (ChalkInstance | undefined)[] = []; constructor(box: LevelBox, styler: Styler | undefined, isEmpty: boolean, root: boolean) { this.box = box; this.styler = styler; this.isEmpty = isEmpty; + this.root = root; // One unconditional store: the field's definite assignment. this["%call"] = (...text: unknown[]): string => root @@ -448,6 +450,12 @@ export class ChalkInstance { : applyStyle(this, text.length === 1 ? "" + String(text[0]) : joinArguments(text)); } + /** One string argument: the compiler calls this instead of `%call` + * (no variadic arguments to box). */ + "%call1"(text: string): string { + return this.root ? text : applyStyle(this, text); + } + get level(): ColorSupportLevel { return this.box.level as ColorSupportLevel; } diff --git a/packages/compiler/src/frontend/lowering/lower-calls.ts b/packages/compiler/src/frontend/lowering/lower-calls.ts index 3c04924c0..9d6f477f6 100644 --- a/packages/compiler/src/frontend/lowering/lower-calls.ts +++ b/packages/compiler/src/frontend/lowering/lower-calls.ts @@ -1632,6 +1632,8 @@ export function lowerCall(lowerer: Lowerer, expr: ts.CallExpression): IrExpr { // A getter returning a callable class instance (chalk's // `chalk.red("x")`): call through its `%call` field. if (callee.type.kind === "object") { + const direct = callableInstanceStringCall(lowerer, expr, callee, locOf(expr)); + if (direct) return direct; callee = lowerer.hybridCallUnwrap(callee); if (callee.type.kind === "func") return lowerCallThroughValue(lowerer, expr, callee, locOf(expr)); } @@ -1886,11 +1888,47 @@ export function lowerCall(lowerer: Lowerer, expr: ts.CallExpression): IrExpr { // a non-func IR type was already rejected while lowering the callee. // HYBRID (function-with-properties) values call through their %call slot. let callee = lowerer.lowerExpr(expr.expression); + { + const direct = callableInstanceStringCall(lowerer, expr, callee, loc); + if (direct) return direct; + } if (callee.type.kind === "record" || callee.type.kind === "object") callee = lowerer.hybridCallUnwrap(callee); return lowerCallThroughValue(lowerer, expr, callee, loc); } +/** A callable class instance called with one string: its `%call1` method + * when the class has one (the chalk shim's typed fast path), skipping the + * variadic closure's checked-dynamic arguments. */ +function callableInstanceStringCall( + lowerer: Lowerer, + expr: ts.CallExpression, + callee: IrExpr, + loc: SrcLoc, +): IrExpr | null { + if (callee.type.kind !== "object" || expr.arguments.nodes.length !== 1) return null; + const arg = expr.arguments.nodes[0]!; + if (ts.isSpreadElement(arg)) return null; + const info = lowerer.classes.get(callee.type.className); + if (info === undefined || !info.fields.has("%call") || !lowerer.findMethodOn(info, "%call1")) + return null; + const value = lowerer.lowerExpr(arg); + if (value.type.kind === "string") + return lowerer.accessorCall(callee.type.className, "%call1", callee, [value], STRING, loc); + // Any other argument (a number behind a cast): the variadic closure, + // through the checked-dynamic call as lowerCallThroughValue would. + const closure = lowerer.hybridCallUnwrap(callee); + if (closure.type.kind !== "func") return null; + return { + kind: "dynCall", + callee: { kind: "dynFrom", value: closure, type: DYN, loc }, + calleeName: ts.isIdentifier(expr.expression) ? expr.expression.text : "value", + args: [lowerer.coerceToExpected(value, DYN)], + type: DYN, + loc, + }; +} + /** A call through an evaluated callee value: checked-dynamic callees, * function unions, typed-rest and runtime-arity closures. */ function lowerCallThroughValue( @@ -9208,6 +9246,10 @@ export function lowerRecordFieldCall( // A HYBRID (function-with-properties) field is callable through its // reserved %call slot — `colors.blue("x")` where blue also carries // `.bold` (the chalk shape). + if (callee.type.kind === "object" && !init) { + const direct = callableInstanceStringCall(lowerer, call, callee, loc); + if (direct) return direct; + } if (callee.type.kind === "record" || callee.type.kind === "object") callee = lowerer.hybridCallUnwrap(callee); // Inferred JavaScript option records can retain checked callables in From 88f9ba91eb94d63c254bc934beaaeb5dd887ea7e Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Wed, 21 Oct 2026 22:08:46 -0700 Subject: [PATCH 019/197] Format the shims and scope the chalk declaration-merge lint exception Co-Authored-By: Claude Opus 5.5 --- packages/compiler/shims/chalk/v5.ts | 241 +++++++++++++----- packages/compiler/shims/hot-shots/client.ts | 85 +++++- .../src/frontend/lowering/lower-calls.ts | 3 +- .../compiler/src/frontend/lowering/lowerer.ts | 5 +- .../compiler/src/frontend/package-shims.ts | 8 +- 5 files changed, 260 insertions(+), 82 deletions(-) diff --git a/packages/compiler/shims/chalk/v5.ts b/packages/compiler/shims/chalk/v5.ts index 494a9c69f..2d34c5c53 100644 --- a/packages/compiler/shims/chalk/v5.ts +++ b/packages/compiler/shims/chalk/v5.ts @@ -138,10 +138,7 @@ export const backgroundColorNames: BackgroundColorName[] = [ "bgCyanBright", "bgWhiteBright", ]; -export const colorNames: ColorName[] = [ - ...foregroundColorNames, - ...backgroundColorNames, -]; +export const colorNames: ColorName[] = [...foregroundColorNames, ...backgroundColorNames]; export const modifiers: Modifiers[] = modifierNames; export const foregroundColors: ForegroundColor[] = foregroundColorNames; export const backgroundColors: BackgroundColor[] = backgroundColorNames; @@ -151,14 +148,12 @@ const ESC = "\u001B["; /** Open and close codes, indexed by style id (the order of the getters). */ const OPEN_CODES: number[] = [ - 0, 1, 2, 3, 4, 53, 7, 8, 9, - 30, 31, 32, 33, 34, 35, 36, 37, 90, 90, 90, 91, 92, 93, 94, 95, 96, 97, - 40, 41, 42, 43, 44, 45, 46, 47, 100, 100, 100, 101, 102, 103, 104, 105, 106, 107, + 0, 1, 2, 3, 4, 53, 7, 8, 9, 30, 31, 32, 33, 34, 35, 36, 37, 90, 90, 90, 91, 92, 93, 94, 95, 96, + 97, 40, 41, 42, 43, 44, 45, 46, 47, 100, 100, 100, 101, 102, 103, 104, 105, 106, 107, ]; const CLOSE_CODES: number[] = [ - 0, 22, 22, 23, 24, 55, 27, 28, 29, - 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, - 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 0, 22, 22, 23, 24, 55, 27, 28, 29, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, + 39, 39, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, ]; const VISIBLE = 45; @@ -238,9 +233,19 @@ function hasFlag(flag: string): boolean { } let flagForceColor: number | undefined = undefined; -if (hasFlag("no-color") || hasFlag("no-colors") || hasFlag("color=false") || hasFlag("color=never")) { +if ( + hasFlag("no-color") || + hasFlag("no-colors") || + hasFlag("color=false") || + hasFlag("color=never") +) { flagForceColor = 0; -} else if (hasFlag("color") || hasFlag("colors") || hasFlag("color=true") || hasFlag("color=always")) { +} else if ( + hasFlag("color") || + hasFlag("colors") || + hasFlag("color=true") || + hasFlag("color=always") +) { flagForceColor = 1; } @@ -277,7 +282,8 @@ function detectLevel(streamIsTTY: boolean): number { if (hasFlag("color=256")) return 2; if (inEnv("TF_BUILD") && inEnv("AGENT_NAME")) return 1; if (!streamIsTTY && forceColor === undefined) return 0; - const min = forceColor === undefined || forceColor === 0 || Number.isNaN(forceColor) ? 0 : forceColor; + const min = + forceColor === undefined || forceColor === 0 || Number.isNaN(forceColor) ? 0 : forceColor; const env = process.env; const term = env["TERM"]; if (term === "dumb") return min; @@ -388,7 +394,10 @@ function stringEncaseCRLFWithFirstIndex( do { const gotCR = string[index - 1] === "\r"; returnValue += - string.slice(endIndex, gotCR ? index - 1 : index) + prefix + (gotCR ? "\r\n" : "\n") + postfix; + string.slice(endIndex, gotCR ? index - 1 : index) + + prefix + + (gotCR ? "\r\n" : "\n") + + postfix; endIndex = index + 1; index = string.indexOf("\n", endIndex); } while (index !== -1); @@ -426,12 +435,19 @@ function joinArguments(args: unknown[]): string { /** A chalk instance or one of its style chains: callable (chalk's call * signature merges in from the interface), with every style a cached * getter. */ +// The merge is the point: the interface gives instances chalk's call +// signature, and the class's `%call` field implements it. +// oxlint-disable-next-line typescript/no-unsafe-declaration-merging export interface ChalkInstance { (...text: unknown[]): string; } +// oxlint-disable-next-line typescript/no-unsafe-declaration-merging export class ChalkInstance { - "%call": (...text: unknown[]) => string; + "%call": (...text: unknown[]) => string = (...text: unknown[]): string => + this.root + ? joinArguments(text) + : applyStyle(this, text.length === 1 ? "" + String(text[0]) : joinArguments(text)); box: LevelBox; styler: Styler | undefined; isEmpty: boolean; @@ -443,11 +459,6 @@ export class ChalkInstance { this.styler = styler; this.isEmpty = isEmpty; this.root = root; - // One unconditional store: the field's definite assignment. - this["%call"] = (...text: unknown[]): string => - root - ? joinArguments(text) - : applyStyle(this, text.length === 1 ? "" + String(text[0]) : joinArguments(text)); } /** One string argument: the compiler calls this instead of `%call` @@ -509,57 +520,153 @@ export class ChalkInstance { return this.model(ansi256Open(10, index), true); } - get reset(): ChalkInstance { return this.style(0); } - get bold(): ChalkInstance { return this.style(1); } - get dim(): ChalkInstance { return this.style(2); } - get italic(): ChalkInstance { return this.style(3); } - get underline(): ChalkInstance { return this.style(4); } - get overline(): ChalkInstance { return this.style(5); } - get inverse(): ChalkInstance { return this.style(6); } - get hidden(): ChalkInstance { return this.style(7); } - get strikethrough(): ChalkInstance { return this.style(8); } - get visible(): ChalkInstance { return this.style(VISIBLE); } - get black(): ChalkInstance { return this.style(9); } - get red(): ChalkInstance { return this.style(10); } - get green(): ChalkInstance { return this.style(11); } - get yellow(): ChalkInstance { return this.style(12); } - get blue(): ChalkInstance { return this.style(13); } - get magenta(): ChalkInstance { return this.style(14); } - get cyan(): ChalkInstance { return this.style(15); } - get white(): ChalkInstance { return this.style(16); } - get blackBright(): ChalkInstance { return this.style(17); } - get gray(): ChalkInstance { return this.style(18); } - get grey(): ChalkInstance { return this.style(19); } - get redBright(): ChalkInstance { return this.style(20); } - get greenBright(): ChalkInstance { return this.style(21); } - get yellowBright(): ChalkInstance { return this.style(22); } - get blueBright(): ChalkInstance { return this.style(23); } - get magentaBright(): ChalkInstance { return this.style(24); } - get cyanBright(): ChalkInstance { return this.style(25); } - get whiteBright(): ChalkInstance { return this.style(26); } - get bgBlack(): ChalkInstance { return this.style(27); } - get bgRed(): ChalkInstance { return this.style(28); } - get bgGreen(): ChalkInstance { return this.style(29); } - get bgYellow(): ChalkInstance { return this.style(30); } - get bgBlue(): ChalkInstance { return this.style(31); } - get bgMagenta(): ChalkInstance { return this.style(32); } - get bgCyan(): ChalkInstance { return this.style(33); } - get bgWhite(): ChalkInstance { return this.style(34); } - get bgBlackBright(): ChalkInstance { return this.style(35); } - get bgGray(): ChalkInstance { return this.style(36); } - get bgGrey(): ChalkInstance { return this.style(37); } - get bgRedBright(): ChalkInstance { return this.style(38); } - get bgGreenBright(): ChalkInstance { return this.style(39); } - get bgYellowBright(): ChalkInstance { return this.style(40); } - get bgBlueBright(): ChalkInstance { return this.style(41); } - get bgMagentaBright(): ChalkInstance { return this.style(42); } - get bgCyanBright(): ChalkInstance { return this.style(43); } - get bgWhiteBright(): ChalkInstance { return this.style(44); } + get reset(): ChalkInstance { + return this.style(0); + } + get bold(): ChalkInstance { + return this.style(1); + } + get dim(): ChalkInstance { + return this.style(2); + } + get italic(): ChalkInstance { + return this.style(3); + } + get underline(): ChalkInstance { + return this.style(4); + } + get overline(): ChalkInstance { + return this.style(5); + } + get inverse(): ChalkInstance { + return this.style(6); + } + get hidden(): ChalkInstance { + return this.style(7); + } + get strikethrough(): ChalkInstance { + return this.style(8); + } + get visible(): ChalkInstance { + return this.style(VISIBLE); + } + get black(): ChalkInstance { + return this.style(9); + } + get red(): ChalkInstance { + return this.style(10); + } + get green(): ChalkInstance { + return this.style(11); + } + get yellow(): ChalkInstance { + return this.style(12); + } + get blue(): ChalkInstance { + return this.style(13); + } + get magenta(): ChalkInstance { + return this.style(14); + } + get cyan(): ChalkInstance { + return this.style(15); + } + get white(): ChalkInstance { + return this.style(16); + } + get blackBright(): ChalkInstance { + return this.style(17); + } + get gray(): ChalkInstance { + return this.style(18); + } + get grey(): ChalkInstance { + return this.style(19); + } + get redBright(): ChalkInstance { + return this.style(20); + } + get greenBright(): ChalkInstance { + return this.style(21); + } + get yellowBright(): ChalkInstance { + return this.style(22); + } + get blueBright(): ChalkInstance { + return this.style(23); + } + get magentaBright(): ChalkInstance { + return this.style(24); + } + get cyanBright(): ChalkInstance { + return this.style(25); + } + get whiteBright(): ChalkInstance { + return this.style(26); + } + get bgBlack(): ChalkInstance { + return this.style(27); + } + get bgRed(): ChalkInstance { + return this.style(28); + } + get bgGreen(): ChalkInstance { + return this.style(29); + } + get bgYellow(): ChalkInstance { + return this.style(30); + } + get bgBlue(): ChalkInstance { + return this.style(31); + } + get bgMagenta(): ChalkInstance { + return this.style(32); + } + get bgCyan(): ChalkInstance { + return this.style(33); + } + get bgWhite(): ChalkInstance { + return this.style(34); + } + get bgBlackBright(): ChalkInstance { + return this.style(35); + } + get bgGray(): ChalkInstance { + return this.style(36); + } + get bgGrey(): ChalkInstance { + return this.style(37); + } + get bgRedBright(): ChalkInstance { + return this.style(38); + } + get bgGreenBright(): ChalkInstance { + return this.style(39); + } + get bgYellowBright(): ChalkInstance { + return this.style(40); + } + get bgBlueBright(): ChalkInstance { + return this.style(41); + } + get bgMagentaBright(): ChalkInstance { + return this.style(42); + } + get bgCyanBright(): ChalkInstance { + return this.style(43); + } + get bgWhiteBright(): ChalkInstance { + return this.style(44); + } } function optionsLevel(options: Options | undefined): number { const level = options === undefined ? undefined : options.level; - if (level !== undefined && level !== 0 && !(Number.isInteger(level) && level >= 0 && level <= 3)) { + if ( + level !== undefined && + level !== 0 && + !(Number.isInteger(level) && level >= 0 && level <= 3) + ) { throw new Error("The `level` option should be an integer from 0 to 3"); } if (level === undefined) return stdoutColor === false ? 0 : stdoutColor.level; diff --git a/packages/compiler/shims/hot-shots/client.ts b/packages/compiler/shims/hot-shots/client.ts index d0b5c8924..2f8ca0580 100644 --- a/packages/compiler/shims/hot-shots/client.ts +++ b/packages/compiler/shims/hot-shots/client.ts @@ -542,7 +542,13 @@ export class StatsD { // ── stats functions ── - timing(stat: string | string[], time: number, sampleRate?: number, tags?: Tags, callback?: StatsCb): void; + timing( + stat: string | string[], + time: number, + sampleRate?: number, + tags?: Tags, + callback?: StatsCb, + ): void; timing(stat: string | string[], time: number, tags?: Tags, callback?: StatsCb): void; timing(stat: string | string[], time: number, callback?: StatsCb): void; timing(stat: string | string[], time: number, sampleRate?: number, callback?: StatsCb): void; @@ -557,7 +563,13 @@ export class StatsD { } increment(stat: string, tags?: Tags): void; - increment(stat: string | string[], value: number, sampleRate?: number, tags?: Tags, callback?: StatsCb): void; + increment( + stat: string | string[], + value: number, + sampleRate?: number, + tags?: Tags, + callback?: StatsCb, + ): void; increment(stat: string | string[], value: number, tags?: Tags, callback?: StatsCb): void; increment(stat: string | string[], value: number, callback?: StatsCb): void; increment(stat: string | string[], value: number, sampleRate?: number, callback?: StatsCb): void; @@ -580,7 +592,13 @@ export class StatsD { } decrement(stat: string, tags?: Tags): void; - decrement(stat: string | string[], value: number, sampleRate?: number, tags?: Tags, callback?: StatsCb): void; + decrement( + stat: string | string[], + value: number, + sampleRate?: number, + tags?: Tags, + callback?: StatsCb, + ): void; decrement(stat: string | string[], value: number, tags?: Tags, callback?: StatsCb): void; decrement(stat: string | string[], value: number, callback?: StatsCb): void; decrement(stat: string | string[], value: number, sampleRate?: number, callback?: StatsCb): void; @@ -601,7 +619,13 @@ export class StatsD { this.sendAll(stat, -amount, "c", sampleRate, tagArg, callback); } - histogram(stat: string | string[], value: number, sampleRate?: number, tags?: Tags, callback?: StatsCb): void; + histogram( + stat: string | string[], + value: number, + sampleRate?: number, + tags?: Tags, + callback?: StatsCb, + ): void; histogram(stat: string | string[], value: number, tags?: Tags, callback?: StatsCb): void; histogram(stat: string | string[], value: number, callback?: StatsCb): void; histogram(stat: string | string[], value: number, sampleRate?: number, callback?: StatsCb): void; @@ -615,10 +639,21 @@ export class StatsD { this.sendAll(stat, value, "h", sampleRate, tags, callback); } - distribution(stat: string | string[], value: number, sampleRate?: number, tags?: Tags, callback?: StatsCb): void; + distribution( + stat: string | string[], + value: number, + sampleRate?: number, + tags?: Tags, + callback?: StatsCb, + ): void; distribution(stat: string | string[], value: number, tags?: Tags, callback?: StatsCb): void; distribution(stat: string | string[], value: number, callback?: StatsCb): void; - distribution(stat: string | string[], value: number, sampleRate?: number, callback?: StatsCb): void; + distribution( + stat: string | string[], + value: number, + sampleRate?: number, + callback?: StatsCb, + ): void; distribution( stat: string | string[], value: number, @@ -629,7 +664,13 @@ export class StatsD { this.sendAll(stat, value, "d", sampleRate, tags, callback); } - gauge(stat: string | string[], value: number, sampleRate?: number, tags?: Tags, callback?: StatsCb): void; + gauge( + stat: string | string[], + value: number, + sampleRate?: number, + tags?: Tags, + callback?: StatsCb, + ): void; gauge(stat: string | string[], value: number, tags?: Tags, callback?: StatsCb): void; gauge(stat: string | string[], value: number, callback?: StatsCb): void; gauge(stat: string | string[], value: number, sampleRate?: number, callback?: StatsCb): void; @@ -643,10 +684,21 @@ export class StatsD { this.sendAll(stat, value, "g", sampleRate, tags, callback); } - set(stat: string | string[], value: number | string, sampleRate?: number, tags?: Tags, callback?: StatsCb): void; + set( + stat: string | string[], + value: number | string, + sampleRate?: number, + tags?: Tags, + callback?: StatsCb, + ): void; set(stat: string | string[], value: number | string, tags?: Tags, callback?: StatsCb): void; set(stat: string | string[], value: number | string, callback?: StatsCb): void; - set(stat: string | string[], value: number | string, sampleRate?: number, callback?: StatsCb): void; + set( + stat: string | string[], + value: number | string, + sampleRate?: number, + callback?: StatsCb, + ): void; set( stat: string | string[], value: number | string, @@ -657,10 +709,21 @@ export class StatsD { this.sendAll(stat, value, "s", sampleRate, tags, callback); } - unique(stat: string | string[], value: number | string, sampleRate?: number, tags?: Tags, callback?: StatsCb): void; + unique( + stat: string | string[], + value: number | string, + sampleRate?: number, + tags?: Tags, + callback?: StatsCb, + ): void; unique(stat: string | string[], value: number | string, tags?: Tags, callback?: StatsCb): void; unique(stat: string | string[], value: number | string, callback?: StatsCb): void; - unique(stat: string | string[], value: number | string, sampleRate?: number, callback?: StatsCb): void; + unique( + stat: string | string[], + value: number | string, + sampleRate?: number, + callback?: StatsCb, + ): void; unique( stat: string | string[], value: number | string, diff --git a/packages/compiler/src/frontend/lowering/lower-calls.ts b/packages/compiler/src/frontend/lowering/lower-calls.ts index 9d6f477f6..b650ca567 100644 --- a/packages/compiler/src/frontend/lowering/lower-calls.ts +++ b/packages/compiler/src/frontend/lowering/lower-calls.ts @@ -1635,7 +1635,8 @@ export function lowerCall(lowerer: Lowerer, expr: ts.CallExpression): IrExpr { const direct = callableInstanceStringCall(lowerer, expr, callee, locOf(expr)); if (direct) return direct; callee = lowerer.hybridCallUnwrap(callee); - if (callee.type.kind === "func") return lowerCallThroughValue(lowerer, expr, callee, locOf(expr)); + if (callee.type.kind === "func") + return lowerCallThroughValue(lowerer, expr, callee, locOf(expr)); } if (callee.type.kind === "func") { const args = completeFuncValueArgs(lowerer, expr, callee.type, locOf(expr)); diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index ac1a778cf..206c42fdc 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -9656,7 +9656,10 @@ export class Lowerer { (id) => this.unions.get(id), ) ) { - return this.coerceToExpected({ kind: "dynFrom", value: un, type: DYN, loc: un.loc }, expected); + return this.coerceToExpected( + { kind: "dynFrom", value: un, type: DYN, loc: un.loc }, + expected, + ); } } // A zero-param function whose RETURN is a wider record array than the diff --git a/packages/compiler/src/frontend/package-shims.ts b/packages/compiler/src/frontend/package-shims.ts index 3af7ebf59..a0c2e4cb9 100644 --- a/packages/compiler/src/frontend/package-shims.ts +++ b/packages/compiler/src/frontend/package-shims.ts @@ -72,7 +72,10 @@ interface ImplementationShim { variants?: readonly string[]; } -const IMPLEMENTATION_SHIMS: ReadonlyMap = new Map([ +const IMPLEMENTATION_SHIMS: ReadonlyMap = new Map< + string, + ImplementationShim +>([ [ "fs-extra", { @@ -330,7 +333,8 @@ function shimOfDeclarationPath(path: string): { pkgDir: string; shim: Implementa for (const shim of IMPLEMENTATION_SHIMS.values()) { if (shim.typesFile === undefined) continue; const suffix = `/${shim.name}/${shim.typesFile}`; - if (path.endsWith(suffix)) return { pkgDir: path.slice(0, path.length - suffix.length + shim.name.length + 1), shim }; + if (path.endsWith(suffix)) + return { pkgDir: path.slice(0, path.length - suffix.length + shim.name.length + 1), shim }; } return null; } From 8c9f5cfc9ff02dccb7a97266d210bfba559a5ccc Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Thu, 22 Oct 2026 01:42:01 -0700 Subject: [PATCH 020/197] Free hot-shots' unclosed udp and tcp transports at exit The socket's error listener captured the transport, so a client that is never closed (mock mode) kept a cycle the RC audit reports at exit; it now captures only the listener list. Co-Authored-By: Claude Opus 5.5 --- packages/compiler/shims/hot-shots/client.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/compiler/shims/hot-shots/client.ts b/packages/compiler/shims/hot-shots/client.ts index 2f8ca0580..aa752d703 100644 --- a/packages/compiler/shims/hot-shots/client.ts +++ b/packages/compiler/shims/hot-shots/client.ts @@ -218,7 +218,11 @@ class UdpTransport implements Transport { this.cacheDnsTtl = cacheDnsTtl; this.socket = dgram.createSocket("udp4"); this.socket.unref(); - this.socket.on("error", (err) => this.errors.emit(err)); + // The listener captures the list, not the transport: a socket that is + // never closed keeps its listener, and a capture of `this` would make + // a cycle the reference counts cannot free at exit. + const errors = this.errors; + this.socket.on("error", (err) => errors.emit(err)); } send(buf: Buffer, callback: SendCallback): void { if (!this.cacheDns) { @@ -263,7 +267,11 @@ class TcpTransport implements Transport { this.socket = net.connect(port, host ?? "localhost"); this.socket.setKeepAlive(true); this.socket.unref(); - this.socket.on("error", (err) => this.errors.emit(err)); + // The listener captures the list, not the transport: a socket that is + // never closed keeps its listener, and a capture of `this` would make + // a cycle the reference counts cannot free at exit. + const errors = this.errors; + this.socket.on("error", (err) => errors.emit(err)); } send(buf: Buffer, callback: SendCallback): void { // The typed socket's write takes the data alone; the completion runs From 2e297caf9955cf8cb4606c9acd7dda8f3eac5877 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Thu, 22 Oct 2026 05:33:34 -0700 Subject: [PATCH 021/197] Log hot-shots' failed final flush through an unknown value Programs whose Error subclasses define accessors still compile the shim's close logging. Co-Authored-By: Claude Opus 5.5 --- packages/compiler/shims/hot-shots/client.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/compiler/shims/hot-shots/client.ts b/packages/compiler/shims/hot-shots/client.ts index aa752d703..b407132f4 100644 --- a/packages/compiler/shims/hot-shots/client.ts +++ b/packages/compiler/shims/hot-shots/client.ts @@ -155,6 +155,13 @@ function formatDate(date: number): number { return Math.round(date); } +/** console.error of an arbitrary value, as hot-shots logs errors. The + * value crosses as unknown: the typed Error path has no inspect lowering + * in programs whose Error subclasses define accessors. */ +function logError(value: unknown): void { + console.error(value); +} + function errorWithCode(message: string, code: string | undefined): Error { const error: NodeJS.ErrnoException = new Error(message); error.code = code; @@ -1013,7 +1020,7 @@ export class StatsD { this.flushQueue((err) => { if (err) { if (callback) return callback(err); - console.error(err); + logError(err); return; } let intervalAttempts = 0; From 164174f811a2f0f927fde6f76f5578c6ff538139 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Thu, 22 Oct 2026 02:29:47 -0700 Subject: [PATCH 022/197] Keep literal key order for records a narrowing throw arm types The key-order witness pass read every libCall typed as a record as an opaque producer of that record, which pins the declaration order. A void runtime call typed by its use site, such as error.nodeThrow in the ternary that narrows a possibly-undefined binding before a member read, never produces a value. A module-level accumulator enumerated in its interface's order instead of its literal's. New corpus program 6915. Co-Authored-By: Claude Opus 5.5 --- .../frontend/lowering/key-order-witness.ts | 6 +- .../6915-key-order-narrowed-throw.ts.json | 6 ++ tests/corpus/6915-key-order-narrowed-throw.ts | 72 +++++++++++++++++++ .../6915-key-order-narrowed-throw.ts.json | 1 + 4 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6915-key-order-narrowed-throw.ts.json create mode 100644 tests/corpus/6915-key-order-narrowed-throw.ts create mode 100644 tests/harness/corpus-costs/6915-key-order-narrowed-throw.ts.json diff --git a/packages/compiler/src/frontend/lowering/key-order-witness.ts b/packages/compiler/src/frontend/lowering/key-order-witness.ts index 2fc9a6cbd..7ce8ae074 100644 --- a/packages/compiler/src/frontend/lowering/key-order-witness.ts +++ b/packages/compiler/src/frontend/lowering/key-order-witness.ts @@ -45,6 +45,7 @@ import type { IrExpr, IrFunction, IrRecordShape, IrStmt, IrType } from "../../ir import { shapeHasAccessorSlots, typeEquals, UNDEFINED_T } from "../../ir/ir.js"; import { everyExprChild, everyStmtChild, everyTypeChild } from "../../ir/traverse.js"; import type { Lowerer } from "./lowerer.js"; +import { LIB_FN_SIGS } from "../../ir/builtin-signatures.js"; /** One property of an object literal, in source order. */ export type KeyOrderSegment = @@ -312,7 +313,10 @@ export function applyKeyOrderWitnesses(lowerer: Lowerer, fns: readonly IrFunctio // Runtime surfaces (console, util.format, workers) that take a // record may walk its keys; runtime builders make new records. for (const arg of e.args) observedRoots.push(arg.type); - producedBy(e.type, true); + // A void runtime call typed by its use site is a throw standing in + // for a value (`error.nodeThrow` in a narrowing ternary): it never + // produces one. + if (LIB_FN_SIGS[e.fn].result.kind !== "void") producedBy(e.type, true); return; case "intrinsic": // Intrinsics hand back values built elsewhere, except allSettled's diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6915-key-order-narrowed-throw.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6915-key-order-narrowed-throw.ts.json new file mode 100644 index 000000000..83f8cc5d5 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6915-key-order-narrowed-throw.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6915-key-order-narrowed-throw.ts" + ], + "diags": [] +} diff --git a/tests/corpus/6915-key-order-narrowed-throw.ts b/tests/corpus/6915-key-order-narrowed-throw.ts new file mode 100644 index 000000000..d4bc693f2 --- /dev/null +++ b/tests/corpus/6915-key-order-narrowed-throw.ts @@ -0,0 +1,72 @@ +// A record whose reads narrow a possibly-undefined binding (the narrowing +// ternary's throw arm stands in for a value of the record's type) still +// takes its literal's key order: a module-level accumulator typed by an +// interface declared in another order, as an event batcher builds it. +interface Entry { + id: string; + at: number; + text?: string; + stream: "out" | "err"; +} +interface Batch { + team: string; + project: string; + job: string; + origin: "task" | "worker"; + host: string; + entries: Entry[]; + sizes?: number[]; + stage?: "live" | "staging"; + branch?: string; + run?: string; + entry?: string; + path?: string; +} +interface Job { + id: string; + run?: string; + project: string; + team: string; + host: string; + stage: string | null; +} +let acc: Batch | undefined; + +function append(j: Job, text: string): void { + if (!acc) { + acc = { + run: j.run, + job: j.id, + stage: j.stage === null ? "staging" : "live", + project: j.project, + team: j.team, + origin: "task", + host: j.host, + entry: ".", + entries: [], + }; + } + acc.entries.push({ id: String(acc.entries.length), at: 1, text, stream: "out" }); + setTimeout(() => { + const batch = acc; + if (!batch) return; + acc = undefined; + for (const line of split(batch)) console.log(line); + }, 1); +} + +function split(batch: Batch): string[] { + const json = JSON.stringify(batch); + if (json.length < 200 || batch.entries.length < 2) return [json]; + const index = Math.floor(batch.entries.length / 2); + const first = { ...batch, entries: batch.entries.slice(0, index) }; + const second = { ...batch, entries: batch.entries.slice(index) }; + return [...split(first), ...split(second)]; +} + +const j: Job = { id: "job", run: "r1", project: "prj", team: "core", host: "h", stage: "live" }; +append(j, "one"); +append(j, "two"); +setTimeout(() => { + for (let i = 0; i < 6; i++) append(j, "line " + i); +}, 5); diff --git a/tests/harness/corpus-costs/6915-key-order-narrowed-throw.ts.json b/tests/harness/corpus-costs/6915-key-order-narrowed-throw.ts.json new file mode 100644 index 000000000..fae2adeb4 --- /dev/null +++ b/tests/harness/corpus-costs/6915-key-order-narrowed-throw.ts.json @@ -0,0 +1 @@ +{ "plain": 1500, "san": 9000 } From dd3ea7c00c80f3fa7eb3ba30d34863a28f2ed928 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 07:47:03 -0500 Subject: [PATCH 023/197] Narrow filter results through written type predicates with checked conversions `.filter` with a written type predicate (`(p): p is Cat => ...`, or a guard function passed as a value such as `filter(isDog)`) was refused with SC1090, because TypeScript does not check the predicate. An inferred predicate that keeps several union arms (`v => v !== undefined` over `(string | number | undefined)[]`) was refused too. The filter loop now keeps the receiver's element type, and the result converts to the predicate's type element by element (filter-predicates.ts): - a union element to one of its arms, or to a union of some of its arms, by its tag; - a checked-dynamic element by dynCheck; - a class instance to a subclass by its class; - a tuple or record to one with the same fields, field by field. An element the predicate's type does not describe, kept because the predicate returned true for it, throws a catchable TypeError: "a kept element is not a 'Cat' (a type predicate returned true for a value of another type)". The diagnostics fixture filter-narrow.ts, which pinned the three refusals, compiles now; corpus 7007 covers its cases. Fixes #1680. --- .../lowering/containers/array-callbacks.ts | 119 +++++++-- .../lowering/containers/filter-predicates.ts | 231 ++++++++++++++++++ .../src/frontend/lowering/lower-calls.ts | 33 ++- ...007-filter-written-type-predicates.ts.json | 6 + .../tests/diagnostics/filter-narrow.ts.json | 6 - .../7007-filter-written-type-predicates.ts | 87 +++++++ tests/diagnostics/filter-narrow.ts | 22 -- .../__snapshots__/filter-narrow.ts.txt | 20 -- ...007-filter-written-type-predicates.ts.json | 1 + 9 files changed, 435 insertions(+), 90 deletions(-) create mode 100644 packages/compiler/src/frontend/lowering/containers/filter-predicates.ts create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7007-filter-written-type-predicates.ts.json delete mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/filter-narrow.ts.json create mode 100644 tests/corpus/7007-filter-written-type-predicates.ts delete mode 100644 tests/diagnostics/filter-narrow.ts delete mode 100644 tests/harness/__snapshots__/filter-narrow.ts.txt create mode 100644 tests/harness/corpus-costs/7007-filter-written-type-predicates.ts.json diff --git a/packages/compiler/src/frontend/lowering/containers/array-callbacks.ts b/packages/compiler/src/frontend/lowering/containers/array-callbacks.ts index 612c47ea0..dcadf3bf3 100644 --- a/packages/compiler/src/frontend/lowering/containers/array-callbacks.ts +++ b/packages/compiler/src/frontend/lowering/containers/array-callbacks.ts @@ -59,6 +59,7 @@ import { } from "./array-iteration.js"; import { dynamicReceiverThrows } from "./dynamic-receivers.js"; import { defaultAfterUndefined, lowerStaticallyUndefinedArgument } from "../optional-arguments.js"; +import { checkedFilterNarrowing } from "./filter-predicates.js"; /** `a.map(fn)` / `a.filter(fn)` / `a.forEach(fn)` desugar to a direct * call of a synthetic module function — one per method + element/result @@ -123,10 +124,11 @@ export function lowerArrayHofCall( } lowerer.badType(argNode, lowerer.typeOf(argNode)); } - const outElem = + const filtered = method === "filter" ? filterResultElem(lowerer, call, argNode, elem, valueT, booleanFilter) - : callbackArrayElement(lowerer, call, fnRet); + : null; + const outElem = filtered !== null ? filtered.elem : callbackArrayElement(lowerer, call, fnRet); // A filter that re-tags its elements keeps the single union-typed visit. const fast = present || booleanFilter || (method === "filter" && !typeEquals(outElem, elem)) @@ -145,19 +147,33 @@ export function lowerArrayHofCall( present, ); const resultType: IrType = method === "map" || method === "filter" ? arrayOf(outElem) : VOID; - return { + const result: IrExpr = { kind: "call", callee: helper, args: [receiver, fnArg, ...(fast ? [fast] : [])], type: resultType, loc, }; + const narrowing = filtered?.narrowing ?? null; + if (narrowing === null) return result; + return { + kind: "call", + callee: narrowing.helper, + args: [result], + type: arrayOf(narrowing.elem), + loc, + }; } -/** The checker-selected element of a filter result, with the same trust - * boundary as the historical narrowing filter: an inferred inline type - * predicate may prove one source arm; a written predicate annotation may - * not. Boolean is safe when every discarded arm is a nullish unit. */ +/** The element a filter keeps, and the checked conversion of the result + * to the checker's narrower element when the predicate does not prove it. + * An inferred inline type predicate proves one source arm, or a class + * view, so the loop keeps that type directly. A written predicate (`(p): p + * is Cat => ...`, a guard function passed as a value) is an assertion + * TypeScript does not check, and an inferred predicate that keeps several + * arms has no single arm to re-tag: the loop keeps the receiver's element + * and `narrowing` (checkedFilterNarrowing) converts each kept element. + * Boolean is safe when every discarded arm is a nullish unit. */ function filterResultElem( lowerer: Lowerer, call: ts.CallExpression, @@ -165,7 +181,37 @@ function filterResultElem( elem: IrType, valueT: IrType, booleanFilter: boolean, -): IrType { +): { elem: IrType; narrowing: { helper: string; elem: IrType } | null } { + const kept = filterKeptElem(lowerer, call, argNode, elem, valueT, booleanFilter); + if (typeof kept !== "string") return { elem: kept, narrowing: null }; + return { elem, narrowing: { helper: kept, elem: filterCheckerElem(lowerer, call, elem) } }; +} + +/** The checker's spelling of a filter result's element type. */ +function filterTypeText(lowerer: Lowerer, call: ts.CallExpression): string { + const result = lowerer.typeOf(call); + const element = lowerer.checker.isArrayType(result) + ? lowerer.checker.getTypeArguments(result as ts.TypeReference)[0] + : undefined; + return element === undefined ? "" : lowerer.checker.typeToString(element); +} + +/** The checker's element of a filter result, or the receiver's when the + * filter introduces no refinement. */ +function filterCheckerElem(lowerer: Lowerer, call: ts.CallExpression, elem: IrType): IrType { + const result = lowerer.mapTypeOf(lowerer.typeOf(call)); + return result?.kind === "array" ? result.elem : elem; +} + +/** filterResultElem's element, or the name of the checked narrowing helper. */ +function filterKeptElem( + lowerer: Lowerer, + call: ts.CallExpression, + argNode: ts.Expression, + elem: IrType, + valueT: IrType, + booleanFilter: boolean, +): IrType | string { const callT = lowerer.typeOf(call); const result = lowerer.mapTypeOf(callT); let outElem = result?.kind === "array" ? result.elem : elem; @@ -191,23 +237,31 @@ function filterResultElem( "keep the receiver's element type instead — annotate the callback's return ': boolean' " + "(the checker then skips the predicate) or annotate the result with the receiver's own " + "element type — and narrow the elements after"; - if (!booleanFilter) { + const written = + !booleanFilter && + ((!ts.isArrowFunction(argNode) && !ts.isFunctionExpression(argNode)) || + argNode.type !== undefined); + const narrowing = booleanFilter + ? null + : (): string | null => + checkedFilterNarrowing(lowerer, elem, outElem, filterTypeText(lowerer, call), locOf(call)); + if (written) { + const helper = narrowing!(); + if (helper !== null) return helper; if (!ts.isArrowFunction(argNode) && !ts.isFunctionExpression(argNode)) { lowerer.unsupported( "SC1090", argNode, - `narrowing '.filter' through a callback VALUE (only an inline callback whose predicate ` + - `the checker inferred can re-tag — ${annotateEscape})`, - ); - } - if (argNode.type) { - lowerer.unsupported( - "SC1090", - argNode, - `narrowing '.filter' with a hand-written type predicate (a written 'x is T' is an ` + - `unchecked assertion nothing validates at runtime — ${annotateEscape})`, + `narrowing '.filter' to '${lowerer.fmt(outElem)}' through a callback VALUE (no checked ` + + `conversion from '${lowerer.fmt(elem)}' — ${annotateEscape})`, ); } + lowerer.unsupported( + "SC1090", + argNode, + `narrowing '.filter' to '${lowerer.fmt(outElem)}' with a hand-written type predicate (no ` + + `checked conversion from '${lowerer.fmt(elem)}' — ${annotateEscape})`, + ); } const classRefinement = !booleanFilter && outElem.kind === "object" && canAssertClassValue(lowerer, valueT, outElem); @@ -215,6 +269,8 @@ function filterResultElem( !classRefinement && (valueT.kind !== "union" || lowerer.armTag(valueT.unionId, outElem) < 0) ) { + const helper = narrowing?.() ?? null; + if (helper !== null) return helper; const multiArm = outElem.kind === "union"; lowerer.unsupported( "SC1090", @@ -2650,6 +2706,7 @@ export function lowerTuplePredicateCall( // the value type an array find narrows from. let foundT: IrType = elem; let retag: string | null = null; + let narrowing: { helper: string; elem: IrType } | null = null; if (method === "filter") { // The element union is built in position order, so the checker's // result element can list the same arms in another order. @@ -2660,9 +2717,11 @@ export function lowerTuplePredicateCall( checkerResult?.kind === "array" && arms(checkerResult.elem).length === arms(elem).length && arms(checkerResult.elem).every((arm) => arms(elem).some((other) => typeEquals(arm, other))); - outElem = unchanged - ? elem - : filterResultElem(lowerer, call, argNode, elem, elem, booleanFilter); + if (!unchanged) { + const filtered = filterResultElem(lowerer, call, argNode, elem, elem, booleanFilter); + outElem = filtered.elem; + narrowing = filtered.narrowing; + } resultT = arrayOf(outElem); } else if (method === "find") { resultT = lowerer.irTypeOf(call); @@ -2795,5 +2854,19 @@ export function lowerTuplePredicateCall( loc, }); } - return { kind: "call", callee: helper, args: [receiver, fnArg], type: resultT, loc }; + const result: IrExpr = { + kind: "call", + callee: helper, + args: [receiver, fnArg], + type: resultT, + loc, + }; + if (narrowing === null) return result; + return { + kind: "call", + callee: narrowing.helper, + args: [result], + type: arrayOf(narrowing.elem), + loc, + }; } diff --git a/packages/compiler/src/frontend/lowering/containers/filter-predicates.ts b/packages/compiler/src/frontend/lowering/containers/filter-predicates.ts new file mode 100644 index 000000000..df57813fe --- /dev/null +++ b/packages/compiler/src/frontend/lowering/containers/filter-predicates.ts @@ -0,0 +1,231 @@ +/* Checked narrowing for `.filter` with a written type predicate. + * + * A written predicate (`(p): p is Cat => ...`, or a guard function passed + * as a value) is an assertion TypeScript does not check, so the filter loop + * keeps the receiver's element type and the result converts to the + * predicate's type afterwards, element by element: + * + * - a union element to one of its arms, or to a union of some of its + * arms, by its tag; + * - a checked-dynamic element by dynCheck; + * - a class instance to a subclass by its class; + * - a tuple or record to one with the same fields, field by field. + * + * Each conversion throws the catchable TypeError of a checked extraction + * when a kept element is not of the predicate's type, which is when the + * predicate returned true for a value it does not describe. */ +import { varRef } from "../../../ir/build.js"; +import { + BOOL, + F64, + STRING, + isUnitType, + typeEquals, + type IrExpr, + type IrStmt, + type IrType, + type SrcLoc, +} from "../../../ir/ir.js"; +import { checkedClassAssertion } from "../class-assertions.js"; +import { buildArrayConversion } from "../coercions/builders.js"; +import type { Lowerer } from "../lowerer.js"; +import { keyedHelperName } from "../stable-names.js"; +import { typeKey } from "../../type-mapper.js"; + +/** The interned helper converting a `from[]` filter result to `to[]`, or + * null when an element of `from` has no checked conversion to `to`. + * `typeText` names the predicate's type in the TypeError. */ +export function checkedFilterNarrowing( + lowerer: Lowerer, + from: IrType, + to: IrType, + typeText: string, + loc: SrcLoc, +): string | null { + const element = elementHelper(lowerer, from, to, loc, typeText); + if (element === null) return null; + const key = `filter.narrow:${typeKey(from)}:${typeKey(to)}:${typeText}`; + const existing = lowerer.valueHelpers.get(key); + if (existing) return existing; + const name = keyedHelperName(lowerer, "%arr.narrow", key); + lowerer.valueHelpers.set(key, name); + const outT: IrType = { kind: "array", elem: to }; + lowerer.liftedFns.push( + buildArrayConversion( + name, + { kind: "array", elem: from }, + { kind: "arrayLit", elems: [], type: outT, loc }, + (value, _index, result) => ({ + kind: "arrIntrinsic", + method: "push", + receiver: result, + args: [{ kind: "call", callee: element, args: [value], type: to, loc }], + type: F64, + loc, + }), + loc, + ), + ); + return name; +} + +/** The interned helper converting one `from` value to `to`, or null. */ +function elementHelper( + lowerer: Lowerer, + from: IrType, + to: IrType, + loc: SrcLoc, + typeText = lowerer.fmt(to), +): string | null { + const key = `filter.narrow.value:${typeKey(from)}:${typeKey(to)}:${typeText}`; + const existing = lowerer.valueHelpers.get(key); + if (existing) return existing; + // A recursive type pair converts through the helper being built. + if (lowerer.coercions.planning.has(key)) return null; + lowerer.coercions.planning.add(key); + const input = varRef("v.0", from, loc); + let body: IrStmt[] | null; + try { + body = conversionBody(lowerer, input, from, to, typeText, loc); + } finally { + lowerer.coercions.planning.delete(key); + } + if (body === null) return null; + const name = keyedHelperName(lowerer, "%narrow.checked", key); + lowerer.valueHelpers.set(key, name); + lowerer.liftedFns.push({ + name, + params: [{ localId: "v.0", name: "v", type: from }], + returnType: to, + locals: [{ id: "v.0", name: "v", type: from, mutable: false }], + body, + loc, + }); + return name; +} + +/** A converted value: `value` itself when the types are equal, or a call of + * the conversion helper. */ +function converted( + lowerer: Lowerer, + value: IrExpr, + from: IrType, + to: IrType, + loc: SrcLoc, +): IrExpr | null { + if (typeEquals(from, to)) return value; + const helper = elementHelper(lowerer, from, to, loc); + return helper === null ? null : { kind: "call", callee: helper, args: [value], type: to, loc }; +} + +/** The TypeError a kept element of another type throws. */ +function typeError(message: string, loc: SrcLoc): IrStmt { + return { + kind: "throw", + value: { + kind: "libCall", + fn: "error.new", + args: [{ kind: "strLit", value: message, type: STRING, loc }], + type: { kind: "object", className: "%TypeError" }, + loc, + }, + loc, + }; +} + +function conversionBody( + lowerer: Lowerer, + input: IrExpr, + from: IrType, + to: IrType, + typeText: string, + loc: SrcLoc, +): IrStmt[] | null { + if (typeEquals(from, to)) return [{ kind: "return", value: input, loc }]; + if (from.kind === "dyn") + return [{ kind: "return", value: { kind: "dynCheck", value: input, type: to, loc }, loc }]; + if (from.kind === "object" && to.kind === "object") { + const asserted = checkedClassAssertion(lowerer, input, to, loc); + return asserted === null ? null : [{ kind: "return", value: asserted, loc }]; + } + if (from.kind === "union") + return unionConversion(lowerer, input, from.unionId, to, typeText, loc); + if (from.kind === "record" && to.kind === "record") { + const source = lowerer.shapes.get(from.shapeId); + const target = lowerer.shapes.get(to.shapeId); + if ( + !source || + !target || + source.tuple !== target.tuple || + source.indexValue || + target.indexValue + ) + return null; + if (source.fields.length !== target.fields.length) return null; + const fields: { name: string; value: IrExpr }[] = []; + for (const field of target.fields) { + const own = source.fields.find((f) => f.name === field.name); + if (!own) return null; + const read: IrExpr = { + kind: "recordGet", + obj: input, + shapeId: from.shapeId, + field: field.name, + type: own.type, + loc, + }; + const value = converted(lowerer, read, own.type, field.type, loc); + if (value === null) return null; + fields.push({ name: field.name, value }); + } + return [{ kind: "return", value: { kind: "recordLit", fields, type: to, loc }, loc }]; + } + return null; +} + +/** A union value to one of its arms, or to a union of some of its arms: + * each arm the target holds returns, any other arm throws. */ +function unionConversion( + lowerer: Lowerer, + input: IrExpr, + unionId: string, + to: IrType, + typeText: string, + loc: SrcLoc, +): IrStmt[] | null { + const source = lowerer.unions.get(unionId); + if (!source) return null; + const targetArms = to.kind === "union" ? (lowerer.unions.get(to.unionId)?.arms ?? null) : [to]; + if (targetArms === null) return null; + const body: IrStmt[] = []; + let kept = 0; + for (let tag = 0; tag < source.arms.length; tag++) { + const arm = source.arms[tag]!; + const targetTag = targetArms.findIndex((t) => typeEquals(t, arm)); + if (targetTag < 0) continue; + kept++; + const payload: IrExpr = isUnitType(arm) + ? { kind: "unitLit", unit: arm.kind === "undefinedT" ? "undefined" : "null", type: arm, loc } + : { kind: "unionNarrow", unionId, tag, value: input, type: arm, loc }; + const value: IrExpr = + to.kind === "union" + ? { kind: "unionWrap", unionId: to.unionId, tag: targetTag, value: payload, type: to, loc } + : payload; + body.push({ + kind: "if", + cond: { kind: "unionIsTag", unionId, tag, negated: false, value: input, type: BOOL, loc }, + then: [{ kind: "return", value, loc }], + else_: null, + loc, + }); + } + // Every arm the target names must be one of the union's own arms. + if (kept === 0 || kept !== targetArms.length) return null; + body.push( + typeError( + `a kept element is not a '${typeText}' (a type predicate returned true for a value of another type)`, + loc, + ), + ); + return body; +} diff --git a/packages/compiler/src/frontend/lowering/lower-calls.ts b/packages/compiler/src/frontend/lowering/lower-calls.ts index 39b7207c0..3acfb256a 100644 --- a/packages/compiler/src/frontend/lowering/lower-calls.ts +++ b/packages/compiler/src/frontend/lowering/lower-calls.ts @@ -6668,6 +6668,15 @@ export function lowerFilterNarrowCall( } const narrowed = outElem !== null && !typeEquals(outElem, elem); if (!narrowed && !isBooleanArg) return null; + // A written predicate (`(p): p is Cat => ...`, a guard function value) + // proves nothing at run time: the generic array HOF path filters the + // receiver's elements and converts the kept ones checked + // (filterResultElem). + if ( + !isBooleanArg && + ((!ts.isArrowFunction(argNode) && !ts.isFunctionExpression(argNode)) || argNode.type) + ) + return null; const annotateEscape = "keep the receiver's element type instead — annotate the callback's return ': boolean' " + @@ -6679,6 +6688,9 @@ export function lowerFilterNarrowCall( lowerer.badType(call, callT); // defensive: a narrowed non-union receiver } tag = lowerer.armTag(elem.unionId, outElem); + // Several kept arms: the generic path converts the kept elements to + // the narrower union (filterResultElem). + if (tag < 0 && !isBooleanArg) return null; if (tag < 0) { lowerer.unsupported( "SC1090", @@ -6700,25 +6712,8 @@ export function lowerFilterNarrowCall( return { kind: "call", callee: helper, args: [receiver], type: arrayOf(outElem ?? elem), loc }; } - // Inferred type predicate: inline function literal, NO return - // annotation (a written one is an unchecked assertion), and the - // checker reports a predicate over parameter 0. - if (!ts.isArrowFunction(argNode) && !ts.isFunctionExpression(argNode)) { - lowerer.unsupported( - "SC1090", - argNode, - `narrowing '.filter' through a callback VALUE ` + - `(only an inline callback whose predicate the checker inferred can re-tag — ${annotateEscape})`, - ); - } - if (argNode.type) { - lowerer.unsupported( - "SC1090", - argNode, - `narrowing '.filter' with a hand-written type predicate ` + - `(a written 'x is T' is an unchecked assertion nothing validates at runtime — ${annotateEscape})`, - ); - } + // An inferred type predicate: an inline function literal with no return + // annotation, over parameter 0 (written predicates returned above). // The receiver evaluates FIRST, in the enclosing function, like JS. const receiver = lowerer.lowerExpr(access.expression); const fnArg = lowerer.lowerExpr(argNode); diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7007-filter-written-type-predicates.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7007-filter-written-type-predicates.ts.json new file mode 100644 index 000000000..d77ee1a57 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7007-filter-written-type-predicates.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7007-filter-written-type-predicates.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/filter-narrow.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/filter-narrow.ts.json deleted file mode 100644 index 40d5e462a..000000000 --- a/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/filter-narrow.ts.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "order": [ - "/tests/diagnostics/filter-narrow.ts" - ], - "diags": [] -} diff --git a/tests/corpus/7007-filter-written-type-predicates.ts b/tests/corpus/7007-filter-written-type-predicates.ts new file mode 100644 index 000000000..c2449b469 --- /dev/null +++ b/tests/corpus/7007-filter-written-type-predicates.ts @@ -0,0 +1,87 @@ +// `.filter` with a written type predicate: an annotated callback +// (`(p): p is Cat => ...`) or a guard function passed as a value. The +// result holds the predicate's type: a union arm, a union of arms, a +// number out of `unknown`, a record or tuple with a narrower field, or a +// subclass. An inferred predicate that keeps several arms narrows too. +interface Cat { + kind: "cat"; + meow: string; +} +interface Dog { + kind: "dog"; + bark: number; +} +interface Fish { + kind: "fish"; + fins: number; +} +type Pet = Cat | Dog | Fish; +const pets: Pet[] = [ + { kind: "cat", meow: "m" }, + { kind: "dog", bark: 2 }, + { kind: "fish", fins: 3 }, + { kind: "dog", bark: 5 }, +]; + +const cats = pets.filter((p): p is Cat => p.kind === "cat"); +console.log(cats.length, cats[0]!.meow); +const furry = pets.filter((p): p is Cat | Dog => p.kind !== "fish"); +console.log(furry.map((p) => (p.kind === "cat" ? p.meow : p.bark)).join(",")); +const notFish = pets.filter((p) => p.kind !== "fish"); +console.log(notFish.length); + +function isDog(p: Pet): p is Dog { + return p.kind === "dog"; +} +console.log(pets.filter(isDog).map((d) => d.bark * 10).join(",")); + +const isNumber = (v: unknown): v is number => typeof v === "number"; +const values: unknown[] = JSON.parse('[1, "a", 2, null, 4]'); +console.log(values.filter(isNumber).reduce((sum, n) => sum + n, 0)); + +const rows: unknown[] = JSON.parse('[{"id":1},{"id":"x"},{"id":3}]'); +const withIds = rows.filter( + (r): r is { id: number } => typeof (r as { id: unknown }).id === "number", +); +console.log(withIds.map((r) => r.id + 1).join(",")); + +const entries: [string, unknown][] = [ + ["a", 1], + ["b", "x"], + ["c", 3], +]; +const numeric = entries.filter((e): e is [string, number] => typeof e[1] === "number"); +console.log(numeric.map(([key, value]) => `${key}${value * 2}`).join(" ")); + +class Shape { + name: string; + constructor(name: string) { + this.name = name; + } +} +class Circle extends Shape { + r: number; + constructor(r: number) { + super("circle"); + this.r = r; + } +} +const shapes: Shape[] = [new Shape("square"), new Circle(2), new Circle(3)]; +const circles = shapes.filter((s): s is Circle => s instanceof Circle); +console.log(circles.length, circles.map((c) => c.r).join("+")); + +const mixed: (string | number | boolean)[] = ["a", 1, true, "b"]; +console.log(mixed.filter((v): v is string => typeof v === "string").join("")); +console.log(mixed.filter((v): v is string | number => typeof v !== "boolean").length); + +const pair: [string | number, string | number] = ["x", 4]; +console.log(pair.filter((v): v is number => typeof v === "number").map((n) => n + 1).join()); + +const optional: (string | number | undefined)[] = ["a", 1, undefined, 2]; +const defined = optional.filter((v) => v !== undefined); +console.log(defined.length, defined.map((v) => typeof v).join(",")); +function isText(v: string | undefined): v is string { + return typeof v === "string"; +} +const maybe: (string | undefined)[] = ["a", undefined, "b"]; +console.log(maybe.filter(isText).join("-")); diff --git a/tests/diagnostics/filter-narrow.ts b/tests/diagnostics/filter-narrow.ts deleted file mode 100644 index 45452e212..000000000 --- a/tests/diagnostics/filter-narrow.ts +++ /dev/null @@ -1,22 +0,0 @@ -// Narrowing filters re-tag ONLY what the runtime test proved: an inferred -// single-arm predicate or filter(Boolean). Everything else stays fenced — -// multi-arm targets (the re-tag doesn't exist), hand-written predicates -// (unchecked assertions), and predicate VALUES (their annotation is -// written by definition). -const xs: (string | number | undefined)[] = ["a", 1, undefined]; - -// Multi-arm narrowing: string | number is not a single arm. -const defined = xs.filter((x) => x !== undefined); -console.log(defined.length); - -// A hand-written predicate on an inline callback. -const ys: (string | undefined)[] = ["a", undefined]; -const strings = ys.filter((x): x is string => typeof x === "string"); -console.log(strings.length); - -// A predicate VALUE (declared function with a written predicate). -function isString(x: string | undefined): x is string { - return typeof x === "string"; -} -const viaValue = ys.filter(isString); -console.log(viaValue.length); diff --git a/tests/harness/__snapshots__/filter-narrow.ts.txt b/tests/harness/__snapshots__/filter-narrow.ts.txt deleted file mode 100644 index bc5180219..000000000 --- a/tests/harness/__snapshots__/filter-narrow.ts.txt +++ /dev/null @@ -1,20 +0,0 @@ -filter-narrow.ts:9:17 - error SC1090: '.filter' narrowing 'number | string | undefined' elements to the multi-arm 'number | string' (only a SINGLE arm re-tags — keep the receiver's element type instead — annotate the callback's return ': boolean' (the checker then skips the predicate) or annotate the result with the receiver's own element type — and narrow the elements after) is not supported yet - - 8 | // Multi-arm narrowing: string | number is not a single arm. - 9 | const defined = xs.filter((x) => x !== undefined); - | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - 10 | console.log(defined.length); - -filter-narrow.ts:14:27 - error SC1090: narrowing '.filter' with a hand-written type predicate (a written 'x is T' is an unchecked assertion nothing validates at runtime — keep the receiver's element type instead — annotate the callback's return ': boolean' (the checker then skips the predicate) or annotate the result with the receiver's own element type — and narrow the elements after) is not supported yet - - 13 | const ys: (string | undefined)[] = ["a", undefined]; - 14 | const strings = ys.filter((x): x is string => typeof x === "string"); - | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - 15 | console.log(strings.length); - -filter-narrow.ts:21:28 - error SC1090: narrowing '.filter' through a callback VALUE (only an inline callback whose predicate the checker inferred can re-tag — keep the receiver's element type instead — annotate the callback's return ': boolean' (the checker then skips the predicate) or annotate the result with the receiver's own element type — and narrow the elements after) is not supported yet - - 20 | } - 21 | const viaValue = ys.filter(isString); - | ^~~~~~~~ - 22 | console.log(viaValue.length); \ No newline at end of file diff --git a/tests/harness/corpus-costs/7007-filter-written-type-predicates.ts.json b/tests/harness/corpus-costs/7007-filter-written-type-predicates.ts.json new file mode 100644 index 000000000..188fa7fab --- /dev/null +++ b/tests/harness/corpus-costs/7007-filter-written-type-predicates.ts.json @@ -0,0 +1 @@ +{ "plain": 3510, "san": 2873 } From 423a70a1828fcda1590df5e264251c2c5c0f537f Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Thu, 22 Oct 2026 05:51:16 -0700 Subject: [PATCH 024/197] Compile zod 3 natively - An untyped JavaScript receiver whose declaring file assigns it an own toString (the namespace downlevel's errorUtil.toString) calls that function instead of the string-typed toString dispatch: zod's number checks threw a failed string check on its undefined result. - Number.parseInt with one JavaScript argument that only the global parser's forms lower compiles through those forms. - A JavaScript then whose declared result has no mapping settles with the handler's value as a checked-dynamic value. New npm cases: zod3 (real zod 3.25.67, v3 ESM build and types), and js-own-tostring. Co-Authored-By: Claude Opus 5.5 --- .../src/frontend/lowering/builtins/numbers.ts | 9 +- .../src/frontend/lowering/lower-calls.ts | 54 +- .../npm/cases/js-own-tostring/main.ts.json | 6 + .../fixtures/npm/cases/zod3/main.ts.json | 6 + .../npm/cases/js-own-tostring/main.ts | 14 + .../node_modules/nsutil/dist/errorUtil.d.ts | 9 + .../node_modules/nsutil/dist/errorUtil.js | 5 + .../node_modules/nsutil/dist/index.d.ts | 6 + .../node_modules/nsutil/dist/index.js | 14 + .../node_modules/nsutil/package.json | 1 + .../npm/cases/js-own-tostring/package.json | 1 + tests/fixtures/npm/cases/zod3/main.ts | 115 + .../npm/cases/zod3/node_modules/zod/LICENSE | 21 + .../zod3/node_modules/zod/dist/esm/index.js | 3 + .../node_modules/zod/dist/esm/package.json | 3 + .../node_modules/zod/dist/esm/v3/ZodError.js | 132 + .../node_modules/zod/dist/esm/v3/errors.js | 9 + .../node_modules/zod/dist/esm/v3/external.js | 6 + .../zod/dist/esm/v3/helpers/enumUtil.js | 1 + .../zod/dist/esm/v3/helpers/errorUtil.js | 6 + .../zod/dist/esm/v3/helpers/parseUtil.js | 109 + .../zod/dist/esm/v3/helpers/partialUtil.js | 1 + .../zod/dist/esm/v3/helpers/typeAliases.js | 1 + .../zod/dist/esm/v3/helpers/util.js | 133 + .../node_modules/zod/dist/esm/v3/index.js | 4 + .../zod/dist/esm/v3/locales/en.js | 107 + .../zod/dist/esm/v3/standard-schema.js | 1 + .../node_modules/zod/dist/esm/v3/types.js | 3691 +++++++++++++++++ .../node_modules/zod/dist/types/index.d.ts | 3 + .../zod/dist/types/v3/ZodError.d.ts | 164 + .../zod/dist/types/v3/errors.d.ts | 5 + .../zod/dist/types/v3/external.d.ts | 6 + .../zod/dist/types/v3/helpers/enumUtil.d.ts | 8 + .../zod/dist/types/v3/helpers/errorUtil.d.ts | 9 + .../zod/dist/types/v3/helpers/parseUtil.d.ts | 78 + .../dist/types/v3/helpers/partialUtil.d.ts | 8 + .../dist/types/v3/helpers/typeAliases.d.ts | 2 + .../zod/dist/types/v3/helpers/util.d.ts | 85 + .../node_modules/zod/dist/types/v3/index.d.ts | 4 + .../zod/dist/types/v3/locales/en.d.ts | 3 + .../zod/dist/types/v3/standard-schema.d.ts | 102 + .../node_modules/zod/dist/types/v3/types.d.ts | 1031 +++++ .../cases/zod3/node_modules/zod/package.json | 120 + tests/fixtures/npm/cases/zod3/package.json | 1 + 44 files changed, 6093 insertions(+), 4 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/js-own-tostring/main.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/zod3/main.ts.json create mode 100644 tests/fixtures/npm/cases/js-own-tostring/main.ts create mode 100644 tests/fixtures/npm/cases/js-own-tostring/node_modules/nsutil/dist/errorUtil.d.ts create mode 100644 tests/fixtures/npm/cases/js-own-tostring/node_modules/nsutil/dist/errorUtil.js create mode 100644 tests/fixtures/npm/cases/js-own-tostring/node_modules/nsutil/dist/index.d.ts create mode 100644 tests/fixtures/npm/cases/js-own-tostring/node_modules/nsutil/dist/index.js create mode 100644 tests/fixtures/npm/cases/js-own-tostring/node_modules/nsutil/package.json create mode 100644 tests/fixtures/npm/cases/js-own-tostring/package.json create mode 100644 tests/fixtures/npm/cases/zod3/main.ts create mode 100644 tests/fixtures/npm/cases/zod3/node_modules/zod/LICENSE create mode 100644 tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/index.js create mode 100644 tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/package.json create mode 100644 tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/ZodError.js create mode 100644 tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/errors.js create mode 100644 tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/external.js create mode 100644 tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/helpers/enumUtil.js create mode 100644 tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/helpers/errorUtil.js create mode 100644 tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/helpers/parseUtil.js create mode 100644 tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/helpers/partialUtil.js create mode 100644 tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/helpers/typeAliases.js create mode 100644 tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/helpers/util.js create mode 100644 tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/index.js create mode 100644 tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/locales/en.js create mode 100644 tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/standard-schema.js create mode 100644 tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/types.js create mode 100644 tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/index.d.ts create mode 100644 tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/ZodError.d.ts create mode 100644 tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/errors.d.ts create mode 100644 tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/external.d.ts create mode 100644 tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/helpers/enumUtil.d.ts create mode 100644 tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/helpers/errorUtil.d.ts create mode 100644 tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/helpers/parseUtil.d.ts create mode 100644 tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/helpers/partialUtil.d.ts create mode 100644 tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/helpers/typeAliases.d.ts create mode 100644 tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/helpers/util.d.ts create mode 100644 tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/index.d.ts create mode 100644 tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/locales/en.d.ts create mode 100644 tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/standard-schema.d.ts create mode 100644 tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/types.d.ts create mode 100644 tests/fixtures/npm/cases/zod3/node_modules/zod/package.json create mode 100644 tests/fixtures/npm/cases/zod3/package.json diff --git a/packages/compiler/src/frontend/lowering/builtins/numbers.ts b/packages/compiler/src/frontend/lowering/builtins/numbers.ts index 1ff35fbb5..9764216b7 100644 --- a/packages/compiler/src/frontend/lowering/builtins/numbers.ts +++ b/packages/compiler/src/frontend/lowering/builtins/numbers.ts @@ -1,7 +1,7 @@ import * as ts from "../../ts7/adapter.js"; import { type Lowerer, own } from "../lowerer.js"; import { lowerCheckedPredicateValue } from "../lower-builtin-values.js"; -import { locOf } from "../../program.js"; +import { isJsSourceFile, locOf } from "../../program.js"; import { BOOL, F64, STRING, type IrExpr, type IrLibFn } from "../../../ir/ir.js"; import { lowerBuiltinValuePreservingUndefined, lowerOptionalNumberPredicate } from "./arguments.js"; import { tryLowerExpression } from "../expressions/try-lower-expression.js"; @@ -104,6 +104,13 @@ export function lowerNumberStaticCall( : { kind: "numLit", value: 0, type: F64, loc }; return { kind: "libCall", fn: "num.parseInt", args: [text, radix], type: F64, loc }; } + // JavaScript whose argument does not lower on its own (zod's + // `Number.parseInt(val.toFixed(n).replace(".", ""))` over an untyped + // parameter): the global parser's own JavaScript forms. + if (probed === null && member === "parseInt" && isJsSourceFile(ts.getSourceFileOfNode(call))) { + const parsed = lowerStaticNumberGlobal(lowerer, member, call, call.arguments.nodes, loc); + if (parsed !== null) return parsed; + } const argText = probed !== null ? lowerer.fmt(probed.type) diff --git a/packages/compiler/src/frontend/lowering/lower-calls.ts b/packages/compiler/src/frontend/lowering/lower-calls.ts index 39b7207c0..7e104c2f1 100644 --- a/packages/compiler/src/frontend/lowering/lower-calls.ts +++ b/packages/compiler/src/frontend/lowering/lower-calls.ts @@ -3776,7 +3776,14 @@ function lowerDynReceiverMethodCall( } // The argument is a radix for bigint and an encoding for Buffer. // Preserve its runtime value so receiver dispatch can choose correctly. - if (access.name.text === "toString" && call.arguments.nodes.length <= 1) { + // An authored own `toString` member (zod's `errorUtil.toString`, which + // answers string | undefined) is an ordinary method call instead. + if ( + access.name.text === "toString" && + call.arguments.nodes.length <= 1 && + !ownCallable && + !(checkerUntyped && jsAssignsOwnToString(lowerer, access.expression)) + ) { const argument = call.arguments.nodes[0]; return { kind: "libCall", @@ -3886,12 +3893,22 @@ function lowerDynReceiverMethodCall( // The same holds when only the MEMBER is checker-untyped: TypeScript // importing chalk's JavaScript source types the default export as a // plain function, so `chalk.bold` reads as `any` on a typed receiver. + const ownToString = + access.name.text === "toString" && + checkerUntyped && + jsAssignsOwnToString(lowerer, access.expression); const stringOnlyName = (checkerUntyped || (lowerer.typeOf(access).flags & (ts.TypeFlags.Any | ts.TypeFlags.Unknown)) !== 0) && !ownCallable && DYN_STRING_PROTO_ONLY_NAMES.has(access.name.text); - if (DYN_PROTO_METHOD_NAMES.has(access.name.text) && !ownCallable && !stringOnlyName) return null; + if ( + DYN_PROTO_METHOD_NAMES.has(access.name.text) && + !ownCallable && + !ownToString && + !stringOnlyName + ) + return null; // Optional forms (`obj.cb?.()`, `obj?.cb()`) belong to the chain // machinery's short-circuit semantics — not modeled here yet. if (call.questionDotToken || access.questionDotToken) return null; @@ -6175,7 +6192,21 @@ export function lowerPromiseMethodCall( `then handlers whose parameter is not the settled value's type (expected '${lowerer.fmt(inner)}', got '${lowerer.fmt(param)}')`, ); } - const resultT = lowerer.mapTypeOf(lowerer.typeOf(call)); + let resultT = lowerer.mapTypeOf(lowerer.typeOf(call)); + // JavaScript whose declared result type has no mapping (zod's + // `Promise.all(items).then((r) => ParseStatus.mergeArray(status, r))`, + // typed by a .d.ts union): the handler's value settles the promise as + // a checked-dynamic value, checked where it is used. + if ( + resultT?.kind !== "promise" && + isJsSourceFile(ts.getSourceFileOfNode(call)) && + cb.type.kind === "func" && + cb.type.ret.kind !== "promise" && + cb.type.ret.kind !== "void" && + (cb.type.ret.kind === "dyn" || lowerer.dynConvertible(cb.type.ret)) + ) { + resultT = { kind: "promise", inner: DYN }; + } if (resultT?.kind !== "promise") { lowerer.noLowering( "then with this handler's result type", @@ -10939,3 +10970,20 @@ function fenceClassCopyIntoFreshObject( `Object.assign copying a '${lowerer.fmt(rawSource.type)}' instance into a fresh object typed as the class (the copy is a plain object holding the instance's own fields, not an instance — build the object literal from the fields you need)`, ); } + +/** True when an untyped JavaScript receiver names a binding whose declaring + * file assigns it an own `toString` (the TypeScript namespace downlevel + * `errorUtil.toString = (message) => ...` in zod): its calls answer what + * that function returns, not Object.prototype.toString's string. */ +function jsAssignsOwnToString(lowerer: Lowerer, receiver: ts.Expression): boolean { + if (!ts.isIdentifier(receiver)) return false; + let sym = lowerer.checker.getSymbolAtLocation(receiver); + if (sym !== undefined && (sym.flags & ts.SymbolFlags.Alias) !== 0) + sym = lowerer.checker.getAliasedSymbol(sym); + const decl = sym?.valueDeclaration ?? sym?.declarations[0]; + if (decl === undefined) return false; + const sf = ts.getSourceFileOfNode(decl); + if (!isJsSourceFile(sf)) return false; + const name = ts.symbolName(sym!); + return new RegExp(`\\b${name.replace(/[$]/g, "\\$")}\\.toString\\s*=[^=]`).test(sf.text); +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/js-own-tostring/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/js-own-tostring/main.ts.json new file mode 100644 index 000000000..eacdfc1d4 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/js-own-tostring/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/js-own-tostring/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/zod3/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/zod3/main.ts.json new file mode 100644 index 000000000..c9ff338fa --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/zod3/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/zod3/main.ts" + ], + "diags": [] +} diff --git a/tests/fixtures/npm/cases/js-own-tostring/main.ts b/tests/fixtures/npm/cases/js-own-tostring/main.ts new file mode 100644 index 000000000..279e2fd66 --- /dev/null +++ b/tests/fixtures/npm/cases/js-own-tostring/main.ts @@ -0,0 +1,14 @@ +// A JavaScript package (the TypeScript namespace downlevel zod uses) whose +// namespace object assigns its own `toString` returning string | undefined; +// Number.parseInt with one argument; and a `then` whose handler result the +// declared union types (zod's ParseStatus.mergeArray pattern). +import { check, direct, mergeLater, wholePart } from "nsutil"; + +console.log(String(direct("a")), String(direct({ message: "b" })), String(direct(undefined))); +console.log(JSON.stringify(check(undefined)), JSON.stringify(check("m"))); +console.log(wholePart(1.23), wholePart(0.35), wholePart(-4.5), wholePart(12)); +async function main(): Promise { + console.log(JSON.stringify(await mergeLater([Promise.resolve(1), Promise.resolve("two")], { ok: true }))); + console.log(JSON.stringify(await mergeLater([Promise.resolve(3), Promise.resolve("four")], { ok: false }))); +} +void main(); diff --git a/tests/fixtures/npm/cases/js-own-tostring/node_modules/nsutil/dist/errorUtil.d.ts b/tests/fixtures/npm/cases/js-own-tostring/node_modules/nsutil/dist/errorUtil.d.ts new file mode 100644 index 000000000..bc9b033ff --- /dev/null +++ b/tests/fixtures/npm/cases/js-own-tostring/node_modules/nsutil/dist/errorUtil.d.ts @@ -0,0 +1,9 @@ +export declare namespace errorUtil { + type ErrMessage = string | { + message?: string | undefined; + }; + const errToObj: (message?: ErrMessage) => { + message?: string | undefined; + }; + const toString: (message?: ErrMessage) => string | undefined; +} diff --git a/tests/fixtures/npm/cases/js-own-tostring/node_modules/nsutil/dist/errorUtil.js b/tests/fixtures/npm/cases/js-own-tostring/node_modules/nsutil/dist/errorUtil.js new file mode 100644 index 000000000..1b48d927b --- /dev/null +++ b/tests/fixtures/npm/cases/js-own-tostring/node_modules/nsutil/dist/errorUtil.js @@ -0,0 +1,5 @@ +export var errorUtil; +(function (errorUtil) { + errorUtil.errToObj = (message) => typeof message === "string" ? { message } : message || {}; + errorUtil.toString = (message) => typeof message === "string" ? message : message?.message; +})(errorUtil || (errorUtil = {})); diff --git a/tests/fixtures/npm/cases/js-own-tostring/node_modules/nsutil/dist/index.d.ts b/tests/fixtures/npm/cases/js-own-tostring/node_modules/nsutil/dist/index.d.ts new file mode 100644 index 000000000..f4eea833a --- /dev/null +++ b/tests/fixtures/npm/cases/js-own-tostring/node_modules/nsutil/dist/index.d.ts @@ -0,0 +1,6 @@ +import { errorUtil } from "./errorUtil.js"; +export declare function check(message?: errorUtil.ErrMessage): { kind: "int"; message?: string }; +export declare function direct(message?: errorUtil.ErrMessage): string | undefined; +export declare function wholePart(value: number): number; +export type Merged = { status: "valid"; value: unknown[] } | { status: "aborted" }; +export declare function mergeLater(items: Promise[], status: { ok: boolean }): Promise; diff --git a/tests/fixtures/npm/cases/js-own-tostring/node_modules/nsutil/dist/index.js b/tests/fixtures/npm/cases/js-own-tostring/node_modules/nsutil/dist/index.js new file mode 100644 index 000000000..272a4631c --- /dev/null +++ b/tests/fixtures/npm/cases/js-own-tostring/node_modules/nsutil/dist/index.js @@ -0,0 +1,14 @@ +import { errorUtil } from "./errorUtil.js"; +export function check(message) { + return { kind: "int", message: errorUtil.toString(message) }; +} +export function direct(message) { + return errorUtil.toString(message); +} +export function wholePart(value) { + return Number.parseInt(String(value).replace(".", "")) + Number.parseInt("0x1f"); +} +export function mergeLater(items, status) { + return Promise.all(items).then((results) => status.ok ? { status: "valid", value: results } : INVALID); +} +const INVALID = Object.freeze({ status: "aborted" }); diff --git a/tests/fixtures/npm/cases/js-own-tostring/node_modules/nsutil/package.json b/tests/fixtures/npm/cases/js-own-tostring/node_modules/nsutil/package.json new file mode 100644 index 000000000..a2757b8a1 --- /dev/null +++ b/tests/fixtures/npm/cases/js-own-tostring/node_modules/nsutil/package.json @@ -0,0 +1 @@ +{ "name": "nsutil", "version": "1.0.0", "type": "module", "main": "./dist/index.js", "types": "./dist/index.d.ts" } diff --git a/tests/fixtures/npm/cases/js-own-tostring/package.json b/tests/fixtures/npm/cases/js-own-tostring/package.json new file mode 100644 index 000000000..5ffd9800b --- /dev/null +++ b/tests/fixtures/npm/cases/js-own-tostring/package.json @@ -0,0 +1 @@ +{ "type": "module" } diff --git a/tests/fixtures/npm/cases/zod3/main.ts b/tests/fixtures/npm/cases/zod3/main.ts new file mode 100644 index 000000000..d7f1a433a --- /dev/null +++ b/tests/fixtures/npm/cases/zod3/main.ts @@ -0,0 +1,115 @@ +// zod 3.25 (the real package, trimmed to its v3 ESM build and types) compiled +// natively: schemas built at module top level, parse and safeParse across the +// common schema kinds, number checks, issue paths, codes and messages, and +// async refinements through parseAsync/safeParseAsync. +import { z } from "zod"; + +const show = (label: string, value: unknown): void => { + console.log(label + " " + JSON.stringify(value)); +}; +const issues = (result: z.SafeParseReturnType): string => + result.success + ? "ok" + : result.error.issues.map((i) => `${i.path.join(".")}:${i.code}:${i.message}`).join(" | "); + +const Region = z.enum(["east", "west", "north"]); +const Job = z.object({ + id: z.string().min(3).max(32), + region: Region, + attempts: z.number().int().nonnegative().default(0), + tags: z.record(z.string()).optional(), + files: z.array(z.object({ path: z.string(), mode: z.number() })), + kind: z.union([z.literal("static"), z.literal("worker")]), + meta: z.object({ owner: z.string().nullable() }).partial(), +}); +type Job = z.infer; + +const job: Job = Job.parse({ + id: "job_1", + region: "east", + files: [{ path: "a.js", mode: 420 }], + kind: "worker", + meta: { owner: null }, +}); +show("job", job); +show("bad job", issues(Job.safeParse({ id: "x", region: "south", files: [{ path: 1 }], kind: "edge" }))); + +// The common schema kinds. +show("number", z.number().parse(3)); +show("default", z.number().default(0).parse(undefined)); +show("optional", z.string().optional().parse(undefined) ?? "undefined"); +show("nullable", z.string().nullable().parse(null)); +show("record", z.record(z.string()).parse({ a: "b" })); +show("array", z.array(z.string()).parse(["a"])); +show("union", z.union([z.literal("a"), z.literal("b")]).parse("b")); +show("partial", z.object({ a: z.string() }).partial().parse({})); +show("discriminated", z.discriminatedUnion("t", [z.object({ t: z.literal("x") })]).parse({ t: "x" })); +show("transform", z.string().transform((s) => s.length).parse("abc")); +show("wrong type", issues(z.object({ a: z.number() }).safeParse({ a: "x" }))); +show("email", issues(z.string().email().safeParse("nope"))); +show("boolean", z.boolean().parse(true)); +show("strict", issues(z.object({ a: z.string() }).strict().safeParse({ a: "x", b: 1 }))); +show("regex", z.string().regex(/^a/).parse("abc")); +show("coerce", z.coerce.number().parse("4")); +show("extend", z.object({ a: z.string() }).extend({ b: z.number() }).parse({ a: "x", b: 1 })); +show("refine", issues(z.string().refine((s) => s.length > 1, "short").safeParse("a"))); +show("enum", z.enum(["a", "b"]).parse("a")); +show("literal", z.literal("q").parse("q")); + +// Number checks, passing and failing, with and without messages. +show("int", z.number().int().parse(3)); +show("int fail", issues(z.number().int().safeParse(1.5))); +show("int message", issues(z.number().int("whole numbers only").safeParse(2.5))); +show("min", z.number().min(1).parse(3)); +show("min fail", issues(z.number().min(1, "too low").safeParse(0))); +show("max fail", issues(z.number().max(10).safeParse(11))); +show("gt lt", issues(z.number().gt(0).lt(5).safeParse(5))); +show("positive", issues(z.number().positive().safeParse(0))); +show("nonnegative", issues(z.number().nonnegative().safeParse(-1))); +show("multipleOf", z.number().multipleOf(2).parse(4)); +show("multipleOf fail", issues(z.number().multipleOf(0.1).safeParse(0.35))); +show("multipleOf float", z.number().multipleOf(0.01).parse(1.23)); +show("step", issues(z.number().step(3).safeParse(10))); +show("finite", issues(z.number().finite().safeParse(Infinity))); +show("safe", issues(z.number().safe().safeParse(2 ** 60))); +show("bounds", [z.number().min(2).max(9).minValue, z.number().min(2).max(9).maxValue, z.number().int().isInt]); + +try { + Job.parse({}); +} catch (error) { + show("thrown", error instanceof z.ZodError ? error.issues.length : "other"); +} + +// Async refinements. +const Unique = z.string().refine(async (name) => name !== "taken", { message: "name is taken" }); +const Account = z.object({ + name: Unique, + age: z.number().int().min(18), +}); + +async function main(): Promise { + show("async ok", await Account.parseAsync({ name: "free", age: 30 })); + show("async fail", issues(await Account.safeParseAsync({ name: "taken", age: 12.5 }))); + try { + await Unique.parseAsync("taken"); + } catch (error) { + show("async thrown", error instanceof z.ZodError ? error.issues[0]!.message : "other"); + } + try { + Unique.parse("free"); + } catch (error) { + show("sync on async", (error as Error).message); + } + const Slow = z.number().superRefine(async (n, ctx) => { + await new Promise((resolve) => setTimeout(resolve, 1)); + if (n % 2 !== 0) ctx.addIssue({ code: z.ZodIssueCode.custom, message: "odd" }); + }); + show("superRefine", issues(await Slow.safeParseAsync(3))); + show("transform async", await z.string().transform(async (s) => s.toUpperCase()).parseAsync("up")); + show("array async", issues(await z.array(Unique).safeParseAsync(["a", "taken", "b"]))); + show("array async ok", await z.array(Unique).parseAsync(["x", "y"])); + show("tuple async", issues(await z.tuple([z.string(), Unique]).safeParseAsync(["a", "taken"]))); + show("tuple async ok", await z.tuple([z.number(), Unique]).parseAsync([1, "z"])); +} + +void main(); diff --git a/tests/fixtures/npm/cases/zod3/node_modules/zod/LICENSE b/tests/fixtures/npm/cases/zod3/node_modules/zod/LICENSE new file mode 100644 index 000000000..c06579627 --- /dev/null +++ b/tests/fixtures/npm/cases/zod3/node_modules/zod/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Colin McDonnell + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/index.js b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/index.js new file mode 100644 index 000000000..8352ab713 --- /dev/null +++ b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/index.js @@ -0,0 +1,3 @@ +import z3 from "./v3/index.js"; +export * from "./v3/index.js"; +export default z3; diff --git a/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/package.json b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/package.json new file mode 100644 index 000000000..aead43de3 --- /dev/null +++ b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/package.json @@ -0,0 +1,3 @@ +{ + "type": "module" +} \ No newline at end of file diff --git a/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/ZodError.js b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/ZodError.js new file mode 100644 index 000000000..c72ce5288 --- /dev/null +++ b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/ZodError.js @@ -0,0 +1,132 @@ +import { util } from "./helpers/util.js"; +export const ZodIssueCode = util.arrayToEnum([ + "invalid_type", + "invalid_literal", + "custom", + "invalid_union", + "invalid_union_discriminator", + "invalid_enum_value", + "unrecognized_keys", + "invalid_arguments", + "invalid_return_type", + "invalid_date", + "invalid_string", + "too_small", + "too_big", + "invalid_intersection_types", + "not_multiple_of", + "not_finite", +]); +export const quotelessJson = (obj) => { + const json = JSON.stringify(obj, null, 2); + return json.replace(/"([^"]+)":/g, "$1:"); +}; +export class ZodError extends Error { + get errors() { + return this.issues; + } + constructor(issues) { + super(); + this.issues = []; + this.addIssue = (sub) => { + this.issues = [...this.issues, sub]; + }; + this.addIssues = (subs = []) => { + this.issues = [...this.issues, ...subs]; + }; + const actualProto = new.target.prototype; + if (Object.setPrototypeOf) { + // eslint-disable-next-line ban/ban + Object.setPrototypeOf(this, actualProto); + } + else { + this.__proto__ = actualProto; + } + this.name = "ZodError"; + this.issues = issues; + } + format(_mapper) { + const mapper = _mapper || + function (issue) { + return issue.message; + }; + const fieldErrors = { _errors: [] }; + const processError = (error) => { + for (const issue of error.issues) { + if (issue.code === "invalid_union") { + issue.unionErrors.map(processError); + } + else if (issue.code === "invalid_return_type") { + processError(issue.returnTypeError); + } + else if (issue.code === "invalid_arguments") { + processError(issue.argumentsError); + } + else if (issue.path.length === 0) { + fieldErrors._errors.push(mapper(issue)); + } + else { + let curr = fieldErrors; + let i = 0; + while (i < issue.path.length) { + const el = issue.path[i]; + const terminal = i === issue.path.length - 1; + if (!terminal) { + curr[el] = curr[el] || { _errors: [] }; + // if (typeof el === "string") { + // curr[el] = curr[el] || { _errors: [] }; + // } else if (typeof el === "number") { + // const errorArray: any = []; + // errorArray._errors = []; + // curr[el] = curr[el] || errorArray; + // } + } + else { + curr[el] = curr[el] || { _errors: [] }; + curr[el]._errors.push(mapper(issue)); + } + curr = curr[el]; + i++; + } + } + } + }; + processError(this); + return fieldErrors; + } + static assert(value) { + if (!(value instanceof ZodError)) { + throw new Error(`Not a ZodError: ${value}`); + } + } + toString() { + return this.message; + } + get message() { + return JSON.stringify(this.issues, util.jsonStringifyReplacer, 2); + } + get isEmpty() { + return this.issues.length === 0; + } + flatten(mapper = (issue) => issue.message) { + const fieldErrors = {}; + const formErrors = []; + for (const sub of this.issues) { + if (sub.path.length > 0) { + fieldErrors[sub.path[0]] = fieldErrors[sub.path[0]] || []; + fieldErrors[sub.path[0]].push(mapper(sub)); + } + else { + formErrors.push(mapper(sub)); + } + } + return { formErrors, fieldErrors }; + } + get formErrors() { + return this.flatten(); + } +} +ZodError.create = (issues) => { + const error = new ZodError(issues); + return error; +}; diff --git a/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/errors.js b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/errors.js new file mode 100644 index 000000000..26fdf628a --- /dev/null +++ b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/errors.js @@ -0,0 +1,9 @@ +import defaultErrorMap from "./locales/en.js"; +let overrideErrorMap = defaultErrorMap; +export { defaultErrorMap }; +export function setErrorMap(map) { + overrideErrorMap = map; +} +export function getErrorMap() { + return overrideErrorMap; +} diff --git a/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/external.js b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/external.js new file mode 100644 index 000000000..f0a4be4c7 --- /dev/null +++ b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/external.js @@ -0,0 +1,6 @@ +export * from "./errors.js"; +export * from "./helpers/parseUtil.js"; +export * from "./helpers/typeAliases.js"; +export * from "./helpers/util.js"; +export * from "./types.js"; +export * from "./ZodError.js"; diff --git a/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/helpers/enumUtil.js b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/helpers/enumUtil.js new file mode 100644 index 000000000..cb0ff5c3b --- /dev/null +++ b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/helpers/enumUtil.js @@ -0,0 +1 @@ +export {}; diff --git a/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/helpers/errorUtil.js b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/helpers/errorUtil.js new file mode 100644 index 000000000..e1640d1a4 --- /dev/null +++ b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/helpers/errorUtil.js @@ -0,0 +1,6 @@ +export var errorUtil; +(function (errorUtil) { + errorUtil.errToObj = (message) => typeof message === "string" ? { message } : message || {}; + // biome-ignore lint: + errorUtil.toString = (message) => typeof message === "string" ? message : message?.message; +})(errorUtil || (errorUtil = {})); diff --git a/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/helpers/parseUtil.js b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/helpers/parseUtil.js new file mode 100644 index 000000000..91b2afb81 --- /dev/null +++ b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/helpers/parseUtil.js @@ -0,0 +1,109 @@ +import { getErrorMap } from "../errors.js"; +import defaultErrorMap from "../locales/en.js"; +export const makeIssue = (params) => { + const { data, path, errorMaps, issueData } = params; + const fullPath = [...path, ...(issueData.path || [])]; + const fullIssue = { + ...issueData, + path: fullPath, + }; + if (issueData.message !== undefined) { + return { + ...issueData, + path: fullPath, + message: issueData.message, + }; + } + let errorMessage = ""; + const maps = errorMaps + .filter((m) => !!m) + .slice() + .reverse(); + for (const map of maps) { + errorMessage = map(fullIssue, { data, defaultError: errorMessage }).message; + } + return { + ...issueData, + path: fullPath, + message: errorMessage, + }; +}; +export const EMPTY_PATH = []; +export function addIssueToContext(ctx, issueData) { + const overrideMap = getErrorMap(); + const issue = makeIssue({ + issueData: issueData, + data: ctx.data, + path: ctx.path, + errorMaps: [ + ctx.common.contextualErrorMap, // contextual error map is first priority + ctx.schemaErrorMap, // then schema-bound map if available + overrideMap, // then global override map + overrideMap === defaultErrorMap ? undefined : defaultErrorMap, // then global default map + ].filter((x) => !!x), + }); + ctx.common.issues.push(issue); +} +export class ParseStatus { + constructor() { + this.value = "valid"; + } + dirty() { + if (this.value === "valid") + this.value = "dirty"; + } + abort() { + if (this.value !== "aborted") + this.value = "aborted"; + } + static mergeArray(status, results) { + const arrayValue = []; + for (const s of results) { + if (s.status === "aborted") + return INVALID; + if (s.status === "dirty") + status.dirty(); + arrayValue.push(s.value); + } + return { status: status.value, value: arrayValue }; + } + static async mergeObjectAsync(status, pairs) { + const syncPairs = []; + for (const pair of pairs) { + const key = await pair.key; + const value = await pair.value; + syncPairs.push({ + key, + value, + }); + } + return ParseStatus.mergeObjectSync(status, syncPairs); + } + static mergeObjectSync(status, pairs) { + const finalObject = {}; + for (const pair of pairs) { + const { key, value } = pair; + if (key.status === "aborted") + return INVALID; + if (value.status === "aborted") + return INVALID; + if (key.status === "dirty") + status.dirty(); + if (value.status === "dirty") + status.dirty(); + if (key.value !== "__proto__" && (typeof value.value !== "undefined" || pair.alwaysSet)) { + finalObject[key.value] = value.value; + } + } + return { status: status.value, value: finalObject }; + } +} +export const INVALID = Object.freeze({ + status: "aborted", +}); +export const DIRTY = (value) => ({ status: "dirty", value }); +export const OK = (value) => ({ status: "valid", value }); +export const isAborted = (x) => x.status === "aborted"; +export const isDirty = (x) => x.status === "dirty"; +export const isValid = (x) => x.status === "valid"; +export const isAsync = (x) => typeof Promise !== "undefined" && x instanceof Promise; diff --git a/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/helpers/partialUtil.js b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/helpers/partialUtil.js new file mode 100644 index 000000000..cb0ff5c3b --- /dev/null +++ b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/helpers/partialUtil.js @@ -0,0 +1 @@ +export {}; diff --git a/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/helpers/typeAliases.js b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/helpers/typeAliases.js new file mode 100644 index 000000000..cb0ff5c3b --- /dev/null +++ b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/helpers/typeAliases.js @@ -0,0 +1 @@ +export {}; diff --git a/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/helpers/util.js b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/helpers/util.js new file mode 100644 index 000000000..b9641f29b --- /dev/null +++ b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/helpers/util.js @@ -0,0 +1,133 @@ +export var util; +(function (util) { + util.assertEqual = (_) => { }; + function assertIs(_arg) { } + util.assertIs = assertIs; + function assertNever(_x) { + throw new Error(); + } + util.assertNever = assertNever; + util.arrayToEnum = (items) => { + const obj = {}; + for (const item of items) { + obj[item] = item; + } + return obj; + }; + util.getValidEnumValues = (obj) => { + const validKeys = util.objectKeys(obj).filter((k) => typeof obj[obj[k]] !== "number"); + const filtered = {}; + for (const k of validKeys) { + filtered[k] = obj[k]; + } + return util.objectValues(filtered); + }; + util.objectValues = (obj) => { + return util.objectKeys(obj).map(function (e) { + return obj[e]; + }); + }; + util.objectKeys = typeof Object.keys === "function" // eslint-disable-line ban/ban + ? (obj) => Object.keys(obj) // eslint-disable-line ban/ban + : (object) => { + const keys = []; + for (const key in object) { + if (Object.prototype.hasOwnProperty.call(object, key)) { + keys.push(key); + } + } + return keys; + }; + util.find = (arr, checker) => { + for (const item of arr) { + if (checker(item)) + return item; + } + return undefined; + }; + util.isInteger = typeof Number.isInteger === "function" + ? (val) => Number.isInteger(val) // eslint-disable-line ban/ban + : (val) => typeof val === "number" && Number.isFinite(val) && Math.floor(val) === val; + function joinValues(array, separator = " | ") { + return array.map((val) => (typeof val === "string" ? `'${val}'` : val)).join(separator); + } + util.joinValues = joinValues; + util.jsonStringifyReplacer = (_, value) => { + if (typeof value === "bigint") { + return value.toString(); + } + return value; + }; +})(util || (util = {})); +export var objectUtil; +(function (objectUtil) { + objectUtil.mergeShapes = (first, second) => { + return { + ...first, + ...second, // second overwrites first + }; + }; +})(objectUtil || (objectUtil = {})); +export const ZodParsedType = util.arrayToEnum([ + "string", + "nan", + "number", + "integer", + "float", + "boolean", + "date", + "bigint", + "symbol", + "function", + "undefined", + "null", + "array", + "object", + "unknown", + "promise", + "void", + "never", + "map", + "set", +]); +export const getParsedType = (data) => { + const t = typeof data; + switch (t) { + case "undefined": + return ZodParsedType.undefined; + case "string": + return ZodParsedType.string; + case "number": + return Number.isNaN(data) ? ZodParsedType.nan : ZodParsedType.number; + case "boolean": + return ZodParsedType.boolean; + case "function": + return ZodParsedType.function; + case "bigint": + return ZodParsedType.bigint; + case "symbol": + return ZodParsedType.symbol; + case "object": + if (Array.isArray(data)) { + return ZodParsedType.array; + } + if (data === null) { + return ZodParsedType.null; + } + if (data.then && typeof data.then === "function" && data.catch && typeof data.catch === "function") { + return ZodParsedType.promise; + } + if (typeof Map !== "undefined" && data instanceof Map) { + return ZodParsedType.map; + } + if (typeof Set !== "undefined" && data instanceof Set) { + return ZodParsedType.set; + } + if (typeof Date !== "undefined" && data instanceof Date) { + return ZodParsedType.date; + } + return ZodParsedType.object; + default: + return ZodParsedType.unknown; + } +}; diff --git a/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/index.js b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/index.js new file mode 100644 index 000000000..a9dab5738 --- /dev/null +++ b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/index.js @@ -0,0 +1,4 @@ +import * as z from "./external.js"; +export * from "./external.js"; +export { z }; +export default z; diff --git a/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/locales/en.js b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/locales/en.js new file mode 100644 index 000000000..3e97ad51a --- /dev/null +++ b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/locales/en.js @@ -0,0 +1,107 @@ +import { ZodIssueCode } from "../ZodError.js"; +import { util, ZodParsedType } from "../helpers/util.js"; +const errorMap = (issue, _ctx) => { + let message; + switch (issue.code) { + case ZodIssueCode.invalid_type: + if (issue.received === ZodParsedType.undefined) { + message = "Required"; + } + else { + message = `Expected ${issue.expected}, received ${issue.received}`; + } + break; + case ZodIssueCode.invalid_literal: + message = `Invalid literal value, expected ${JSON.stringify(issue.expected, util.jsonStringifyReplacer)}`; + break; + case ZodIssueCode.unrecognized_keys: + message = `Unrecognized key(s) in object: ${util.joinValues(issue.keys, ", ")}`; + break; + case ZodIssueCode.invalid_union: + message = `Invalid input`; + break; + case ZodIssueCode.invalid_union_discriminator: + message = `Invalid discriminator value. Expected ${util.joinValues(issue.options)}`; + break; + case ZodIssueCode.invalid_enum_value: + message = `Invalid enum value. Expected ${util.joinValues(issue.options)}, received '${issue.received}'`; + break; + case ZodIssueCode.invalid_arguments: + message = `Invalid function arguments`; + break; + case ZodIssueCode.invalid_return_type: + message = `Invalid function return type`; + break; + case ZodIssueCode.invalid_date: + message = `Invalid date`; + break; + case ZodIssueCode.invalid_string: + if (typeof issue.validation === "object") { + if ("includes" in issue.validation) { + message = `Invalid input: must include "${issue.validation.includes}"`; + if (typeof issue.validation.position === "number") { + message = `${message} at one or more positions greater than or equal to ${issue.validation.position}`; + } + } + else if ("startsWith" in issue.validation) { + message = `Invalid input: must start with "${issue.validation.startsWith}"`; + } + else if ("endsWith" in issue.validation) { + message = `Invalid input: must end with "${issue.validation.endsWith}"`; + } + else { + util.assertNever(issue.validation); + } + } + else if (issue.validation !== "regex") { + message = `Invalid ${issue.validation}`; + } + else { + message = "Invalid"; + } + break; + case ZodIssueCode.too_small: + if (issue.type === "array") + message = `Array must contain ${issue.exact ? "exactly" : issue.inclusive ? `at least` : `more than`} ${issue.minimum} element(s)`; + else if (issue.type === "string") + message = `String must contain ${issue.exact ? "exactly" : issue.inclusive ? `at least` : `over`} ${issue.minimum} character(s)`; + else if (issue.type === "number") + message = `Number must be ${issue.exact ? `exactly equal to ` : issue.inclusive ? `greater than or equal to ` : `greater than `}${issue.minimum}`; + else if (issue.type === "date") + message = `Date must be ${issue.exact ? `exactly equal to ` : issue.inclusive ? `greater than or equal to ` : `greater than `}${new Date(Number(issue.minimum))}`; + else + message = "Invalid input"; + break; + case ZodIssueCode.too_big: + if (issue.type === "array") + message = `Array must contain ${issue.exact ? `exactly` : issue.inclusive ? `at most` : `less than`} ${issue.maximum} element(s)`; + else if (issue.type === "string") + message = `String must contain ${issue.exact ? `exactly` : issue.inclusive ? `at most` : `under`} ${issue.maximum} character(s)`; + else if (issue.type === "number") + message = `Number must be ${issue.exact ? `exactly` : issue.inclusive ? `less than or equal to` : `less than`} ${issue.maximum}`; + else if (issue.type === "bigint") + message = `BigInt must be ${issue.exact ? `exactly` : issue.inclusive ? `less than or equal to` : `less than`} ${issue.maximum}`; + else if (issue.type === "date") + message = `Date must be ${issue.exact ? `exactly` : issue.inclusive ? `smaller than or equal to` : `smaller than`} ${new Date(Number(issue.maximum))}`; + else + message = "Invalid input"; + break; + case ZodIssueCode.custom: + message = `Invalid input`; + break; + case ZodIssueCode.invalid_intersection_types: + message = `Intersection results could not be merged`; + break; + case ZodIssueCode.not_multiple_of: + message = `Number must be a multiple of ${issue.multipleOf}`; + break; + case ZodIssueCode.not_finite: + message = "Number must be finite"; + break; + default: + message = _ctx.defaultError; + util.assertNever(issue); + } + return { message }; +}; +export default errorMap; diff --git a/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/standard-schema.js b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/standard-schema.js new file mode 100644 index 000000000..cb0ff5c3b --- /dev/null +++ b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/standard-schema.js @@ -0,0 +1 @@ +export {}; diff --git a/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/types.js b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/types.js new file mode 100644 index 000000000..de1e6213e --- /dev/null +++ b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/esm/v3/types.js @@ -0,0 +1,3691 @@ +import { ZodError, ZodIssueCode, } from "./ZodError.js"; +import { defaultErrorMap, getErrorMap } from "./errors.js"; +import { errorUtil } from "./helpers/errorUtil.js"; +import { DIRTY, INVALID, OK, ParseStatus, addIssueToContext, isAborted, isAsync, isDirty, isValid, makeIssue, } from "./helpers/parseUtil.js"; +import { util, ZodParsedType, getParsedType } from "./helpers/util.js"; +class ParseInputLazyPath { + constructor(parent, value, path, key) { + this._cachedPath = []; + this.parent = parent; + this.data = value; + this._path = path; + this._key = key; + } + get path() { + if (!this._cachedPath.length) { + if (Array.isArray(this._key)) { + this._cachedPath.push(...this._path, ...this._key); + } + else { + this._cachedPath.push(...this._path, this._key); + } + } + return this._cachedPath; + } +} +const handleResult = (ctx, result) => { + if (isValid(result)) { + return { success: true, data: result.value }; + } + else { + if (!ctx.common.issues.length) { + throw new Error("Validation failed but no issues detected."); + } + return { + success: false, + get error() { + if (this._error) + return this._error; + const error = new ZodError(ctx.common.issues); + this._error = error; + return this._error; + }, + }; + } +}; +function processCreateParams(params) { + if (!params) + return {}; + const { errorMap, invalid_type_error, required_error, description } = params; + if (errorMap && (invalid_type_error || required_error)) { + throw new Error(`Can't use "invalid_type_error" or "required_error" in conjunction with custom error map.`); + } + if (errorMap) + return { errorMap: errorMap, description }; + const customMap = (iss, ctx) => { + const { message } = params; + if (iss.code === "invalid_enum_value") { + return { message: message ?? ctx.defaultError }; + } + if (typeof ctx.data === "undefined") { + return { message: message ?? required_error ?? ctx.defaultError }; + } + if (iss.code !== "invalid_type") + return { message: ctx.defaultError }; + return { message: message ?? invalid_type_error ?? ctx.defaultError }; + }; + return { errorMap: customMap, description }; +} +export class ZodType { + get description() { + return this._def.description; + } + _getType(input) { + return getParsedType(input.data); + } + _getOrReturnCtx(input, ctx) { + return (ctx || { + common: input.parent.common, + data: input.data, + parsedType: getParsedType(input.data), + schemaErrorMap: this._def.errorMap, + path: input.path, + parent: input.parent, + }); + } + _processInputParams(input) { + return { + status: new ParseStatus(), + ctx: { + common: input.parent.common, + data: input.data, + parsedType: getParsedType(input.data), + schemaErrorMap: this._def.errorMap, + path: input.path, + parent: input.parent, + }, + }; + } + _parseSync(input) { + const result = this._parse(input); + if (isAsync(result)) { + throw new Error("Synchronous parse encountered promise."); + } + return result; + } + _parseAsync(input) { + const result = this._parse(input); + return Promise.resolve(result); + } + parse(data, params) { + const result = this.safeParse(data, params); + if (result.success) + return result.data; + throw result.error; + } + safeParse(data, params) { + const ctx = { + common: { + issues: [], + async: params?.async ?? false, + contextualErrorMap: params?.errorMap, + }, + path: params?.path || [], + schemaErrorMap: this._def.errorMap, + parent: null, + data, + parsedType: getParsedType(data), + }; + const result = this._parseSync({ data, path: ctx.path, parent: ctx }); + return handleResult(ctx, result); + } + "~validate"(data) { + const ctx = { + common: { + issues: [], + async: !!this["~standard"].async, + }, + path: [], + schemaErrorMap: this._def.errorMap, + parent: null, + data, + parsedType: getParsedType(data), + }; + if (!this["~standard"].async) { + try { + const result = this._parseSync({ data, path: [], parent: ctx }); + return isValid(result) + ? { + value: result.value, + } + : { + issues: ctx.common.issues, + }; + } + catch (err) { + if (err?.message?.toLowerCase()?.includes("encountered")) { + this["~standard"].async = true; + } + ctx.common = { + issues: [], + async: true, + }; + } + } + return this._parseAsync({ data, path: [], parent: ctx }).then((result) => isValid(result) + ? { + value: result.value, + } + : { + issues: ctx.common.issues, + }); + } + async parseAsync(data, params) { + const result = await this.safeParseAsync(data, params); + if (result.success) + return result.data; + throw result.error; + } + async safeParseAsync(data, params) { + const ctx = { + common: { + issues: [], + contextualErrorMap: params?.errorMap, + async: true, + }, + path: params?.path || [], + schemaErrorMap: this._def.errorMap, + parent: null, + data, + parsedType: getParsedType(data), + }; + const maybeAsyncResult = this._parse({ data, path: ctx.path, parent: ctx }); + const result = await (isAsync(maybeAsyncResult) ? maybeAsyncResult : Promise.resolve(maybeAsyncResult)); + return handleResult(ctx, result); + } + refine(check, message) { + const getIssueProperties = (val) => { + if (typeof message === "string" || typeof message === "undefined") { + return { message }; + } + else if (typeof message === "function") { + return message(val); + } + else { + return message; + } + }; + return this._refinement((val, ctx) => { + const result = check(val); + const setError = () => ctx.addIssue({ + code: ZodIssueCode.custom, + ...getIssueProperties(val), + }); + if (typeof Promise !== "undefined" && result instanceof Promise) { + return result.then((data) => { + if (!data) { + setError(); + return false; + } + else { + return true; + } + }); + } + if (!result) { + setError(); + return false; + } + else { + return true; + } + }); + } + refinement(check, refinementData) { + return this._refinement((val, ctx) => { + if (!check(val)) { + ctx.addIssue(typeof refinementData === "function" ? refinementData(val, ctx) : refinementData); + return false; + } + else { + return true; + } + }); + } + _refinement(refinement) { + return new ZodEffects({ + schema: this, + typeName: ZodFirstPartyTypeKind.ZodEffects, + effect: { type: "refinement", refinement }, + }); + } + superRefine(refinement) { + return this._refinement(refinement); + } + constructor(def) { + /** Alias of safeParseAsync */ + this.spa = this.safeParseAsync; + this._def = def; + this.parse = this.parse.bind(this); + this.safeParse = this.safeParse.bind(this); + this.parseAsync = this.parseAsync.bind(this); + this.safeParseAsync = this.safeParseAsync.bind(this); + this.spa = this.spa.bind(this); + this.refine = this.refine.bind(this); + this.refinement = this.refinement.bind(this); + this.superRefine = this.superRefine.bind(this); + this.optional = this.optional.bind(this); + this.nullable = this.nullable.bind(this); + this.nullish = this.nullish.bind(this); + this.array = this.array.bind(this); + this.promise = this.promise.bind(this); + this.or = this.or.bind(this); + this.and = this.and.bind(this); + this.transform = this.transform.bind(this); + this.brand = this.brand.bind(this); + this.default = this.default.bind(this); + this.catch = this.catch.bind(this); + this.describe = this.describe.bind(this); + this.pipe = this.pipe.bind(this); + this.readonly = this.readonly.bind(this); + this.isNullable = this.isNullable.bind(this); + this.isOptional = this.isOptional.bind(this); + this["~standard"] = { + version: 1, + vendor: "zod", + validate: (data) => this["~validate"](data), + }; + } + optional() { + return ZodOptional.create(this, this._def); + } + nullable() { + return ZodNullable.create(this, this._def); + } + nullish() { + return this.nullable().optional(); + } + array() { + return ZodArray.create(this); + } + promise() { + return ZodPromise.create(this, this._def); + } + or(option) { + return ZodUnion.create([this, option], this._def); + } + and(incoming) { + return ZodIntersection.create(this, incoming, this._def); + } + transform(transform) { + return new ZodEffects({ + ...processCreateParams(this._def), + schema: this, + typeName: ZodFirstPartyTypeKind.ZodEffects, + effect: { type: "transform", transform }, + }); + } + default(def) { + const defaultValueFunc = typeof def === "function" ? def : () => def; + return new ZodDefault({ + ...processCreateParams(this._def), + innerType: this, + defaultValue: defaultValueFunc, + typeName: ZodFirstPartyTypeKind.ZodDefault, + }); + } + brand() { + return new ZodBranded({ + typeName: ZodFirstPartyTypeKind.ZodBranded, + type: this, + ...processCreateParams(this._def), + }); + } + catch(def) { + const catchValueFunc = typeof def === "function" ? def : () => def; + return new ZodCatch({ + ...processCreateParams(this._def), + innerType: this, + catchValue: catchValueFunc, + typeName: ZodFirstPartyTypeKind.ZodCatch, + }); + } + describe(description) { + const This = this.constructor; + return new This({ + ...this._def, + description, + }); + } + pipe(target) { + return ZodPipeline.create(this, target); + } + readonly() { + return ZodReadonly.create(this); + } + isOptional() { + return this.safeParse(undefined).success; + } + isNullable() { + return this.safeParse(null).success; + } +} +const cuidRegex = /^c[^\s-]{8,}$/i; +const cuid2Regex = /^[0-9a-z]+$/; +const ulidRegex = /^[0-9A-HJKMNP-TV-Z]{26}$/i; +// const uuidRegex = +// /^([a-f0-9]{8}-[a-f0-9]{4}-[1-5][a-f0-9]{3}-[a-f0-9]{4}-[a-f0-9]{12}|00000000-0000-0000-0000-000000000000)$/i; +const uuidRegex = /^[0-9a-fA-F]{8}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{12}$/i; +const nanoidRegex = /^[a-z0-9_-]{21}$/i; +const jwtRegex = /^[A-Za-z0-9-_]+\.[A-Za-z0-9-_]+\.[A-Za-z0-9-_]*$/; +const durationRegex = /^[-+]?P(?!$)(?:(?:[-+]?\d+Y)|(?:[-+]?\d+[.,]\d+Y$))?(?:(?:[-+]?\d+M)|(?:[-+]?\d+[.,]\d+M$))?(?:(?:[-+]?\d+W)|(?:[-+]?\d+[.,]\d+W$))?(?:(?:[-+]?\d+D)|(?:[-+]?\d+[.,]\d+D$))?(?:T(?=[\d+-])(?:(?:[-+]?\d+H)|(?:[-+]?\d+[.,]\d+H$))?(?:(?:[-+]?\d+M)|(?:[-+]?\d+[.,]\d+M$))?(?:[-+]?\d+(?:[.,]\d+)?S)?)??$/; +// from https://stackoverflow.com/a/46181/1550155 +// old version: too slow, didn't support unicode +// const emailRegex = /^((([a-z]|\d|[!#\$%&'\*\+\-\/=\?\^_`{\|}~]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])+(\.([a-z]|\d|[!#\$%&'\*\+\-\/=\?\^_`{\|}~]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])+)*)|((\x22)((((\x20|\x09)*(\x0d\x0a))?(\x20|\x09)+)?(([\x01-\x08\x0b\x0c\x0e-\x1f\x7f]|\x21|[\x23-\x5b]|[\x5d-\x7e]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(\\([\x01-\x09\x0b\x0c\x0d-\x7f]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF]))))*(((\x20|\x09)*(\x0d\x0a))?(\x20|\x09)+)?(\x22)))@((([a-z]|\d|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(([a-z]|\d|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])*([a-z]|\d|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])))\.)+(([a-z]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(([a-z]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])*([a-z]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])))$/i; +//old email regex +// const emailRegex = /^(([^<>()[\].,;:\s@"]+(\.[^<>()[\].,;:\s@"]+)*)|(".+"))@((?!-)([^<>()[\].,;:\s@"]+\.)+[^<>()[\].,;:\s@"]{1,})[^-<>()[\].,;:\s@"]$/i; +// eslint-disable-next-line +// const emailRegex = +// /^(([^<>()[\]\\.,;:\s@\"]+(\.[^<>()[\]\\.,;:\s@\"]+)*)|(\".+\"))@((\[(((25[0-5])|(2[0-4][0-9])|(1[0-9]{2})|([0-9]{1,2}))\.){3}((25[0-5])|(2[0-4][0-9])|(1[0-9]{2})|([0-9]{1,2}))\])|(\[IPv6:(([a-f0-9]{1,4}:){7}|::([a-f0-9]{1,4}:){0,6}|([a-f0-9]{1,4}:){1}:([a-f0-9]{1,4}:){0,5}|([a-f0-9]{1,4}:){2}:([a-f0-9]{1,4}:){0,4}|([a-f0-9]{1,4}:){3}:([a-f0-9]{1,4}:){0,3}|([a-f0-9]{1,4}:){4}:([a-f0-9]{1,4}:){0,2}|([a-f0-9]{1,4}:){5}:([a-f0-9]{1,4}:){0,1})([a-f0-9]{1,4}|(((25[0-5])|(2[0-4][0-9])|(1[0-9]{2})|([0-9]{1,2}))\.){3}((25[0-5])|(2[0-4][0-9])|(1[0-9]{2})|([0-9]{1,2})))\])|([A-Za-z0-9]([A-Za-z0-9-]*[A-Za-z0-9])*(\.[A-Za-z]{2,})+))$/; +// const emailRegex = +// /^[a-zA-Z0-9\.\!\#\$\%\&\'\*\+\/\=\?\^\_\`\{\|\}\~\-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$/; +// const emailRegex = +// /^(?:[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\])$/i; +const emailRegex = /^(?!\.)(?!.*\.\.)([A-Z0-9_'+\-\.]*)[A-Z0-9_+-]@([A-Z0-9][A-Z0-9\-]*\.)+[A-Z]{2,}$/i; +// const emailRegex = +// /^[a-z0-9.!#$%&’*+/=?^_`{|}~-]+@[a-z0-9-]+(?:\.[a-z0-9\-]+)*$/i; +// from https://thekevinscott.com/emojis-in-javascript/#writing-a-regular-expression +const _emojiRegex = `^(\\p{Extended_Pictographic}|\\p{Emoji_Component})+$`; +let emojiRegex; +// faster, simpler, safer +const ipv4Regex = /^(?:(?:25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[0-9])\.){3}(?:25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[0-9])$/; +const ipv4CidrRegex = /^(?:(?:25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[0-9])\.){3}(?:25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[0-9])\/(3[0-2]|[12]?[0-9])$/; +// const ipv6Regex = +// /^(([a-f0-9]{1,4}:){7}|::([a-f0-9]{1,4}:){0,6}|([a-f0-9]{1,4}:){1}:([a-f0-9]{1,4}:){0,5}|([a-f0-9]{1,4}:){2}:([a-f0-9]{1,4}:){0,4}|([a-f0-9]{1,4}:){3}:([a-f0-9]{1,4}:){0,3}|([a-f0-9]{1,4}:){4}:([a-f0-9]{1,4}:){0,2}|([a-f0-9]{1,4}:){5}:([a-f0-9]{1,4}:){0,1})([a-f0-9]{1,4}|(((25[0-5])|(2[0-4][0-9])|(1[0-9]{2})|([0-9]{1,2}))\.){3}((25[0-5])|(2[0-4][0-9])|(1[0-9]{2})|([0-9]{1,2})))$/; +const ipv6Regex = /^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))$/; +const ipv6CidrRegex = /^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))\/(12[0-8]|1[01][0-9]|[1-9]?[0-9])$/; +// https://stackoverflow.com/questions/7860392/determine-if-string-is-in-base64-using-javascript +const base64Regex = /^([0-9a-zA-Z+/]{4})*(([0-9a-zA-Z+/]{2}==)|([0-9a-zA-Z+/]{3}=))?$/; +// https://base64.guru/standards/base64url +const base64urlRegex = /^([0-9a-zA-Z-_]{4})*(([0-9a-zA-Z-_]{2}(==)?)|([0-9a-zA-Z-_]{3}(=)?))?$/; +// simple +// const dateRegexSource = `\\d{4}-\\d{2}-\\d{2}`; +// no leap year validation +// const dateRegexSource = `\\d{4}-((0[13578]|10|12)-31|(0[13-9]|1[0-2])-30|(0[1-9]|1[0-2])-(0[1-9]|1\\d|2\\d))`; +// with leap year validation +const dateRegexSource = `((\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-((0[13578]|1[02])-(0[1-9]|[12]\\d|3[01])|(0[469]|11)-(0[1-9]|[12]\\d|30)|(02)-(0[1-9]|1\\d|2[0-8])))`; +const dateRegex = new RegExp(`^${dateRegexSource}$`); +function timeRegexSource(args) { + let secondsRegexSource = `[0-5]\\d`; + if (args.precision) { + secondsRegexSource = `${secondsRegexSource}\\.\\d{${args.precision}}`; + } + else if (args.precision == null) { + secondsRegexSource = `${secondsRegexSource}(\\.\\d+)?`; + } + const secondsQuantifier = args.precision ? "+" : "?"; // require seconds if precision is nonzero + return `([01]\\d|2[0-3]):[0-5]\\d(:${secondsRegexSource})${secondsQuantifier}`; +} +function timeRegex(args) { + return new RegExp(`^${timeRegexSource(args)}$`); +} +// Adapted from https://stackoverflow.com/a/3143231 +export function datetimeRegex(args) { + let regex = `${dateRegexSource}T${timeRegexSource(args)}`; + const opts = []; + opts.push(args.local ? `Z?` : `Z`); + if (args.offset) + opts.push(`([+-]\\d{2}:?\\d{2})`); + regex = `${regex}(${opts.join("|")})`; + return new RegExp(`^${regex}$`); +} +function isValidIP(ip, version) { + if ((version === "v4" || !version) && ipv4Regex.test(ip)) { + return true; + } + if ((version === "v6" || !version) && ipv6Regex.test(ip)) { + return true; + } + return false; +} +function isValidJWT(jwt, alg) { + if (!jwtRegex.test(jwt)) + return false; + try { + const [header] = jwt.split("."); + // Convert base64url to base64 + const base64 = header + .replace(/-/g, "+") + .replace(/_/g, "/") + .padEnd(header.length + ((4 - (header.length % 4)) % 4), "="); + const decoded = JSON.parse(atob(base64)); + if (typeof decoded !== "object" || decoded === null) + return false; + if ("typ" in decoded && decoded?.typ !== "JWT") + return false; + if (!decoded.alg) + return false; + if (alg && decoded.alg !== alg) + return false; + return true; + } + catch { + return false; + } +} +function isValidCidr(ip, version) { + if ((version === "v4" || !version) && ipv4CidrRegex.test(ip)) { + return true; + } + if ((version === "v6" || !version) && ipv6CidrRegex.test(ip)) { + return true; + } + return false; +} +export class ZodString extends ZodType { + _parse(input) { + if (this._def.coerce) { + input.data = String(input.data); + } + const parsedType = this._getType(input); + if (parsedType !== ZodParsedType.string) { + const ctx = this._getOrReturnCtx(input); + addIssueToContext(ctx, { + code: ZodIssueCode.invalid_type, + expected: ZodParsedType.string, + received: ctx.parsedType, + }); + return INVALID; + } + const status = new ParseStatus(); + let ctx = undefined; + for (const check of this._def.checks) { + if (check.kind === "min") { + if (input.data.length < check.value) { + ctx = this._getOrReturnCtx(input, ctx); + addIssueToContext(ctx, { + code: ZodIssueCode.too_small, + minimum: check.value, + type: "string", + inclusive: true, + exact: false, + message: check.message, + }); + status.dirty(); + } + } + else if (check.kind === "max") { + if (input.data.length > check.value) { + ctx = this._getOrReturnCtx(input, ctx); + addIssueToContext(ctx, { + code: ZodIssueCode.too_big, + maximum: check.value, + type: "string", + inclusive: true, + exact: false, + message: check.message, + }); + status.dirty(); + } + } + else if (check.kind === "length") { + const tooBig = input.data.length > check.value; + const tooSmall = input.data.length < check.value; + if (tooBig || tooSmall) { + ctx = this._getOrReturnCtx(input, ctx); + if (tooBig) { + addIssueToContext(ctx, { + code: ZodIssueCode.too_big, + maximum: check.value, + type: "string", + inclusive: true, + exact: true, + message: check.message, + }); + } + else if (tooSmall) { + addIssueToContext(ctx, { + code: ZodIssueCode.too_small, + minimum: check.value, + type: "string", + inclusive: true, + exact: true, + message: check.message, + }); + } + status.dirty(); + } + } + else if (check.kind === "email") { + if (!emailRegex.test(input.data)) { + ctx = this._getOrReturnCtx(input, ctx); + addIssueToContext(ctx, { + validation: "email", + code: ZodIssueCode.invalid_string, + message: check.message, + }); + status.dirty(); + } + } + else if (check.kind === "emoji") { + if (!emojiRegex) { + emojiRegex = new RegExp(_emojiRegex, "u"); + } + if (!emojiRegex.test(input.data)) { + ctx = this._getOrReturnCtx(input, ctx); + addIssueToContext(ctx, { + validation: "emoji", + code: ZodIssueCode.invalid_string, + message: check.message, + }); + status.dirty(); + } + } + else if (check.kind === "uuid") { + if (!uuidRegex.test(input.data)) { + ctx = this._getOrReturnCtx(input, ctx); + addIssueToContext(ctx, { + validation: "uuid", + code: ZodIssueCode.invalid_string, + message: check.message, + }); + status.dirty(); + } + } + else if (check.kind === "nanoid") { + if (!nanoidRegex.test(input.data)) { + ctx = this._getOrReturnCtx(input, ctx); + addIssueToContext(ctx, { + validation: "nanoid", + code: ZodIssueCode.invalid_string, + message: check.message, + }); + status.dirty(); + } + } + else if (check.kind === "cuid") { + if (!cuidRegex.test(input.data)) { + ctx = this._getOrReturnCtx(input, ctx); + addIssueToContext(ctx, { + validation: "cuid", + code: ZodIssueCode.invalid_string, + message: check.message, + }); + status.dirty(); + } + } + else if (check.kind === "cuid2") { + if (!cuid2Regex.test(input.data)) { + ctx = this._getOrReturnCtx(input, ctx); + addIssueToContext(ctx, { + validation: "cuid2", + code: ZodIssueCode.invalid_string, + message: check.message, + }); + status.dirty(); + } + } + else if (check.kind === "ulid") { + if (!ulidRegex.test(input.data)) { + ctx = this._getOrReturnCtx(input, ctx); + addIssueToContext(ctx, { + validation: "ulid", + code: ZodIssueCode.invalid_string, + message: check.message, + }); + status.dirty(); + } + } + else if (check.kind === "url") { + try { + new URL(input.data); + } + catch { + ctx = this._getOrReturnCtx(input, ctx); + addIssueToContext(ctx, { + validation: "url", + code: ZodIssueCode.invalid_string, + message: check.message, + }); + status.dirty(); + } + } + else if (check.kind === "regex") { + check.regex.lastIndex = 0; + const testResult = check.regex.test(input.data); + if (!testResult) { + ctx = this._getOrReturnCtx(input, ctx); + addIssueToContext(ctx, { + validation: "regex", + code: ZodIssueCode.invalid_string, + message: check.message, + }); + status.dirty(); + } + } + else if (check.kind === "trim") { + input.data = input.data.trim(); + } + else if (check.kind === "includes") { + if (!input.data.includes(check.value, check.position)) { + ctx = this._getOrReturnCtx(input, ctx); + addIssueToContext(ctx, { + code: ZodIssueCode.invalid_string, + validation: { includes: check.value, position: check.position }, + message: check.message, + }); + status.dirty(); + } + } + else if (check.kind === "toLowerCase") { + input.data = input.data.toLowerCase(); + } + else if (check.kind === "toUpperCase") { + input.data = input.data.toUpperCase(); + } + else if (check.kind === "startsWith") { + if (!input.data.startsWith(check.value)) { + ctx = this._getOrReturnCtx(input, ctx); + addIssueToContext(ctx, { + code: ZodIssueCode.invalid_string, + validation: { startsWith: check.value }, + message: check.message, + }); + status.dirty(); + } + } + else if (check.kind === "endsWith") { + if (!input.data.endsWith(check.value)) { + ctx = this._getOrReturnCtx(input, ctx); + addIssueToContext(ctx, { + code: ZodIssueCode.invalid_string, + validation: { endsWith: check.value }, + message: check.message, + }); + status.dirty(); + } + } + else if (check.kind === "datetime") { + const regex = datetimeRegex(check); + if (!regex.test(input.data)) { + ctx = this._getOrReturnCtx(input, ctx); + addIssueToContext(ctx, { + code: ZodIssueCode.invalid_string, + validation: "datetime", + message: check.message, + }); + status.dirty(); + } + } + else if (check.kind === "date") { + const regex = dateRegex; + if (!regex.test(input.data)) { + ctx = this._getOrReturnCtx(input, ctx); + addIssueToContext(ctx, { + code: ZodIssueCode.invalid_string, + validation: "date", + message: check.message, + }); + status.dirty(); + } + } + else if (check.kind === "time") { + const regex = timeRegex(check); + if (!regex.test(input.data)) { + ctx = this._getOrReturnCtx(input, ctx); + addIssueToContext(ctx, { + code: ZodIssueCode.invalid_string, + validation: "time", + message: check.message, + }); + status.dirty(); + } + } + else if (check.kind === "duration") { + if (!durationRegex.test(input.data)) { + ctx = this._getOrReturnCtx(input, ctx); + addIssueToContext(ctx, { + validation: "duration", + code: ZodIssueCode.invalid_string, + message: check.message, + }); + status.dirty(); + } + } + else if (check.kind === "ip") { + if (!isValidIP(input.data, check.version)) { + ctx = this._getOrReturnCtx(input, ctx); + addIssueToContext(ctx, { + validation: "ip", + code: ZodIssueCode.invalid_string, + message: check.message, + }); + status.dirty(); + } + } + else if (check.kind === "jwt") { + if (!isValidJWT(input.data, check.alg)) { + ctx = this._getOrReturnCtx(input, ctx); + addIssueToContext(ctx, { + validation: "jwt", + code: ZodIssueCode.invalid_string, + message: check.message, + }); + status.dirty(); + } + } + else if (check.kind === "cidr") { + if (!isValidCidr(input.data, check.version)) { + ctx = this._getOrReturnCtx(input, ctx); + addIssueToContext(ctx, { + validation: "cidr", + code: ZodIssueCode.invalid_string, + message: check.message, + }); + status.dirty(); + } + } + else if (check.kind === "base64") { + if (!base64Regex.test(input.data)) { + ctx = this._getOrReturnCtx(input, ctx); + addIssueToContext(ctx, { + validation: "base64", + code: ZodIssueCode.invalid_string, + message: check.message, + }); + status.dirty(); + } + } + else if (check.kind === "base64url") { + if (!base64urlRegex.test(input.data)) { + ctx = this._getOrReturnCtx(input, ctx); + addIssueToContext(ctx, { + validation: "base64url", + code: ZodIssueCode.invalid_string, + message: check.message, + }); + status.dirty(); + } + } + else { + util.assertNever(check); + } + } + return { status: status.value, value: input.data }; + } + _regex(regex, validation, message) { + return this.refinement((data) => regex.test(data), { + validation, + code: ZodIssueCode.invalid_string, + ...errorUtil.errToObj(message), + }); + } + _addCheck(check) { + return new ZodString({ + ...this._def, + checks: [...this._def.checks, check], + }); + } + email(message) { + return this._addCheck({ kind: "email", ...errorUtil.errToObj(message) }); + } + url(message) { + return this._addCheck({ kind: "url", ...errorUtil.errToObj(message) }); + } + emoji(message) { + return this._addCheck({ kind: "emoji", ...errorUtil.errToObj(message) }); + } + uuid(message) { + return this._addCheck({ kind: "uuid", ...errorUtil.errToObj(message) }); + } + nanoid(message) { + return this._addCheck({ kind: "nanoid", ...errorUtil.errToObj(message) }); + } + cuid(message) { + return this._addCheck({ kind: "cuid", ...errorUtil.errToObj(message) }); + } + cuid2(message) { + return this._addCheck({ kind: "cuid2", ...errorUtil.errToObj(message) }); + } + ulid(message) { + return this._addCheck({ kind: "ulid", ...errorUtil.errToObj(message) }); + } + base64(message) { + return this._addCheck({ kind: "base64", ...errorUtil.errToObj(message) }); + } + base64url(message) { + // base64url encoding is a modification of base64 that can safely be used in URLs and filenames + return this._addCheck({ + kind: "base64url", + ...errorUtil.errToObj(message), + }); + } + jwt(options) { + return this._addCheck({ kind: "jwt", ...errorUtil.errToObj(options) }); + } + ip(options) { + return this._addCheck({ kind: "ip", ...errorUtil.errToObj(options) }); + } + cidr(options) { + return this._addCheck({ kind: "cidr", ...errorUtil.errToObj(options) }); + } + datetime(options) { + if (typeof options === "string") { + return this._addCheck({ + kind: "datetime", + precision: null, + offset: false, + local: false, + message: options, + }); + } + return this._addCheck({ + kind: "datetime", + precision: typeof options?.precision === "undefined" ? null : options?.precision, + offset: options?.offset ?? false, + local: options?.local ?? false, + ...errorUtil.errToObj(options?.message), + }); + } + date(message) { + return this._addCheck({ kind: "date", message }); + } + time(options) { + if (typeof options === "string") { + return this._addCheck({ + kind: "time", + precision: null, + message: options, + }); + } + return this._addCheck({ + kind: "time", + precision: typeof options?.precision === "undefined" ? null : options?.precision, + ...errorUtil.errToObj(options?.message), + }); + } + duration(message) { + return this._addCheck({ kind: "duration", ...errorUtil.errToObj(message) }); + } + regex(regex, message) { + return this._addCheck({ + kind: "regex", + regex: regex, + ...errorUtil.errToObj(message), + }); + } + includes(value, options) { + return this._addCheck({ + kind: "includes", + value: value, + position: options?.position, + ...errorUtil.errToObj(options?.message), + }); + } + startsWith(value, message) { + return this._addCheck({ + kind: "startsWith", + value: value, + ...errorUtil.errToObj(message), + }); + } + endsWith(value, message) { + return this._addCheck({ + kind: "endsWith", + value: value, + ...errorUtil.errToObj(message), + }); + } + min(minLength, message) { + return this._addCheck({ + kind: "min", + value: minLength, + ...errorUtil.errToObj(message), + }); + } + max(maxLength, message) { + return this._addCheck({ + kind: "max", + value: maxLength, + ...errorUtil.errToObj(message), + }); + } + length(len, message) { + return this._addCheck({ + kind: "length", + value: len, + ...errorUtil.errToObj(message), + }); + } + /** + * Equivalent to `.min(1)` + */ + nonempty(message) { + return this.min(1, errorUtil.errToObj(message)); + } + trim() { + return new ZodString({ + ...this._def, + checks: [...this._def.checks, { kind: "trim" }], + }); + } + toLowerCase() { + return new ZodString({ + ...this._def, + checks: [...this._def.checks, { kind: "toLowerCase" }], + }); + } + toUpperCase() { + return new ZodString({ + ...this._def, + checks: [...this._def.checks, { kind: "toUpperCase" }], + }); + } + get isDatetime() { + return !!this._def.checks.find((ch) => ch.kind === "datetime"); + } + get isDate() { + return !!this._def.checks.find((ch) => ch.kind === "date"); + } + get isTime() { + return !!this._def.checks.find((ch) => ch.kind === "time"); + } + get isDuration() { + return !!this._def.checks.find((ch) => ch.kind === "duration"); + } + get isEmail() { + return !!this._def.checks.find((ch) => ch.kind === "email"); + } + get isURL() { + return !!this._def.checks.find((ch) => ch.kind === "url"); + } + get isEmoji() { + return !!this._def.checks.find((ch) => ch.kind === "emoji"); + } + get isUUID() { + return !!this._def.checks.find((ch) => ch.kind === "uuid"); + } + get isNANOID() { + return !!this._def.checks.find((ch) => ch.kind === "nanoid"); + } + get isCUID() { + return !!this._def.checks.find((ch) => ch.kind === "cuid"); + } + get isCUID2() { + return !!this._def.checks.find((ch) => ch.kind === "cuid2"); + } + get isULID() { + return !!this._def.checks.find((ch) => ch.kind === "ulid"); + } + get isIP() { + return !!this._def.checks.find((ch) => ch.kind === "ip"); + } + get isCIDR() { + return !!this._def.checks.find((ch) => ch.kind === "cidr"); + } + get isBase64() { + return !!this._def.checks.find((ch) => ch.kind === "base64"); + } + get isBase64url() { + // base64url encoding is a modification of base64 that can safely be used in URLs and filenames + return !!this._def.checks.find((ch) => ch.kind === "base64url"); + } + get minLength() { + let min = null; + for (const ch of this._def.checks) { + if (ch.kind === "min") { + if (min === null || ch.value > min) + min = ch.value; + } + } + return min; + } + get maxLength() { + let max = null; + for (const ch of this._def.checks) { + if (ch.kind === "max") { + if (max === null || ch.value < max) + max = ch.value; + } + } + return max; + } +} +ZodString.create = (params) => { + return new ZodString({ + checks: [], + typeName: ZodFirstPartyTypeKind.ZodString, + coerce: params?.coerce ?? false, + ...processCreateParams(params), + }); +}; +// https://stackoverflow.com/questions/3966484/why-does-modulus-operator-return-fractional-number-in-javascript/31711034#31711034 +function floatSafeRemainder(val, step) { + const valDecCount = (val.toString().split(".")[1] || "").length; + const stepDecCount = (step.toString().split(".")[1] || "").length; + const decCount = valDecCount > stepDecCount ? valDecCount : stepDecCount; + const valInt = Number.parseInt(val.toFixed(decCount).replace(".", "")); + const stepInt = Number.parseInt(step.toFixed(decCount).replace(".", "")); + return (valInt % stepInt) / 10 ** decCount; +} +export class ZodNumber extends ZodType { + constructor() { + super(...arguments); + this.min = this.gte; + this.max = this.lte; + this.step = this.multipleOf; + } + _parse(input) { + if (this._def.coerce) { + input.data = Number(input.data); + } + const parsedType = this._getType(input); + if (parsedType !== ZodParsedType.number) { + const ctx = this._getOrReturnCtx(input); + addIssueToContext(ctx, { + code: ZodIssueCode.invalid_type, + expected: ZodParsedType.number, + received: ctx.parsedType, + }); + return INVALID; + } + let ctx = undefined; + const status = new ParseStatus(); + for (const check of this._def.checks) { + if (check.kind === "int") { + if (!util.isInteger(input.data)) { + ctx = this._getOrReturnCtx(input, ctx); + addIssueToContext(ctx, { + code: ZodIssueCode.invalid_type, + expected: "integer", + received: "float", + message: check.message, + }); + status.dirty(); + } + } + else if (check.kind === "min") { + const tooSmall = check.inclusive ? input.data < check.value : input.data <= check.value; + if (tooSmall) { + ctx = this._getOrReturnCtx(input, ctx); + addIssueToContext(ctx, { + code: ZodIssueCode.too_small, + minimum: check.value, + type: "number", + inclusive: check.inclusive, + exact: false, + message: check.message, + }); + status.dirty(); + } + } + else if (check.kind === "max") { + const tooBig = check.inclusive ? input.data > check.value : input.data >= check.value; + if (tooBig) { + ctx = this._getOrReturnCtx(input, ctx); + addIssueToContext(ctx, { + code: ZodIssueCode.too_big, + maximum: check.value, + type: "number", + inclusive: check.inclusive, + exact: false, + message: check.message, + }); + status.dirty(); + } + } + else if (check.kind === "multipleOf") { + if (floatSafeRemainder(input.data, check.value) !== 0) { + ctx = this._getOrReturnCtx(input, ctx); + addIssueToContext(ctx, { + code: ZodIssueCode.not_multiple_of, + multipleOf: check.value, + message: check.message, + }); + status.dirty(); + } + } + else if (check.kind === "finite") { + if (!Number.isFinite(input.data)) { + ctx = this._getOrReturnCtx(input, ctx); + addIssueToContext(ctx, { + code: ZodIssueCode.not_finite, + message: check.message, + }); + status.dirty(); + } + } + else { + util.assertNever(check); + } + } + return { status: status.value, value: input.data }; + } + gte(value, message) { + return this.setLimit("min", value, true, errorUtil.toString(message)); + } + gt(value, message) { + return this.setLimit("min", value, false, errorUtil.toString(message)); + } + lte(value, message) { + return this.setLimit("max", value, true, errorUtil.toString(message)); + } + lt(value, message) { + return this.setLimit("max", value, false, errorUtil.toString(message)); + } + setLimit(kind, value, inclusive, message) { + return new ZodNumber({ + ...this._def, + checks: [ + ...this._def.checks, + { + kind, + value, + inclusive, + message: errorUtil.toString(message), + }, + ], + }); + } + _addCheck(check) { + return new ZodNumber({ + ...this._def, + checks: [...this._def.checks, check], + }); + } + int(message) { + return this._addCheck({ + kind: "int", + message: errorUtil.toString(message), + }); + } + positive(message) { + return this._addCheck({ + kind: "min", + value: 0, + inclusive: false, + message: errorUtil.toString(message), + }); + } + negative(message) { + return this._addCheck({ + kind: "max", + value: 0, + inclusive: false, + message: errorUtil.toString(message), + }); + } + nonpositive(message) { + return this._addCheck({ + kind: "max", + value: 0, + inclusive: true, + message: errorUtil.toString(message), + }); + } + nonnegative(message) { + return this._addCheck({ + kind: "min", + value: 0, + inclusive: true, + message: errorUtil.toString(message), + }); + } + multipleOf(value, message) { + return this._addCheck({ + kind: "multipleOf", + value: value, + message: errorUtil.toString(message), + }); + } + finite(message) { + return this._addCheck({ + kind: "finite", + message: errorUtil.toString(message), + }); + } + safe(message) { + return this._addCheck({ + kind: "min", + inclusive: true, + value: Number.MIN_SAFE_INTEGER, + message: errorUtil.toString(message), + })._addCheck({ + kind: "max", + inclusive: true, + value: Number.MAX_SAFE_INTEGER, + message: errorUtil.toString(message), + }); + } + get minValue() { + let min = null; + for (const ch of this._def.checks) { + if (ch.kind === "min") { + if (min === null || ch.value > min) + min = ch.value; + } + } + return min; + } + get maxValue() { + let max = null; + for (const ch of this._def.checks) { + if (ch.kind === "max") { + if (max === null || ch.value < max) + max = ch.value; + } + } + return max; + } + get isInt() { + return !!this._def.checks.find((ch) => ch.kind === "int" || (ch.kind === "multipleOf" && util.isInteger(ch.value))); + } + get isFinite() { + let max = null; + let min = null; + for (const ch of this._def.checks) { + if (ch.kind === "finite" || ch.kind === "int" || ch.kind === "multipleOf") { + return true; + } + else if (ch.kind === "min") { + if (min === null || ch.value > min) + min = ch.value; + } + else if (ch.kind === "max") { + if (max === null || ch.value < max) + max = ch.value; + } + } + return Number.isFinite(min) && Number.isFinite(max); + } +} +ZodNumber.create = (params) => { + return new ZodNumber({ + checks: [], + typeName: ZodFirstPartyTypeKind.ZodNumber, + coerce: params?.coerce || false, + ...processCreateParams(params), + }); +}; +export class ZodBigInt extends ZodType { + constructor() { + super(...arguments); + this.min = this.gte; + this.max = this.lte; + } + _parse(input) { + if (this._def.coerce) { + try { + input.data = BigInt(input.data); + } + catch { + return this._getInvalidInput(input); + } + } + const parsedType = this._getType(input); + if (parsedType !== ZodParsedType.bigint) { + return this._getInvalidInput(input); + } + let ctx = undefined; + const status = new ParseStatus(); + for (const check of this._def.checks) { + if (check.kind === "min") { + const tooSmall = check.inclusive ? input.data < check.value : input.data <= check.value; + if (tooSmall) { + ctx = this._getOrReturnCtx(input, ctx); + addIssueToContext(ctx, { + code: ZodIssueCode.too_small, + type: "bigint", + minimum: check.value, + inclusive: check.inclusive, + message: check.message, + }); + status.dirty(); + } + } + else if (check.kind === "max") { + const tooBig = check.inclusive ? input.data > check.value : input.data >= check.value; + if (tooBig) { + ctx = this._getOrReturnCtx(input, ctx); + addIssueToContext(ctx, { + code: ZodIssueCode.too_big, + type: "bigint", + maximum: check.value, + inclusive: check.inclusive, + message: check.message, + }); + status.dirty(); + } + } + else if (check.kind === "multipleOf") { + if (input.data % check.value !== BigInt(0)) { + ctx = this._getOrReturnCtx(input, ctx); + addIssueToContext(ctx, { + code: ZodIssueCode.not_multiple_of, + multipleOf: check.value, + message: check.message, + }); + status.dirty(); + } + } + else { + util.assertNever(check); + } + } + return { status: status.value, value: input.data }; + } + _getInvalidInput(input) { + const ctx = this._getOrReturnCtx(input); + addIssueToContext(ctx, { + code: ZodIssueCode.invalid_type, + expected: ZodParsedType.bigint, + received: ctx.parsedType, + }); + return INVALID; + } + gte(value, message) { + return this.setLimit("min", value, true, errorUtil.toString(message)); + } + gt(value, message) { + return this.setLimit("min", value, false, errorUtil.toString(message)); + } + lte(value, message) { + return this.setLimit("max", value, true, errorUtil.toString(message)); + } + lt(value, message) { + return this.setLimit("max", value, false, errorUtil.toString(message)); + } + setLimit(kind, value, inclusive, message) { + return new ZodBigInt({ + ...this._def, + checks: [ + ...this._def.checks, + { + kind, + value, + inclusive, + message: errorUtil.toString(message), + }, + ], + }); + } + _addCheck(check) { + return new ZodBigInt({ + ...this._def, + checks: [...this._def.checks, check], + }); + } + positive(message) { + return this._addCheck({ + kind: "min", + value: BigInt(0), + inclusive: false, + message: errorUtil.toString(message), + }); + } + negative(message) { + return this._addCheck({ + kind: "max", + value: BigInt(0), + inclusive: false, + message: errorUtil.toString(message), + }); + } + nonpositive(message) { + return this._addCheck({ + kind: "max", + value: BigInt(0), + inclusive: true, + message: errorUtil.toString(message), + }); + } + nonnegative(message) { + return this._addCheck({ + kind: "min", + value: BigInt(0), + inclusive: true, + message: errorUtil.toString(message), + }); + } + multipleOf(value, message) { + return this._addCheck({ + kind: "multipleOf", + value, + message: errorUtil.toString(message), + }); + } + get minValue() { + let min = null; + for (const ch of this._def.checks) { + if (ch.kind === "min") { + if (min === null || ch.value > min) + min = ch.value; + } + } + return min; + } + get maxValue() { + let max = null; + for (const ch of this._def.checks) { + if (ch.kind === "max") { + if (max === null || ch.value < max) + max = ch.value; + } + } + return max; + } +} +ZodBigInt.create = (params) => { + return new ZodBigInt({ + checks: [], + typeName: ZodFirstPartyTypeKind.ZodBigInt, + coerce: params?.coerce ?? false, + ...processCreateParams(params), + }); +}; +export class ZodBoolean extends ZodType { + _parse(input) { + if (this._def.coerce) { + input.data = Boolean(input.data); + } + const parsedType = this._getType(input); + if (parsedType !== ZodParsedType.boolean) { + const ctx = this._getOrReturnCtx(input); + addIssueToContext(ctx, { + code: ZodIssueCode.invalid_type, + expected: ZodParsedType.boolean, + received: ctx.parsedType, + }); + return INVALID; + } + return OK(input.data); + } +} +ZodBoolean.create = (params) => { + return new ZodBoolean({ + typeName: ZodFirstPartyTypeKind.ZodBoolean, + coerce: params?.coerce || false, + ...processCreateParams(params), + }); +}; +export class ZodDate extends ZodType { + _parse(input) { + if (this._def.coerce) { + input.data = new Date(input.data); + } + const parsedType = this._getType(input); + if (parsedType !== ZodParsedType.date) { + const ctx = this._getOrReturnCtx(input); + addIssueToContext(ctx, { + code: ZodIssueCode.invalid_type, + expected: ZodParsedType.date, + received: ctx.parsedType, + }); + return INVALID; + } + if (Number.isNaN(input.data.getTime())) { + const ctx = this._getOrReturnCtx(input); + addIssueToContext(ctx, { + code: ZodIssueCode.invalid_date, + }); + return INVALID; + } + const status = new ParseStatus(); + let ctx = undefined; + for (const check of this._def.checks) { + if (check.kind === "min") { + if (input.data.getTime() < check.value) { + ctx = this._getOrReturnCtx(input, ctx); + addIssueToContext(ctx, { + code: ZodIssueCode.too_small, + message: check.message, + inclusive: true, + exact: false, + minimum: check.value, + type: "date", + }); + status.dirty(); + } + } + else if (check.kind === "max") { + if (input.data.getTime() > check.value) { + ctx = this._getOrReturnCtx(input, ctx); + addIssueToContext(ctx, { + code: ZodIssueCode.too_big, + message: check.message, + inclusive: true, + exact: false, + maximum: check.value, + type: "date", + }); + status.dirty(); + } + } + else { + util.assertNever(check); + } + } + return { + status: status.value, + value: new Date(input.data.getTime()), + }; + } + _addCheck(check) { + return new ZodDate({ + ...this._def, + checks: [...this._def.checks, check], + }); + } + min(minDate, message) { + return this._addCheck({ + kind: "min", + value: minDate.getTime(), + message: errorUtil.toString(message), + }); + } + max(maxDate, message) { + return this._addCheck({ + kind: "max", + value: maxDate.getTime(), + message: errorUtil.toString(message), + }); + } + get minDate() { + let min = null; + for (const ch of this._def.checks) { + if (ch.kind === "min") { + if (min === null || ch.value > min) + min = ch.value; + } + } + return min != null ? new Date(min) : null; + } + get maxDate() { + let max = null; + for (const ch of this._def.checks) { + if (ch.kind === "max") { + if (max === null || ch.value < max) + max = ch.value; + } + } + return max != null ? new Date(max) : null; + } +} +ZodDate.create = (params) => { + return new ZodDate({ + checks: [], + coerce: params?.coerce || false, + typeName: ZodFirstPartyTypeKind.ZodDate, + ...processCreateParams(params), + }); +}; +export class ZodSymbol extends ZodType { + _parse(input) { + const parsedType = this._getType(input); + if (parsedType !== ZodParsedType.symbol) { + const ctx = this._getOrReturnCtx(input); + addIssueToContext(ctx, { + code: ZodIssueCode.invalid_type, + expected: ZodParsedType.symbol, + received: ctx.parsedType, + }); + return INVALID; + } + return OK(input.data); + } +} +ZodSymbol.create = (params) => { + return new ZodSymbol({ + typeName: ZodFirstPartyTypeKind.ZodSymbol, + ...processCreateParams(params), + }); +}; +export class ZodUndefined extends ZodType { + _parse(input) { + const parsedType = this._getType(input); + if (parsedType !== ZodParsedType.undefined) { + const ctx = this._getOrReturnCtx(input); + addIssueToContext(ctx, { + code: ZodIssueCode.invalid_type, + expected: ZodParsedType.undefined, + received: ctx.parsedType, + }); + return INVALID; + } + return OK(input.data); + } +} +ZodUndefined.create = (params) => { + return new ZodUndefined({ + typeName: ZodFirstPartyTypeKind.ZodUndefined, + ...processCreateParams(params), + }); +}; +export class ZodNull extends ZodType { + _parse(input) { + const parsedType = this._getType(input); + if (parsedType !== ZodParsedType.null) { + const ctx = this._getOrReturnCtx(input); + addIssueToContext(ctx, { + code: ZodIssueCode.invalid_type, + expected: ZodParsedType.null, + received: ctx.parsedType, + }); + return INVALID; + } + return OK(input.data); + } +} +ZodNull.create = (params) => { + return new ZodNull({ + typeName: ZodFirstPartyTypeKind.ZodNull, + ...processCreateParams(params), + }); +}; +export class ZodAny extends ZodType { + constructor() { + super(...arguments); + // to prevent instances of other classes from extending ZodAny. this causes issues with catchall in ZodObject. + this._any = true; + } + _parse(input) { + return OK(input.data); + } +} +ZodAny.create = (params) => { + return new ZodAny({ + typeName: ZodFirstPartyTypeKind.ZodAny, + ...processCreateParams(params), + }); +}; +export class ZodUnknown extends ZodType { + constructor() { + super(...arguments); + // required + this._unknown = true; + } + _parse(input) { + return OK(input.data); + } +} +ZodUnknown.create = (params) => { + return new ZodUnknown({ + typeName: ZodFirstPartyTypeKind.ZodUnknown, + ...processCreateParams(params), + }); +}; +export class ZodNever extends ZodType { + _parse(input) { + const ctx = this._getOrReturnCtx(input); + addIssueToContext(ctx, { + code: ZodIssueCode.invalid_type, + expected: ZodParsedType.never, + received: ctx.parsedType, + }); + return INVALID; + } +} +ZodNever.create = (params) => { + return new ZodNever({ + typeName: ZodFirstPartyTypeKind.ZodNever, + ...processCreateParams(params), + }); +}; +export class ZodVoid extends ZodType { + _parse(input) { + const parsedType = this._getType(input); + if (parsedType !== ZodParsedType.undefined) { + const ctx = this._getOrReturnCtx(input); + addIssueToContext(ctx, { + code: ZodIssueCode.invalid_type, + expected: ZodParsedType.void, + received: ctx.parsedType, + }); + return INVALID; + } + return OK(input.data); + } +} +ZodVoid.create = (params) => { + return new ZodVoid({ + typeName: ZodFirstPartyTypeKind.ZodVoid, + ...processCreateParams(params), + }); +}; +export class ZodArray extends ZodType { + _parse(input) { + const { ctx, status } = this._processInputParams(input); + const def = this._def; + if (ctx.parsedType !== ZodParsedType.array) { + addIssueToContext(ctx, { + code: ZodIssueCode.invalid_type, + expected: ZodParsedType.array, + received: ctx.parsedType, + }); + return INVALID; + } + if (def.exactLength !== null) { + const tooBig = ctx.data.length > def.exactLength.value; + const tooSmall = ctx.data.length < def.exactLength.value; + if (tooBig || tooSmall) { + addIssueToContext(ctx, { + code: tooBig ? ZodIssueCode.too_big : ZodIssueCode.too_small, + minimum: (tooSmall ? def.exactLength.value : undefined), + maximum: (tooBig ? def.exactLength.value : undefined), + type: "array", + inclusive: true, + exact: true, + message: def.exactLength.message, + }); + status.dirty(); + } + } + if (def.minLength !== null) { + if (ctx.data.length < def.minLength.value) { + addIssueToContext(ctx, { + code: ZodIssueCode.too_small, + minimum: def.minLength.value, + type: "array", + inclusive: true, + exact: false, + message: def.minLength.message, + }); + status.dirty(); + } + } + if (def.maxLength !== null) { + if (ctx.data.length > def.maxLength.value) { + addIssueToContext(ctx, { + code: ZodIssueCode.too_big, + maximum: def.maxLength.value, + type: "array", + inclusive: true, + exact: false, + message: def.maxLength.message, + }); + status.dirty(); + } + } + if (ctx.common.async) { + return Promise.all([...ctx.data].map((item, i) => { + return def.type._parseAsync(new ParseInputLazyPath(ctx, item, ctx.path, i)); + })).then((result) => { + return ParseStatus.mergeArray(status, result); + }); + } + const result = [...ctx.data].map((item, i) => { + return def.type._parseSync(new ParseInputLazyPath(ctx, item, ctx.path, i)); + }); + return ParseStatus.mergeArray(status, result); + } + get element() { + return this._def.type; + } + min(minLength, message) { + return new ZodArray({ + ...this._def, + minLength: { value: minLength, message: errorUtil.toString(message) }, + }); + } + max(maxLength, message) { + return new ZodArray({ + ...this._def, + maxLength: { value: maxLength, message: errorUtil.toString(message) }, + }); + } + length(len, message) { + return new ZodArray({ + ...this._def, + exactLength: { value: len, message: errorUtil.toString(message) }, + }); + } + nonempty(message) { + return this.min(1, message); + } +} +ZodArray.create = (schema, params) => { + return new ZodArray({ + type: schema, + minLength: null, + maxLength: null, + exactLength: null, + typeName: ZodFirstPartyTypeKind.ZodArray, + ...processCreateParams(params), + }); +}; +function deepPartialify(schema) { + if (schema instanceof ZodObject) { + const newShape = {}; + for (const key in schema.shape) { + const fieldSchema = schema.shape[key]; + newShape[key] = ZodOptional.create(deepPartialify(fieldSchema)); + } + return new ZodObject({ + ...schema._def, + shape: () => newShape, + }); + } + else if (schema instanceof ZodArray) { + return new ZodArray({ + ...schema._def, + type: deepPartialify(schema.element), + }); + } + else if (schema instanceof ZodOptional) { + return ZodOptional.create(deepPartialify(schema.unwrap())); + } + else if (schema instanceof ZodNullable) { + return ZodNullable.create(deepPartialify(schema.unwrap())); + } + else if (schema instanceof ZodTuple) { + return ZodTuple.create(schema.items.map((item) => deepPartialify(item))); + } + else { + return schema; + } +} +export class ZodObject extends ZodType { + constructor() { + super(...arguments); + this._cached = null; + /** + * @deprecated In most cases, this is no longer needed - unknown properties are now silently stripped. + * If you want to pass through unknown properties, use `.passthrough()` instead. + */ + this.nonstrict = this.passthrough; + // extend< + // Augmentation extends ZodRawShape, + // NewOutput extends util.flatten<{ + // [k in keyof Augmentation | keyof Output]: k extends keyof Augmentation + // ? Augmentation[k]["_output"] + // : k extends keyof Output + // ? Output[k] + // : never; + // }>, + // NewInput extends util.flatten<{ + // [k in keyof Augmentation | keyof Input]: k extends keyof Augmentation + // ? Augmentation[k]["_input"] + // : k extends keyof Input + // ? Input[k] + // : never; + // }> + // >( + // augmentation: Augmentation + // ): ZodObject< + // extendShape, + // UnknownKeys, + // Catchall, + // NewOutput, + // NewInput + // > { + // return new ZodObject({ + // ...this._def, + // shape: () => ({ + // ...this._def.shape(), + // ...augmentation, + // }), + // }) as any; + // } + /** + * @deprecated Use `.extend` instead + * */ + this.augment = this.extend; + } + _getCached() { + if (this._cached !== null) + return this._cached; + const shape = this._def.shape(); + const keys = util.objectKeys(shape); + this._cached = { shape, keys }; + return this._cached; + } + _parse(input) { + const parsedType = this._getType(input); + if (parsedType !== ZodParsedType.object) { + const ctx = this._getOrReturnCtx(input); + addIssueToContext(ctx, { + code: ZodIssueCode.invalid_type, + expected: ZodParsedType.object, + received: ctx.parsedType, + }); + return INVALID; + } + const { status, ctx } = this._processInputParams(input); + const { shape, keys: shapeKeys } = this._getCached(); + const extraKeys = []; + if (!(this._def.catchall instanceof ZodNever && this._def.unknownKeys === "strip")) { + for (const key in ctx.data) { + if (!shapeKeys.includes(key)) { + extraKeys.push(key); + } + } + } + const pairs = []; + for (const key of shapeKeys) { + const keyValidator = shape[key]; + const value = ctx.data[key]; + pairs.push({ + key: { status: "valid", value: key }, + value: keyValidator._parse(new ParseInputLazyPath(ctx, value, ctx.path, key)), + alwaysSet: key in ctx.data, + }); + } + if (this._def.catchall instanceof ZodNever) { + const unknownKeys = this._def.unknownKeys; + if (unknownKeys === "passthrough") { + for (const key of extraKeys) { + pairs.push({ + key: { status: "valid", value: key }, + value: { status: "valid", value: ctx.data[key] }, + }); + } + } + else if (unknownKeys === "strict") { + if (extraKeys.length > 0) { + addIssueToContext(ctx, { + code: ZodIssueCode.unrecognized_keys, + keys: extraKeys, + }); + status.dirty(); + } + } + else if (unknownKeys === "strip") { + } + else { + throw new Error(`Internal ZodObject error: invalid unknownKeys value.`); + } + } + else { + // run catchall validation + const catchall = this._def.catchall; + for (const key of extraKeys) { + const value = ctx.data[key]; + pairs.push({ + key: { status: "valid", value: key }, + value: catchall._parse(new ParseInputLazyPath(ctx, value, ctx.path, key) //, ctx.child(key), value, getParsedType(value) + ), + alwaysSet: key in ctx.data, + }); + } + } + if (ctx.common.async) { + return Promise.resolve() + .then(async () => { + const syncPairs = []; + for (const pair of pairs) { + const key = await pair.key; + const value = await pair.value; + syncPairs.push({ + key, + value, + alwaysSet: pair.alwaysSet, + }); + } + return syncPairs; + }) + .then((syncPairs) => { + return ParseStatus.mergeObjectSync(status, syncPairs); + }); + } + else { + return ParseStatus.mergeObjectSync(status, pairs); + } + } + get shape() { + return this._def.shape(); + } + strict(message) { + errorUtil.errToObj; + return new ZodObject({ + ...this._def, + unknownKeys: "strict", + ...(message !== undefined + ? { + errorMap: (issue, ctx) => { + const defaultError = this._def.errorMap?.(issue, ctx).message ?? ctx.defaultError; + if (issue.code === "unrecognized_keys") + return { + message: errorUtil.errToObj(message).message ?? defaultError, + }; + return { + message: defaultError, + }; + }, + } + : {}), + }); + } + strip() { + return new ZodObject({ + ...this._def, + unknownKeys: "strip", + }); + } + passthrough() { + return new ZodObject({ + ...this._def, + unknownKeys: "passthrough", + }); + } + // const AugmentFactory = + // (def: Def) => + // ( + // augmentation: Augmentation + // ): ZodObject< + // extendShape, Augmentation>, + // Def["unknownKeys"], + // Def["catchall"] + // > => { + // return new ZodObject({ + // ...def, + // shape: () => ({ + // ...def.shape(), + // ...augmentation, + // }), + // }) as any; + // }; + extend(augmentation) { + return new ZodObject({ + ...this._def, + shape: () => ({ + ...this._def.shape(), + ...augmentation, + }), + }); + } + /** + * Prior to zod@1.0.12 there was a bug in the + * inferred type of merged objects. Please + * upgrade if you are experiencing issues. + */ + merge(merging) { + const merged = new ZodObject({ + unknownKeys: merging._def.unknownKeys, + catchall: merging._def.catchall, + shape: () => ({ + ...this._def.shape(), + ...merging._def.shape(), + }), + typeName: ZodFirstPartyTypeKind.ZodObject, + }); + return merged; + } + // merge< + // Incoming extends AnyZodObject, + // Augmentation extends Incoming["shape"], + // NewOutput extends { + // [k in keyof Augmentation | keyof Output]: k extends keyof Augmentation + // ? Augmentation[k]["_output"] + // : k extends keyof Output + // ? Output[k] + // : never; + // }, + // NewInput extends { + // [k in keyof Augmentation | keyof Input]: k extends keyof Augmentation + // ? Augmentation[k]["_input"] + // : k extends keyof Input + // ? Input[k] + // : never; + // } + // >( + // merging: Incoming + // ): ZodObject< + // extendShape>, + // Incoming["_def"]["unknownKeys"], + // Incoming["_def"]["catchall"], + // NewOutput, + // NewInput + // > { + // const merged: any = new ZodObject({ + // unknownKeys: merging._def.unknownKeys, + // catchall: merging._def.catchall, + // shape: () => + // objectUtil.mergeShapes(this._def.shape(), merging._def.shape()), + // typeName: ZodFirstPartyTypeKind.ZodObject, + // }) as any; + // return merged; + // } + setKey(key, schema) { + return this.augment({ [key]: schema }); + } + // merge( + // merging: Incoming + // ): //ZodObject = (merging) => { + // ZodObject< + // extendShape>, + // Incoming["_def"]["unknownKeys"], + // Incoming["_def"]["catchall"] + // > { + // // const mergedShape = objectUtil.mergeShapes( + // // this._def.shape(), + // // merging._def.shape() + // // ); + // const merged: any = new ZodObject({ + // unknownKeys: merging._def.unknownKeys, + // catchall: merging._def.catchall, + // shape: () => + // objectUtil.mergeShapes(this._def.shape(), merging._def.shape()), + // typeName: ZodFirstPartyTypeKind.ZodObject, + // }) as any; + // return merged; + // } + catchall(index) { + return new ZodObject({ + ...this._def, + catchall: index, + }); + } + pick(mask) { + const shape = {}; + for (const key of util.objectKeys(mask)) { + if (mask[key] && this.shape[key]) { + shape[key] = this.shape[key]; + } + } + return new ZodObject({ + ...this._def, + shape: () => shape, + }); + } + omit(mask) { + const shape = {}; + for (const key of util.objectKeys(this.shape)) { + if (!mask[key]) { + shape[key] = this.shape[key]; + } + } + return new ZodObject({ + ...this._def, + shape: () => shape, + }); + } + /** + * @deprecated + */ + deepPartial() { + return deepPartialify(this); + } + partial(mask) { + const newShape = {}; + for (const key of util.objectKeys(this.shape)) { + const fieldSchema = this.shape[key]; + if (mask && !mask[key]) { + newShape[key] = fieldSchema; + } + else { + newShape[key] = fieldSchema.optional(); + } + } + return new ZodObject({ + ...this._def, + shape: () => newShape, + }); + } + required(mask) { + const newShape = {}; + for (const key of util.objectKeys(this.shape)) { + if (mask && !mask[key]) { + newShape[key] = this.shape[key]; + } + else { + const fieldSchema = this.shape[key]; + let newField = fieldSchema; + while (newField instanceof ZodOptional) { + newField = newField._def.innerType; + } + newShape[key] = newField; + } + } + return new ZodObject({ + ...this._def, + shape: () => newShape, + }); + } + keyof() { + return createZodEnum(util.objectKeys(this.shape)); + } +} +ZodObject.create = (shape, params) => { + return new ZodObject({ + shape: () => shape, + unknownKeys: "strip", + catchall: ZodNever.create(), + typeName: ZodFirstPartyTypeKind.ZodObject, + ...processCreateParams(params), + }); +}; +ZodObject.strictCreate = (shape, params) => { + return new ZodObject({ + shape: () => shape, + unknownKeys: "strict", + catchall: ZodNever.create(), + typeName: ZodFirstPartyTypeKind.ZodObject, + ...processCreateParams(params), + }); +}; +ZodObject.lazycreate = (shape, params) => { + return new ZodObject({ + shape, + unknownKeys: "strip", + catchall: ZodNever.create(), + typeName: ZodFirstPartyTypeKind.ZodObject, + ...processCreateParams(params), + }); +}; +export class ZodUnion extends ZodType { + _parse(input) { + const { ctx } = this._processInputParams(input); + const options = this._def.options; + function handleResults(results) { + // return first issue-free validation if it exists + for (const result of results) { + if (result.result.status === "valid") { + return result.result; + } + } + for (const result of results) { + if (result.result.status === "dirty") { + // add issues from dirty option + ctx.common.issues.push(...result.ctx.common.issues); + return result.result; + } + } + // return invalid + const unionErrors = results.map((result) => new ZodError(result.ctx.common.issues)); + addIssueToContext(ctx, { + code: ZodIssueCode.invalid_union, + unionErrors, + }); + return INVALID; + } + if (ctx.common.async) { + return Promise.all(options.map(async (option) => { + const childCtx = { + ...ctx, + common: { + ...ctx.common, + issues: [], + }, + parent: null, + }; + return { + result: await option._parseAsync({ + data: ctx.data, + path: ctx.path, + parent: childCtx, + }), + ctx: childCtx, + }; + })).then(handleResults); + } + else { + let dirty = undefined; + const issues = []; + for (const option of options) { + const childCtx = { + ...ctx, + common: { + ...ctx.common, + issues: [], + }, + parent: null, + }; + const result = option._parseSync({ + data: ctx.data, + path: ctx.path, + parent: childCtx, + }); + if (result.status === "valid") { + return result; + } + else if (result.status === "dirty" && !dirty) { + dirty = { result, ctx: childCtx }; + } + if (childCtx.common.issues.length) { + issues.push(childCtx.common.issues); + } + } + if (dirty) { + ctx.common.issues.push(...dirty.ctx.common.issues); + return dirty.result; + } + const unionErrors = issues.map((issues) => new ZodError(issues)); + addIssueToContext(ctx, { + code: ZodIssueCode.invalid_union, + unionErrors, + }); + return INVALID; + } + } + get options() { + return this._def.options; + } +} +ZodUnion.create = (types, params) => { + return new ZodUnion({ + options: types, + typeName: ZodFirstPartyTypeKind.ZodUnion, + ...processCreateParams(params), + }); +}; +///////////////////////////////////////////////////// +///////////////////////////////////////////////////// +////////// ////////// +////////// ZodDiscriminatedUnion ////////// +////////// ////////// +///////////////////////////////////////////////////// +///////////////////////////////////////////////////// +const getDiscriminator = (type) => { + if (type instanceof ZodLazy) { + return getDiscriminator(type.schema); + } + else if (type instanceof ZodEffects) { + return getDiscriminator(type.innerType()); + } + else if (type instanceof ZodLiteral) { + return [type.value]; + } + else if (type instanceof ZodEnum) { + return type.options; + } + else if (type instanceof ZodNativeEnum) { + // eslint-disable-next-line ban/ban + return util.objectValues(type.enum); + } + else if (type instanceof ZodDefault) { + return getDiscriminator(type._def.innerType); + } + else if (type instanceof ZodUndefined) { + return [undefined]; + } + else if (type instanceof ZodNull) { + return [null]; + } + else if (type instanceof ZodOptional) { + return [undefined, ...getDiscriminator(type.unwrap())]; + } + else if (type instanceof ZodNullable) { + return [null, ...getDiscriminator(type.unwrap())]; + } + else if (type instanceof ZodBranded) { + return getDiscriminator(type.unwrap()); + } + else if (type instanceof ZodReadonly) { + return getDiscriminator(type.unwrap()); + } + else if (type instanceof ZodCatch) { + return getDiscriminator(type._def.innerType); + } + else { + return []; + } +}; +export class ZodDiscriminatedUnion extends ZodType { + _parse(input) { + const { ctx } = this._processInputParams(input); + if (ctx.parsedType !== ZodParsedType.object) { + addIssueToContext(ctx, { + code: ZodIssueCode.invalid_type, + expected: ZodParsedType.object, + received: ctx.parsedType, + }); + return INVALID; + } + const discriminator = this.discriminator; + const discriminatorValue = ctx.data[discriminator]; + const option = this.optionsMap.get(discriminatorValue); + if (!option) { + addIssueToContext(ctx, { + code: ZodIssueCode.invalid_union_discriminator, + options: Array.from(this.optionsMap.keys()), + path: [discriminator], + }); + return INVALID; + } + if (ctx.common.async) { + return option._parseAsync({ + data: ctx.data, + path: ctx.path, + parent: ctx, + }); + } + else { + return option._parseSync({ + data: ctx.data, + path: ctx.path, + parent: ctx, + }); + } + } + get discriminator() { + return this._def.discriminator; + } + get options() { + return this._def.options; + } + get optionsMap() { + return this._def.optionsMap; + } + /** + * The constructor of the discriminated union schema. Its behaviour is very similar to that of the normal z.union() constructor. + * However, it only allows a union of objects, all of which need to share a discriminator property. This property must + * have a different value for each object in the union. + * @param discriminator the name of the discriminator property + * @param types an array of object schemas + * @param params + */ + static create(discriminator, options, params) { + // Get all the valid discriminator values + const optionsMap = new Map(); + // try { + for (const type of options) { + const discriminatorValues = getDiscriminator(type.shape[discriminator]); + if (!discriminatorValues.length) { + throw new Error(`A discriminator value for key \`${discriminator}\` could not be extracted from all schema options`); + } + for (const value of discriminatorValues) { + if (optionsMap.has(value)) { + throw new Error(`Discriminator property ${String(discriminator)} has duplicate value ${String(value)}`); + } + optionsMap.set(value, type); + } + } + return new ZodDiscriminatedUnion({ + typeName: ZodFirstPartyTypeKind.ZodDiscriminatedUnion, + discriminator, + options, + optionsMap, + ...processCreateParams(params), + }); + } +} +function mergeValues(a, b) { + const aType = getParsedType(a); + const bType = getParsedType(b); + if (a === b) { + return { valid: true, data: a }; + } + else if (aType === ZodParsedType.object && bType === ZodParsedType.object) { + const bKeys = util.objectKeys(b); + const sharedKeys = util.objectKeys(a).filter((key) => bKeys.indexOf(key) !== -1); + const newObj = { ...a, ...b }; + for (const key of sharedKeys) { + const sharedValue = mergeValues(a[key], b[key]); + if (!sharedValue.valid) { + return { valid: false }; + } + newObj[key] = sharedValue.data; + } + return { valid: true, data: newObj }; + } + else if (aType === ZodParsedType.array && bType === ZodParsedType.array) { + if (a.length !== b.length) { + return { valid: false }; + } + const newArray = []; + for (let index = 0; index < a.length; index++) { + const itemA = a[index]; + const itemB = b[index]; + const sharedValue = mergeValues(itemA, itemB); + if (!sharedValue.valid) { + return { valid: false }; + } + newArray.push(sharedValue.data); + } + return { valid: true, data: newArray }; + } + else if (aType === ZodParsedType.date && bType === ZodParsedType.date && +a === +b) { + return { valid: true, data: a }; + } + else { + return { valid: false }; + } +} +export class ZodIntersection extends ZodType { + _parse(input) { + const { status, ctx } = this._processInputParams(input); + const handleParsed = (parsedLeft, parsedRight) => { + if (isAborted(parsedLeft) || isAborted(parsedRight)) { + return INVALID; + } + const merged = mergeValues(parsedLeft.value, parsedRight.value); + if (!merged.valid) { + addIssueToContext(ctx, { + code: ZodIssueCode.invalid_intersection_types, + }); + return INVALID; + } + if (isDirty(parsedLeft) || isDirty(parsedRight)) { + status.dirty(); + } + return { status: status.value, value: merged.data }; + }; + if (ctx.common.async) { + return Promise.all([ + this._def.left._parseAsync({ + data: ctx.data, + path: ctx.path, + parent: ctx, + }), + this._def.right._parseAsync({ + data: ctx.data, + path: ctx.path, + parent: ctx, + }), + ]).then(([left, right]) => handleParsed(left, right)); + } + else { + return handleParsed(this._def.left._parseSync({ + data: ctx.data, + path: ctx.path, + parent: ctx, + }), this._def.right._parseSync({ + data: ctx.data, + path: ctx.path, + parent: ctx, + })); + } + } +} +ZodIntersection.create = (left, right, params) => { + return new ZodIntersection({ + left: left, + right: right, + typeName: ZodFirstPartyTypeKind.ZodIntersection, + ...processCreateParams(params), + }); +}; +// type ZodTupleItems = [ZodTypeAny, ...ZodTypeAny[]]; +export class ZodTuple extends ZodType { + _parse(input) { + const { status, ctx } = this._processInputParams(input); + if (ctx.parsedType !== ZodParsedType.array) { + addIssueToContext(ctx, { + code: ZodIssueCode.invalid_type, + expected: ZodParsedType.array, + received: ctx.parsedType, + }); + return INVALID; + } + if (ctx.data.length < this._def.items.length) { + addIssueToContext(ctx, { + code: ZodIssueCode.too_small, + minimum: this._def.items.length, + inclusive: true, + exact: false, + type: "array", + }); + return INVALID; + } + const rest = this._def.rest; + if (!rest && ctx.data.length > this._def.items.length) { + addIssueToContext(ctx, { + code: ZodIssueCode.too_big, + maximum: this._def.items.length, + inclusive: true, + exact: false, + type: "array", + }); + status.dirty(); + } + const items = [...ctx.data] + .map((item, itemIndex) => { + const schema = this._def.items[itemIndex] || this._def.rest; + if (!schema) + return null; + return schema._parse(new ParseInputLazyPath(ctx, item, ctx.path, itemIndex)); + }) + .filter((x) => !!x); // filter nulls + if (ctx.common.async) { + return Promise.all(items).then((results) => { + return ParseStatus.mergeArray(status, results); + }); + } + else { + return ParseStatus.mergeArray(status, items); + } + } + get items() { + return this._def.items; + } + rest(rest) { + return new ZodTuple({ + ...this._def, + rest, + }); + } +} +ZodTuple.create = (schemas, params) => { + if (!Array.isArray(schemas)) { + throw new Error("You must pass an array of schemas to z.tuple([ ... ])"); + } + return new ZodTuple({ + items: schemas, + typeName: ZodFirstPartyTypeKind.ZodTuple, + rest: null, + ...processCreateParams(params), + }); +}; +export class ZodRecord extends ZodType { + get keySchema() { + return this._def.keyType; + } + get valueSchema() { + return this._def.valueType; + } + _parse(input) { + const { status, ctx } = this._processInputParams(input); + if (ctx.parsedType !== ZodParsedType.object) { + addIssueToContext(ctx, { + code: ZodIssueCode.invalid_type, + expected: ZodParsedType.object, + received: ctx.parsedType, + }); + return INVALID; + } + const pairs = []; + const keyType = this._def.keyType; + const valueType = this._def.valueType; + for (const key in ctx.data) { + pairs.push({ + key: keyType._parse(new ParseInputLazyPath(ctx, key, ctx.path, key)), + value: valueType._parse(new ParseInputLazyPath(ctx, ctx.data[key], ctx.path, key)), + alwaysSet: key in ctx.data, + }); + } + if (ctx.common.async) { + return ParseStatus.mergeObjectAsync(status, pairs); + } + else { + return ParseStatus.mergeObjectSync(status, pairs); + } + } + get element() { + return this._def.valueType; + } + static create(first, second, third) { + if (second instanceof ZodType) { + return new ZodRecord({ + keyType: first, + valueType: second, + typeName: ZodFirstPartyTypeKind.ZodRecord, + ...processCreateParams(third), + }); + } + return new ZodRecord({ + keyType: ZodString.create(), + valueType: first, + typeName: ZodFirstPartyTypeKind.ZodRecord, + ...processCreateParams(second), + }); + } +} +export class ZodMap extends ZodType { + get keySchema() { + return this._def.keyType; + } + get valueSchema() { + return this._def.valueType; + } + _parse(input) { + const { status, ctx } = this._processInputParams(input); + if (ctx.parsedType !== ZodParsedType.map) { + addIssueToContext(ctx, { + code: ZodIssueCode.invalid_type, + expected: ZodParsedType.map, + received: ctx.parsedType, + }); + return INVALID; + } + const keyType = this._def.keyType; + const valueType = this._def.valueType; + const pairs = [...ctx.data.entries()].map(([key, value], index) => { + return { + key: keyType._parse(new ParseInputLazyPath(ctx, key, ctx.path, [index, "key"])), + value: valueType._parse(new ParseInputLazyPath(ctx, value, ctx.path, [index, "value"])), + }; + }); + if (ctx.common.async) { + const finalMap = new Map(); + return Promise.resolve().then(async () => { + for (const pair of pairs) { + const key = await pair.key; + const value = await pair.value; + if (key.status === "aborted" || value.status === "aborted") { + return INVALID; + } + if (key.status === "dirty" || value.status === "dirty") { + status.dirty(); + } + finalMap.set(key.value, value.value); + } + return { status: status.value, value: finalMap }; + }); + } + else { + const finalMap = new Map(); + for (const pair of pairs) { + const key = pair.key; + const value = pair.value; + if (key.status === "aborted" || value.status === "aborted") { + return INVALID; + } + if (key.status === "dirty" || value.status === "dirty") { + status.dirty(); + } + finalMap.set(key.value, value.value); + } + return { status: status.value, value: finalMap }; + } + } +} +ZodMap.create = (keyType, valueType, params) => { + return new ZodMap({ + valueType, + keyType, + typeName: ZodFirstPartyTypeKind.ZodMap, + ...processCreateParams(params), + }); +}; +export class ZodSet extends ZodType { + _parse(input) { + const { status, ctx } = this._processInputParams(input); + if (ctx.parsedType !== ZodParsedType.set) { + addIssueToContext(ctx, { + code: ZodIssueCode.invalid_type, + expected: ZodParsedType.set, + received: ctx.parsedType, + }); + return INVALID; + } + const def = this._def; + if (def.minSize !== null) { + if (ctx.data.size < def.minSize.value) { + addIssueToContext(ctx, { + code: ZodIssueCode.too_small, + minimum: def.minSize.value, + type: "set", + inclusive: true, + exact: false, + message: def.minSize.message, + }); + status.dirty(); + } + } + if (def.maxSize !== null) { + if (ctx.data.size > def.maxSize.value) { + addIssueToContext(ctx, { + code: ZodIssueCode.too_big, + maximum: def.maxSize.value, + type: "set", + inclusive: true, + exact: false, + message: def.maxSize.message, + }); + status.dirty(); + } + } + const valueType = this._def.valueType; + function finalizeSet(elements) { + const parsedSet = new Set(); + for (const element of elements) { + if (element.status === "aborted") + return INVALID; + if (element.status === "dirty") + status.dirty(); + parsedSet.add(element.value); + } + return { status: status.value, value: parsedSet }; + } + const elements = [...ctx.data.values()].map((item, i) => valueType._parse(new ParseInputLazyPath(ctx, item, ctx.path, i))); + if (ctx.common.async) { + return Promise.all(elements).then((elements) => finalizeSet(elements)); + } + else { + return finalizeSet(elements); + } + } + min(minSize, message) { + return new ZodSet({ + ...this._def, + minSize: { value: minSize, message: errorUtil.toString(message) }, + }); + } + max(maxSize, message) { + return new ZodSet({ + ...this._def, + maxSize: { value: maxSize, message: errorUtil.toString(message) }, + }); + } + size(size, message) { + return this.min(size, message).max(size, message); + } + nonempty(message) { + return this.min(1, message); + } +} +ZodSet.create = (valueType, params) => { + return new ZodSet({ + valueType, + minSize: null, + maxSize: null, + typeName: ZodFirstPartyTypeKind.ZodSet, + ...processCreateParams(params), + }); +}; +export class ZodFunction extends ZodType { + constructor() { + super(...arguments); + this.validate = this.implement; + } + _parse(input) { + const { ctx } = this._processInputParams(input); + if (ctx.parsedType !== ZodParsedType.function) { + addIssueToContext(ctx, { + code: ZodIssueCode.invalid_type, + expected: ZodParsedType.function, + received: ctx.parsedType, + }); + return INVALID; + } + function makeArgsIssue(args, error) { + return makeIssue({ + data: args, + path: ctx.path, + errorMaps: [ctx.common.contextualErrorMap, ctx.schemaErrorMap, getErrorMap(), defaultErrorMap].filter((x) => !!x), + issueData: { + code: ZodIssueCode.invalid_arguments, + argumentsError: error, + }, + }); + } + function makeReturnsIssue(returns, error) { + return makeIssue({ + data: returns, + path: ctx.path, + errorMaps: [ctx.common.contextualErrorMap, ctx.schemaErrorMap, getErrorMap(), defaultErrorMap].filter((x) => !!x), + issueData: { + code: ZodIssueCode.invalid_return_type, + returnTypeError: error, + }, + }); + } + const params = { errorMap: ctx.common.contextualErrorMap }; + const fn = ctx.data; + if (this._def.returns instanceof ZodPromise) { + // Would love a way to avoid disabling this rule, but we need + // an alias (using an arrow function was what caused 2651). + // eslint-disable-next-line @typescript-eslint/no-this-alias + const me = this; + return OK(async function (...args) { + const error = new ZodError([]); + const parsedArgs = await me._def.args.parseAsync(args, params).catch((e) => { + error.addIssue(makeArgsIssue(args, e)); + throw error; + }); + const result = await Reflect.apply(fn, this, parsedArgs); + const parsedReturns = await me._def.returns._def.type + .parseAsync(result, params) + .catch((e) => { + error.addIssue(makeReturnsIssue(result, e)); + throw error; + }); + return parsedReturns; + }); + } + else { + // Would love a way to avoid disabling this rule, but we need + // an alias (using an arrow function was what caused 2651). + // eslint-disable-next-line @typescript-eslint/no-this-alias + const me = this; + return OK(function (...args) { + const parsedArgs = me._def.args.safeParse(args, params); + if (!parsedArgs.success) { + throw new ZodError([makeArgsIssue(args, parsedArgs.error)]); + } + const result = Reflect.apply(fn, this, parsedArgs.data); + const parsedReturns = me._def.returns.safeParse(result, params); + if (!parsedReturns.success) { + throw new ZodError([makeReturnsIssue(result, parsedReturns.error)]); + } + return parsedReturns.data; + }); + } + } + parameters() { + return this._def.args; + } + returnType() { + return this._def.returns; + } + args(...items) { + return new ZodFunction({ + ...this._def, + args: ZodTuple.create(items).rest(ZodUnknown.create()), + }); + } + returns(returnType) { + return new ZodFunction({ + ...this._def, + returns: returnType, + }); + } + implement(func) { + const validatedFunc = this.parse(func); + return validatedFunc; + } + strictImplement(func) { + const validatedFunc = this.parse(func); + return validatedFunc; + } + static create(args, returns, params) { + return new ZodFunction({ + args: (args ? args : ZodTuple.create([]).rest(ZodUnknown.create())), + returns: returns || ZodUnknown.create(), + typeName: ZodFirstPartyTypeKind.ZodFunction, + ...processCreateParams(params), + }); + } +} +export class ZodLazy extends ZodType { + get schema() { + return this._def.getter(); + } + _parse(input) { + const { ctx } = this._processInputParams(input); + const lazySchema = this._def.getter(); + return lazySchema._parse({ data: ctx.data, path: ctx.path, parent: ctx }); + } +} +ZodLazy.create = (getter, params) => { + return new ZodLazy({ + getter: getter, + typeName: ZodFirstPartyTypeKind.ZodLazy, + ...processCreateParams(params), + }); +}; +export class ZodLiteral extends ZodType { + _parse(input) { + if (input.data !== this._def.value) { + const ctx = this._getOrReturnCtx(input); + addIssueToContext(ctx, { + received: ctx.data, + code: ZodIssueCode.invalid_literal, + expected: this._def.value, + }); + return INVALID; + } + return { status: "valid", value: input.data }; + } + get value() { + return this._def.value; + } +} +ZodLiteral.create = (value, params) => { + return new ZodLiteral({ + value: value, + typeName: ZodFirstPartyTypeKind.ZodLiteral, + ...processCreateParams(params), + }); +}; +function createZodEnum(values, params) { + return new ZodEnum({ + values, + typeName: ZodFirstPartyTypeKind.ZodEnum, + ...processCreateParams(params), + }); +} +export class ZodEnum extends ZodType { + _parse(input) { + if (typeof input.data !== "string") { + const ctx = this._getOrReturnCtx(input); + const expectedValues = this._def.values; + addIssueToContext(ctx, { + expected: util.joinValues(expectedValues), + received: ctx.parsedType, + code: ZodIssueCode.invalid_type, + }); + return INVALID; + } + if (!this._cache) { + this._cache = new Set(this._def.values); + } + if (!this._cache.has(input.data)) { + const ctx = this._getOrReturnCtx(input); + const expectedValues = this._def.values; + addIssueToContext(ctx, { + received: ctx.data, + code: ZodIssueCode.invalid_enum_value, + options: expectedValues, + }); + return INVALID; + } + return OK(input.data); + } + get options() { + return this._def.values; + } + get enum() { + const enumValues = {}; + for (const val of this._def.values) { + enumValues[val] = val; + } + return enumValues; + } + get Values() { + const enumValues = {}; + for (const val of this._def.values) { + enumValues[val] = val; + } + return enumValues; + } + get Enum() { + const enumValues = {}; + for (const val of this._def.values) { + enumValues[val] = val; + } + return enumValues; + } + extract(values, newDef = this._def) { + return ZodEnum.create(values, { + ...this._def, + ...newDef, + }); + } + exclude(values, newDef = this._def) { + return ZodEnum.create(this.options.filter((opt) => !values.includes(opt)), { + ...this._def, + ...newDef, + }); + } +} +ZodEnum.create = createZodEnum; +export class ZodNativeEnum extends ZodType { + _parse(input) { + const nativeEnumValues = util.getValidEnumValues(this._def.values); + const ctx = this._getOrReturnCtx(input); + if (ctx.parsedType !== ZodParsedType.string && ctx.parsedType !== ZodParsedType.number) { + const expectedValues = util.objectValues(nativeEnumValues); + addIssueToContext(ctx, { + expected: util.joinValues(expectedValues), + received: ctx.parsedType, + code: ZodIssueCode.invalid_type, + }); + return INVALID; + } + if (!this._cache) { + this._cache = new Set(util.getValidEnumValues(this._def.values)); + } + if (!this._cache.has(input.data)) { + const expectedValues = util.objectValues(nativeEnumValues); + addIssueToContext(ctx, { + received: ctx.data, + code: ZodIssueCode.invalid_enum_value, + options: expectedValues, + }); + return INVALID; + } + return OK(input.data); + } + get enum() { + return this._def.values; + } +} +ZodNativeEnum.create = (values, params) => { + return new ZodNativeEnum({ + values: values, + typeName: ZodFirstPartyTypeKind.ZodNativeEnum, + ...processCreateParams(params), + }); +}; +export class ZodPromise extends ZodType { + unwrap() { + return this._def.type; + } + _parse(input) { + const { ctx } = this._processInputParams(input); + if (ctx.parsedType !== ZodParsedType.promise && ctx.common.async === false) { + addIssueToContext(ctx, { + code: ZodIssueCode.invalid_type, + expected: ZodParsedType.promise, + received: ctx.parsedType, + }); + return INVALID; + } + const promisified = ctx.parsedType === ZodParsedType.promise ? ctx.data : Promise.resolve(ctx.data); + return OK(promisified.then((data) => { + return this._def.type.parseAsync(data, { + path: ctx.path, + errorMap: ctx.common.contextualErrorMap, + }); + })); + } +} +ZodPromise.create = (schema, params) => { + return new ZodPromise({ + type: schema, + typeName: ZodFirstPartyTypeKind.ZodPromise, + ...processCreateParams(params), + }); +}; +export class ZodEffects extends ZodType { + innerType() { + return this._def.schema; + } + sourceType() { + return this._def.schema._def.typeName === ZodFirstPartyTypeKind.ZodEffects + ? this._def.schema.sourceType() + : this._def.schema; + } + _parse(input) { + const { status, ctx } = this._processInputParams(input); + const effect = this._def.effect || null; + const checkCtx = { + addIssue: (arg) => { + addIssueToContext(ctx, arg); + if (arg.fatal) { + status.abort(); + } + else { + status.dirty(); + } + }, + get path() { + return ctx.path; + }, + }; + checkCtx.addIssue = checkCtx.addIssue.bind(checkCtx); + if (effect.type === "preprocess") { + const processed = effect.transform(ctx.data, checkCtx); + if (ctx.common.async) { + return Promise.resolve(processed).then(async (processed) => { + if (status.value === "aborted") + return INVALID; + const result = await this._def.schema._parseAsync({ + data: processed, + path: ctx.path, + parent: ctx, + }); + if (result.status === "aborted") + return INVALID; + if (result.status === "dirty") + return DIRTY(result.value); + if (status.value === "dirty") + return DIRTY(result.value); + return result; + }); + } + else { + if (status.value === "aborted") + return INVALID; + const result = this._def.schema._parseSync({ + data: processed, + path: ctx.path, + parent: ctx, + }); + if (result.status === "aborted") + return INVALID; + if (result.status === "dirty") + return DIRTY(result.value); + if (status.value === "dirty") + return DIRTY(result.value); + return result; + } + } + if (effect.type === "refinement") { + const executeRefinement = (acc) => { + const result = effect.refinement(acc, checkCtx); + if (ctx.common.async) { + return Promise.resolve(result); + } + if (result instanceof Promise) { + throw new Error("Async refinement encountered during synchronous parse operation. Use .parseAsync instead."); + } + return acc; + }; + if (ctx.common.async === false) { + const inner = this._def.schema._parseSync({ + data: ctx.data, + path: ctx.path, + parent: ctx, + }); + if (inner.status === "aborted") + return INVALID; + if (inner.status === "dirty") + status.dirty(); + // return value is ignored + executeRefinement(inner.value); + return { status: status.value, value: inner.value }; + } + else { + return this._def.schema._parseAsync({ data: ctx.data, path: ctx.path, parent: ctx }).then((inner) => { + if (inner.status === "aborted") + return INVALID; + if (inner.status === "dirty") + status.dirty(); + return executeRefinement(inner.value).then(() => { + return { status: status.value, value: inner.value }; + }); + }); + } + } + if (effect.type === "transform") { + if (ctx.common.async === false) { + const base = this._def.schema._parseSync({ + data: ctx.data, + path: ctx.path, + parent: ctx, + }); + if (!isValid(base)) + return INVALID; + const result = effect.transform(base.value, checkCtx); + if (result instanceof Promise) { + throw new Error(`Asynchronous transform encountered during synchronous parse operation. Use .parseAsync instead.`); + } + return { status: status.value, value: result }; + } + else { + return this._def.schema._parseAsync({ data: ctx.data, path: ctx.path, parent: ctx }).then((base) => { + if (!isValid(base)) + return INVALID; + return Promise.resolve(effect.transform(base.value, checkCtx)).then((result) => ({ + status: status.value, + value: result, + })); + }); + } + } + util.assertNever(effect); + } +} +ZodEffects.create = (schema, effect, params) => { + return new ZodEffects({ + schema, + typeName: ZodFirstPartyTypeKind.ZodEffects, + effect, + ...processCreateParams(params), + }); +}; +ZodEffects.createWithPreprocess = (preprocess, schema, params) => { + return new ZodEffects({ + schema, + effect: { type: "preprocess", transform: preprocess }, + typeName: ZodFirstPartyTypeKind.ZodEffects, + ...processCreateParams(params), + }); +}; +export { ZodEffects as ZodTransformer }; +export class ZodOptional extends ZodType { + _parse(input) { + const parsedType = this._getType(input); + if (parsedType === ZodParsedType.undefined) { + return OK(undefined); + } + return this._def.innerType._parse(input); + } + unwrap() { + return this._def.innerType; + } +} +ZodOptional.create = (type, params) => { + return new ZodOptional({ + innerType: type, + typeName: ZodFirstPartyTypeKind.ZodOptional, + ...processCreateParams(params), + }); +}; +export class ZodNullable extends ZodType { + _parse(input) { + const parsedType = this._getType(input); + if (parsedType === ZodParsedType.null) { + return OK(null); + } + return this._def.innerType._parse(input); + } + unwrap() { + return this._def.innerType; + } +} +ZodNullable.create = (type, params) => { + return new ZodNullable({ + innerType: type, + typeName: ZodFirstPartyTypeKind.ZodNullable, + ...processCreateParams(params), + }); +}; +export class ZodDefault extends ZodType { + _parse(input) { + const { ctx } = this._processInputParams(input); + let data = ctx.data; + if (ctx.parsedType === ZodParsedType.undefined) { + data = this._def.defaultValue(); + } + return this._def.innerType._parse({ + data, + path: ctx.path, + parent: ctx, + }); + } + removeDefault() { + return this._def.innerType; + } +} +ZodDefault.create = (type, params) => { + return new ZodDefault({ + innerType: type, + typeName: ZodFirstPartyTypeKind.ZodDefault, + defaultValue: typeof params.default === "function" ? params.default : () => params.default, + ...processCreateParams(params), + }); +}; +export class ZodCatch extends ZodType { + _parse(input) { + const { ctx } = this._processInputParams(input); + // newCtx is used to not collect issues from inner types in ctx + const newCtx = { + ...ctx, + common: { + ...ctx.common, + issues: [], + }, + }; + const result = this._def.innerType._parse({ + data: newCtx.data, + path: newCtx.path, + parent: { + ...newCtx, + }, + }); + if (isAsync(result)) { + return result.then((result) => { + return { + status: "valid", + value: result.status === "valid" + ? result.value + : this._def.catchValue({ + get error() { + return new ZodError(newCtx.common.issues); + }, + input: newCtx.data, + }), + }; + }); + } + else { + return { + status: "valid", + value: result.status === "valid" + ? result.value + : this._def.catchValue({ + get error() { + return new ZodError(newCtx.common.issues); + }, + input: newCtx.data, + }), + }; + } + } + removeCatch() { + return this._def.innerType; + } +} +ZodCatch.create = (type, params) => { + return new ZodCatch({ + innerType: type, + typeName: ZodFirstPartyTypeKind.ZodCatch, + catchValue: typeof params.catch === "function" ? params.catch : () => params.catch, + ...processCreateParams(params), + }); +}; +export class ZodNaN extends ZodType { + _parse(input) { + const parsedType = this._getType(input); + if (parsedType !== ZodParsedType.nan) { + const ctx = this._getOrReturnCtx(input); + addIssueToContext(ctx, { + code: ZodIssueCode.invalid_type, + expected: ZodParsedType.nan, + received: ctx.parsedType, + }); + return INVALID; + } + return { status: "valid", value: input.data }; + } +} +ZodNaN.create = (params) => { + return new ZodNaN({ + typeName: ZodFirstPartyTypeKind.ZodNaN, + ...processCreateParams(params), + }); +}; +export const BRAND = Symbol("zod_brand"); +export class ZodBranded extends ZodType { + _parse(input) { + const { ctx } = this._processInputParams(input); + const data = ctx.data; + return this._def.type._parse({ + data, + path: ctx.path, + parent: ctx, + }); + } + unwrap() { + return this._def.type; + } +} +export class ZodPipeline extends ZodType { + _parse(input) { + const { status, ctx } = this._processInputParams(input); + if (ctx.common.async) { + const handleAsync = async () => { + const inResult = await this._def.in._parseAsync({ + data: ctx.data, + path: ctx.path, + parent: ctx, + }); + if (inResult.status === "aborted") + return INVALID; + if (inResult.status === "dirty") { + status.dirty(); + return DIRTY(inResult.value); + } + else { + return this._def.out._parseAsync({ + data: inResult.value, + path: ctx.path, + parent: ctx, + }); + } + }; + return handleAsync(); + } + else { + const inResult = this._def.in._parseSync({ + data: ctx.data, + path: ctx.path, + parent: ctx, + }); + if (inResult.status === "aborted") + return INVALID; + if (inResult.status === "dirty") { + status.dirty(); + return { + status: "dirty", + value: inResult.value, + }; + } + else { + return this._def.out._parseSync({ + data: inResult.value, + path: ctx.path, + parent: ctx, + }); + } + } + } + static create(a, b) { + return new ZodPipeline({ + in: a, + out: b, + typeName: ZodFirstPartyTypeKind.ZodPipeline, + }); + } +} +export class ZodReadonly extends ZodType { + _parse(input) { + const result = this._def.innerType._parse(input); + const freeze = (data) => { + if (isValid(data)) { + data.value = Object.freeze(data.value); + } + return data; + }; + return isAsync(result) ? result.then((data) => freeze(data)) : freeze(result); + } + unwrap() { + return this._def.innerType; + } +} +ZodReadonly.create = (type, params) => { + return new ZodReadonly({ + innerType: type, + typeName: ZodFirstPartyTypeKind.ZodReadonly, + ...processCreateParams(params), + }); +}; +//////////////////////////////////////// +//////////////////////////////////////// +////////// ////////// +////////// z.custom ////////// +////////// ////////// +//////////////////////////////////////// +//////////////////////////////////////// +function cleanParams(params, data) { + const p = typeof params === "function" ? params(data) : typeof params === "string" ? { message: params } : params; + const p2 = typeof p === "string" ? { message: p } : p; + return p2; +} +export function custom(check, _params = {}, +/** + * @deprecated + * + * Pass `fatal` into the params object instead: + * + * ```ts + * z.string().custom((val) => val.length > 5, { fatal: false }) + * ``` + * + */ +fatal) { + if (check) + return ZodAny.create().superRefine((data, ctx) => { + const r = check(data); + if (r instanceof Promise) { + return r.then((r) => { + if (!r) { + const params = cleanParams(_params, data); + const _fatal = params.fatal ?? fatal ?? true; + ctx.addIssue({ code: "custom", ...params, fatal: _fatal }); + } + }); + } + if (!r) { + const params = cleanParams(_params, data); + const _fatal = params.fatal ?? fatal ?? true; + ctx.addIssue({ code: "custom", ...params, fatal: _fatal }); + } + return; + }); + return ZodAny.create(); +} +export { ZodType as Schema, ZodType as ZodSchema }; +export const late = { + object: ZodObject.lazycreate, +}; +export var ZodFirstPartyTypeKind; +(function (ZodFirstPartyTypeKind) { + ZodFirstPartyTypeKind["ZodString"] = "ZodString"; + ZodFirstPartyTypeKind["ZodNumber"] = "ZodNumber"; + ZodFirstPartyTypeKind["ZodNaN"] = "ZodNaN"; + ZodFirstPartyTypeKind["ZodBigInt"] = "ZodBigInt"; + ZodFirstPartyTypeKind["ZodBoolean"] = "ZodBoolean"; + ZodFirstPartyTypeKind["ZodDate"] = "ZodDate"; + ZodFirstPartyTypeKind["ZodSymbol"] = "ZodSymbol"; + ZodFirstPartyTypeKind["ZodUndefined"] = "ZodUndefined"; + ZodFirstPartyTypeKind["ZodNull"] = "ZodNull"; + ZodFirstPartyTypeKind["ZodAny"] = "ZodAny"; + ZodFirstPartyTypeKind["ZodUnknown"] = "ZodUnknown"; + ZodFirstPartyTypeKind["ZodNever"] = "ZodNever"; + ZodFirstPartyTypeKind["ZodVoid"] = "ZodVoid"; + ZodFirstPartyTypeKind["ZodArray"] = "ZodArray"; + ZodFirstPartyTypeKind["ZodObject"] = "ZodObject"; + ZodFirstPartyTypeKind["ZodUnion"] = "ZodUnion"; + ZodFirstPartyTypeKind["ZodDiscriminatedUnion"] = "ZodDiscriminatedUnion"; + ZodFirstPartyTypeKind["ZodIntersection"] = "ZodIntersection"; + ZodFirstPartyTypeKind["ZodTuple"] = "ZodTuple"; + ZodFirstPartyTypeKind["ZodRecord"] = "ZodRecord"; + ZodFirstPartyTypeKind["ZodMap"] = "ZodMap"; + ZodFirstPartyTypeKind["ZodSet"] = "ZodSet"; + ZodFirstPartyTypeKind["ZodFunction"] = "ZodFunction"; + ZodFirstPartyTypeKind["ZodLazy"] = "ZodLazy"; + ZodFirstPartyTypeKind["ZodLiteral"] = "ZodLiteral"; + ZodFirstPartyTypeKind["ZodEnum"] = "ZodEnum"; + ZodFirstPartyTypeKind["ZodEffects"] = "ZodEffects"; + ZodFirstPartyTypeKind["ZodNativeEnum"] = "ZodNativeEnum"; + ZodFirstPartyTypeKind["ZodOptional"] = "ZodOptional"; + ZodFirstPartyTypeKind["ZodNullable"] = "ZodNullable"; + ZodFirstPartyTypeKind["ZodDefault"] = "ZodDefault"; + ZodFirstPartyTypeKind["ZodCatch"] = "ZodCatch"; + ZodFirstPartyTypeKind["ZodPromise"] = "ZodPromise"; + ZodFirstPartyTypeKind["ZodBranded"] = "ZodBranded"; + ZodFirstPartyTypeKind["ZodPipeline"] = "ZodPipeline"; + ZodFirstPartyTypeKind["ZodReadonly"] = "ZodReadonly"; +})(ZodFirstPartyTypeKind || (ZodFirstPartyTypeKind = {})); +// requires TS 4.4+ +class Class { + constructor(..._) { } +} +const instanceOfType = ( +// const instanceOfType = any>( +cls, params = { + message: `Input not instance of ${cls.name}`, +}) => custom((data) => data instanceof cls, params); +const stringType = ZodString.create; +const numberType = ZodNumber.create; +const nanType = ZodNaN.create; +const bigIntType = ZodBigInt.create; +const booleanType = ZodBoolean.create; +const dateType = ZodDate.create; +const symbolType = ZodSymbol.create; +const undefinedType = ZodUndefined.create; +const nullType = ZodNull.create; +const anyType = ZodAny.create; +const unknownType = ZodUnknown.create; +const neverType = ZodNever.create; +const voidType = ZodVoid.create; +const arrayType = ZodArray.create; +const objectType = ZodObject.create; +const strictObjectType = ZodObject.strictCreate; +const unionType = ZodUnion.create; +const discriminatedUnionType = ZodDiscriminatedUnion.create; +const intersectionType = ZodIntersection.create; +const tupleType = ZodTuple.create; +const recordType = ZodRecord.create; +const mapType = ZodMap.create; +const setType = ZodSet.create; +const functionType = ZodFunction.create; +const lazyType = ZodLazy.create; +const literalType = ZodLiteral.create; +const enumType = ZodEnum.create; +const nativeEnumType = ZodNativeEnum.create; +const promiseType = ZodPromise.create; +const effectsType = ZodEffects.create; +const optionalType = ZodOptional.create; +const nullableType = ZodNullable.create; +const preprocessType = ZodEffects.createWithPreprocess; +const pipelineType = ZodPipeline.create; +const ostring = () => stringType().optional(); +const onumber = () => numberType().optional(); +const oboolean = () => booleanType().optional(); +export const coerce = { + string: ((arg) => ZodString.create({ ...arg, coerce: true })), + number: ((arg) => ZodNumber.create({ ...arg, coerce: true })), + boolean: ((arg) => ZodBoolean.create({ + ...arg, + coerce: true, + })), + bigint: ((arg) => ZodBigInt.create({ ...arg, coerce: true })), + date: ((arg) => ZodDate.create({ ...arg, coerce: true })), +}; +export { anyType as any, arrayType as array, bigIntType as bigint, booleanType as boolean, dateType as date, discriminatedUnionType as discriminatedUnion, effectsType as effect, enumType as enum, functionType as function, instanceOfType as instanceof, intersectionType as intersection, lazyType as lazy, literalType as literal, mapType as map, nanType as nan, nativeEnumType as nativeEnum, neverType as never, nullType as null, nullableType as nullable, numberType as number, objectType as object, oboolean, onumber, optionalType as optional, ostring, pipelineType as pipeline, preprocessType as preprocess, promiseType as promise, recordType as record, setType as set, strictObjectType as strictObject, stringType as string, symbolType as symbol, effectsType as transformer, tupleType as tuple, undefinedType as undefined, unionType as union, unknownType as unknown, voidType as void, }; +export const NEVER = INVALID; diff --git a/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/index.d.ts b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/index.d.ts new file mode 100644 index 000000000..8352ab713 --- /dev/null +++ b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/index.d.ts @@ -0,0 +1,3 @@ +import z3 from "./v3/index.js"; +export * from "./v3/index.js"; +export default z3; diff --git a/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/ZodError.d.ts b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/ZodError.d.ts new file mode 100644 index 000000000..4e71355a3 --- /dev/null +++ b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/ZodError.d.ts @@ -0,0 +1,164 @@ +import type { Primitive } from "./helpers/typeAliases.js"; +import { util, type ZodParsedType } from "./helpers/util.js"; +import type { TypeOf, ZodType } from "./index.js"; +type allKeys = T extends any ? keyof T : never; +export type inferFlattenedErrors, U = string> = typeToFlattenedError, U>; +export type typeToFlattenedError = { + formErrors: U[]; + fieldErrors: { + [P in allKeys]?: U[]; + }; +}; +export declare const ZodIssueCode: { + invalid_type: "invalid_type"; + invalid_literal: "invalid_literal"; + custom: "custom"; + invalid_union: "invalid_union"; + invalid_union_discriminator: "invalid_union_discriminator"; + invalid_enum_value: "invalid_enum_value"; + unrecognized_keys: "unrecognized_keys"; + invalid_arguments: "invalid_arguments"; + invalid_return_type: "invalid_return_type"; + invalid_date: "invalid_date"; + invalid_string: "invalid_string"; + too_small: "too_small"; + too_big: "too_big"; + invalid_intersection_types: "invalid_intersection_types"; + not_multiple_of: "not_multiple_of"; + not_finite: "not_finite"; +}; +export type ZodIssueCode = keyof typeof ZodIssueCode; +export type ZodIssueBase = { + path: (string | number)[]; + message?: string | undefined; +}; +export interface ZodInvalidTypeIssue extends ZodIssueBase { + code: typeof ZodIssueCode.invalid_type; + expected: ZodParsedType; + received: ZodParsedType; +} +export interface ZodInvalidLiteralIssue extends ZodIssueBase { + code: typeof ZodIssueCode.invalid_literal; + expected: unknown; + received: unknown; +} +export interface ZodUnrecognizedKeysIssue extends ZodIssueBase { + code: typeof ZodIssueCode.unrecognized_keys; + keys: string[]; +} +export interface ZodInvalidUnionIssue extends ZodIssueBase { + code: typeof ZodIssueCode.invalid_union; + unionErrors: ZodError[]; +} +export interface ZodInvalidUnionDiscriminatorIssue extends ZodIssueBase { + code: typeof ZodIssueCode.invalid_union_discriminator; + options: Primitive[]; +} +export interface ZodInvalidEnumValueIssue extends ZodIssueBase { + received: string | number; + code: typeof ZodIssueCode.invalid_enum_value; + options: (string | number)[]; +} +export interface ZodInvalidArgumentsIssue extends ZodIssueBase { + code: typeof ZodIssueCode.invalid_arguments; + argumentsError: ZodError; +} +export interface ZodInvalidReturnTypeIssue extends ZodIssueBase { + code: typeof ZodIssueCode.invalid_return_type; + returnTypeError: ZodError; +} +export interface ZodInvalidDateIssue extends ZodIssueBase { + code: typeof ZodIssueCode.invalid_date; +} +export type StringValidation = "email" | "url" | "emoji" | "uuid" | "nanoid" | "regex" | "cuid" | "cuid2" | "ulid" | "datetime" | "date" | "time" | "duration" | "ip" | "cidr" | "base64" | "jwt" | "base64url" | { + includes: string; + position?: number | undefined; +} | { + startsWith: string; +} | { + endsWith: string; +}; +export interface ZodInvalidStringIssue extends ZodIssueBase { + code: typeof ZodIssueCode.invalid_string; + validation: StringValidation; +} +export interface ZodTooSmallIssue extends ZodIssueBase { + code: typeof ZodIssueCode.too_small; + minimum: number | bigint; + inclusive: boolean; + exact?: boolean; + type: "array" | "string" | "number" | "set" | "date" | "bigint"; +} +export interface ZodTooBigIssue extends ZodIssueBase { + code: typeof ZodIssueCode.too_big; + maximum: number | bigint; + inclusive: boolean; + exact?: boolean; + type: "array" | "string" | "number" | "set" | "date" | "bigint"; +} +export interface ZodInvalidIntersectionTypesIssue extends ZodIssueBase { + code: typeof ZodIssueCode.invalid_intersection_types; +} +export interface ZodNotMultipleOfIssue extends ZodIssueBase { + code: typeof ZodIssueCode.not_multiple_of; + multipleOf: number | bigint; +} +export interface ZodNotFiniteIssue extends ZodIssueBase { + code: typeof ZodIssueCode.not_finite; +} +export interface ZodCustomIssue extends ZodIssueBase { + code: typeof ZodIssueCode.custom; + params?: { + [k: string]: any; + }; +} +export type DenormalizedError = { + [k: string]: DenormalizedError | string[]; +}; +export type ZodIssueOptionalMessage = ZodInvalidTypeIssue | ZodInvalidLiteralIssue | ZodUnrecognizedKeysIssue | ZodInvalidUnionIssue | ZodInvalidUnionDiscriminatorIssue | ZodInvalidEnumValueIssue | ZodInvalidArgumentsIssue | ZodInvalidReturnTypeIssue | ZodInvalidDateIssue | ZodInvalidStringIssue | ZodTooSmallIssue | ZodTooBigIssue | ZodInvalidIntersectionTypesIssue | ZodNotMultipleOfIssue | ZodNotFiniteIssue | ZodCustomIssue; +export type ZodIssue = ZodIssueOptionalMessage & { + fatal?: boolean | undefined; + message: string; +}; +export declare const quotelessJson: (obj: any) => string; +type recursiveZodFormattedError = T extends [any, ...any[]] ? { + [K in keyof T]?: ZodFormattedError; +} : T extends any[] ? { + [k: number]: ZodFormattedError; +} : T extends object ? { + [K in keyof T]?: ZodFormattedError; +} : unknown; +export type ZodFormattedError = { + _errors: U[]; +} & recursiveZodFormattedError>; +export type inferFormattedError, U = string> = ZodFormattedError, U>; +export declare class ZodError extends Error { + issues: ZodIssue[]; + get errors(): ZodIssue[]; + constructor(issues: ZodIssue[]); + format(): ZodFormattedError; + format(mapper: (issue: ZodIssue) => U): ZodFormattedError; + static create: (issues: ZodIssue[]) => ZodError; + static assert(value: unknown): asserts value is ZodError; + toString(): string; + get message(): string; + get isEmpty(): boolean; + addIssue: (sub: ZodIssue) => void; + addIssues: (subs?: ZodIssue[]) => void; + flatten(): typeToFlattenedError; + flatten(mapper?: (issue: ZodIssue) => U): typeToFlattenedError; + get formErrors(): typeToFlattenedError; +} +type stripPath = T extends any ? util.OmitKeys : never; +export type IssueData = stripPath & { + path?: (string | number)[]; + fatal?: boolean | undefined; +}; +export type ErrorMapCtx = { + defaultError: string; + data: any; +}; +export type ZodErrorMap = (issue: ZodIssueOptionalMessage, _ctx: ErrorMapCtx) => { + message: string; +}; +export {}; diff --git a/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/errors.d.ts b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/errors.d.ts new file mode 100644 index 000000000..52eacf72e --- /dev/null +++ b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/errors.d.ts @@ -0,0 +1,5 @@ +import type { ZodErrorMap } from "./ZodError.js"; +import defaultErrorMap from "./locales/en.js"; +export { defaultErrorMap }; +export declare function setErrorMap(map: ZodErrorMap): void; +export declare function getErrorMap(): ZodErrorMap; diff --git a/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/external.d.ts b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/external.d.ts new file mode 100644 index 000000000..f0a4be4c7 --- /dev/null +++ b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/external.d.ts @@ -0,0 +1,6 @@ +export * from "./errors.js"; +export * from "./helpers/parseUtil.js"; +export * from "./helpers/typeAliases.js"; +export * from "./helpers/util.js"; +export * from "./types.js"; +export * from "./ZodError.js"; diff --git a/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/helpers/enumUtil.d.ts b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/helpers/enumUtil.d.ts new file mode 100644 index 000000000..fad28a711 --- /dev/null +++ b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/helpers/enumUtil.d.ts @@ -0,0 +1,8 @@ +export declare namespace enumUtil { + type UnionToIntersectionFn = (T extends unknown ? (k: () => T) => void : never) extends (k: infer Intersection) => void ? Intersection : never; + type GetUnionLast = UnionToIntersectionFn extends () => infer Last ? Last : never; + type UnionToTuple = [T] extends [never] ? Tuple : UnionToTuple>, [GetUnionLast, ...Tuple]>; + type CastToStringTuple = T extends [string, ...string[]] ? T : never; + export type UnionToTupleString = CastToStringTuple>; + export {}; +} diff --git a/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/helpers/errorUtil.d.ts b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/helpers/errorUtil.d.ts new file mode 100644 index 000000000..bc9b033ff --- /dev/null +++ b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/helpers/errorUtil.d.ts @@ -0,0 +1,9 @@ +export declare namespace errorUtil { + type ErrMessage = string | { + message?: string | undefined; + }; + const errToObj: (message?: ErrMessage) => { + message?: string | undefined; + }; + const toString: (message?: ErrMessage) => string | undefined; +} diff --git a/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/helpers/parseUtil.d.ts b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/helpers/parseUtil.d.ts new file mode 100644 index 000000000..e4dc1d7b3 --- /dev/null +++ b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/helpers/parseUtil.d.ts @@ -0,0 +1,78 @@ +import type { IssueData, ZodErrorMap, ZodIssue } from "../ZodError.js"; +import type { ZodParsedType } from "./util.js"; +export declare const makeIssue: (params: { + data: any; + path: (string | number)[]; + errorMaps: ZodErrorMap[]; + issueData: IssueData; +}) => ZodIssue; +export type ParseParams = { + path: (string | number)[]; + errorMap: ZodErrorMap; + async: boolean; +}; +export type ParsePathComponent = string | number; +export type ParsePath = ParsePathComponent[]; +export declare const EMPTY_PATH: ParsePath; +export interface ParseContext { + readonly common: { + readonly issues: ZodIssue[]; + readonly contextualErrorMap?: ZodErrorMap | undefined; + readonly async: boolean; + }; + readonly path: ParsePath; + readonly schemaErrorMap?: ZodErrorMap | undefined; + readonly parent: ParseContext | null; + readonly data: any; + readonly parsedType: ZodParsedType; +} +export type ParseInput = { + data: any; + path: (string | number)[]; + parent: ParseContext; +}; +export declare function addIssueToContext(ctx: ParseContext, issueData: IssueData): void; +export type ObjectPair = { + key: SyncParseReturnType; + value: SyncParseReturnType; +}; +export declare class ParseStatus { + value: "aborted" | "dirty" | "valid"; + dirty(): void; + abort(): void; + static mergeArray(status: ParseStatus, results: SyncParseReturnType[]): SyncParseReturnType; + static mergeObjectAsync(status: ParseStatus, pairs: { + key: ParseReturnType; + value: ParseReturnType; + }[]): Promise>; + static mergeObjectSync(status: ParseStatus, pairs: { + key: SyncParseReturnType; + value: SyncParseReturnType; + alwaysSet?: boolean; + }[]): SyncParseReturnType; +} +export interface ParseResult { + status: "aborted" | "dirty" | "valid"; + data: any; +} +export type INVALID = { + status: "aborted"; +}; +export declare const INVALID: INVALID; +export type DIRTY = { + status: "dirty"; + value: T; +}; +export declare const DIRTY: (value: T) => DIRTY; +export type OK = { + status: "valid"; + value: T; +}; +export declare const OK: (value: T) => OK; +export type SyncParseReturnType = OK | DIRTY | INVALID; +export type AsyncParseReturnType = Promise>; +export type ParseReturnType = SyncParseReturnType | AsyncParseReturnType; +export declare const isAborted: (x: ParseReturnType) => x is INVALID; +export declare const isDirty: (x: ParseReturnType) => x is OK | DIRTY; +export declare const isValid: (x: ParseReturnType) => x is OK; +export declare const isAsync: (x: ParseReturnType) => x is AsyncParseReturnType; diff --git a/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/helpers/partialUtil.d.ts b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/helpers/partialUtil.d.ts new file mode 100644 index 000000000..d07657a25 --- /dev/null +++ b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/helpers/partialUtil.d.ts @@ -0,0 +1,8 @@ +import type { ZodArray, ZodNullable, ZodObject, ZodOptional, ZodRawShape, ZodTuple, ZodTupleItems, ZodTypeAny } from "../types.js"; +export declare namespace partialUtil { + type DeepPartial = T extends ZodObject ? ZodObject<{ + [k in keyof T["shape"]]: ZodOptional>; + }, T["_def"]["unknownKeys"], T["_def"]["catchall"]> : T extends ZodArray ? ZodArray, Card> : T extends ZodOptional ? ZodOptional> : T extends ZodNullable ? ZodNullable> : T extends ZodTuple ? { + [k in keyof Items]: Items[k] extends ZodTypeAny ? DeepPartial : never; + } extends infer PI ? PI extends ZodTupleItems ? ZodTuple : never : never : T; +} diff --git a/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/helpers/typeAliases.d.ts b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/helpers/typeAliases.d.ts new file mode 100644 index 000000000..32df022ca --- /dev/null +++ b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/helpers/typeAliases.d.ts @@ -0,0 +1,2 @@ +export type Primitive = string | number | symbol | bigint | boolean | null | undefined; +export type Scalars = Primitive | Primitive[]; diff --git a/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/helpers/util.d.ts b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/helpers/util.d.ts new file mode 100644 index 000000000..eeaaeee1c --- /dev/null +++ b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/helpers/util.d.ts @@ -0,0 +1,85 @@ +export declare namespace util { + type AssertEqual = (() => V extends T ? 1 : 2) extends () => V extends U ? 1 : 2 ? true : false; + export type isAny = 0 extends 1 & T ? true : false; + export const assertEqual: (_: AssertEqual) => void; + export function assertIs(_arg: T): void; + export function assertNever(_x: never): never; + export type Omit = Pick>; + export type OmitKeys = Pick>; + export type MakePartial = Omit & Partial>; + export type Exactly = T & Record, never>; + export type InexactPartial = { + [k in keyof T]?: T[k] | undefined; + }; + export const arrayToEnum: (items: U) => { [k in U[number]]: k; }; + export const getValidEnumValues: (obj: any) => any[]; + export const objectValues: (obj: any) => any[]; + export const objectKeys: ObjectConstructor["keys"]; + export const find: (arr: T[], checker: (arg: T) => any) => T | undefined; + export type identity = objectUtil.identity; + export type flatten = objectUtil.flatten; + export type noUndefined = T extends undefined ? never : T; + export const isInteger: NumberConstructor["isInteger"]; + export function joinValues(array: T, separator?: string): string; + export const jsonStringifyReplacer: (_: string, value: any) => any; + export {}; +} +export declare namespace objectUtil { + export type MergeShapes = keyof U & keyof V extends never ? U & V : { + [k in Exclude]: U[k]; + } & V; + type optionalKeys = { + [k in keyof T]: undefined extends T[k] ? k : never; + }[keyof T]; + type requiredKeys = { + [k in keyof T]: undefined extends T[k] ? never : k; + }[keyof T]; + export type addQuestionMarks = { + [K in requiredKeys]: T[K]; + } & { + [K in optionalKeys]?: T[K]; + } & { + [k in keyof T]?: unknown; + }; + export type identity = T; + export type flatten = identity<{ + [k in keyof T]: T[k]; + }>; + export type noNeverKeys = { + [k in keyof T]: [T[k]] extends [never] ? never : k; + }[keyof T]; + export type noNever = identity<{ + [k in noNeverKeys]: k extends keyof T ? T[k] : never; + }>; + export const mergeShapes: (first: U, second: T) => T & U; + export type extendShape = keyof A & keyof B extends never ? A & B : { + [K in keyof A as K extends keyof B ? never : K]: A[K]; + } & { + [K in keyof B]: B[K]; + }; + export {}; +} +export declare const ZodParsedType: { + string: "string"; + nan: "nan"; + number: "number"; + integer: "integer"; + float: "float"; + boolean: "boolean"; + date: "date"; + bigint: "bigint"; + symbol: "symbol"; + function: "function"; + undefined: "undefined"; + null: "null"; + array: "array"; + object: "object"; + unknown: "unknown"; + promise: "promise"; + void: "void"; + never: "never"; + map: "map"; + set: "set"; +}; +export type ZodParsedType = keyof typeof ZodParsedType; +export declare const getParsedType: (data: any) => ZodParsedType; diff --git a/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/index.d.ts b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/index.d.ts new file mode 100644 index 000000000..a9dab5738 --- /dev/null +++ b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/index.d.ts @@ -0,0 +1,4 @@ +import * as z from "./external.js"; +export * from "./external.js"; +export { z }; +export default z; diff --git a/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/locales/en.d.ts b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/locales/en.d.ts new file mode 100644 index 000000000..b2bea1ec4 --- /dev/null +++ b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/locales/en.d.ts @@ -0,0 +1,3 @@ +import { type ZodErrorMap } from "../ZodError.js"; +declare const errorMap: ZodErrorMap; +export default errorMap; diff --git a/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/standard-schema.d.ts b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/standard-schema.d.ts new file mode 100644 index 000000000..7ddff3b7f --- /dev/null +++ b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/standard-schema.d.ts @@ -0,0 +1,102 @@ +/** + * The Standard Schema interface. + */ +export type StandardSchemaV1 = { + /** + * The Standard Schema properties. + */ + readonly "~standard": StandardSchemaV1.Props; +}; +export declare namespace StandardSchemaV1 { + /** + * The Standard Schema properties interface. + */ + export interface Props { + /** + * The version number of the standard. + */ + readonly version: 1; + /** + * The vendor name of the schema library. + */ + readonly vendor: string; + /** + * Validates unknown input values. + */ + readonly validate: (value: unknown) => Result | Promise>; + /** + * Inferred types associated with the schema. + */ + readonly types?: Types | undefined; + } + /** + * The result interface of the validate function. + */ + export type Result = SuccessResult | FailureResult; + /** + * The result interface if validation succeeds. + */ + export interface SuccessResult { + /** + * The typed output value. + */ + readonly value: Output; + /** + * The non-existent issues. + */ + readonly issues?: undefined; + } + /** + * The result interface if validation fails. + */ + export interface FailureResult { + /** + * The issues of failed validation. + */ + readonly issues: ReadonlyArray; + } + /** + * The issue interface of the failure output. + */ + export interface Issue { + /** + * The error message of the issue. + */ + readonly message: string; + /** + * The path of the issue, if any. + */ + readonly path?: ReadonlyArray | undefined; + } + /** + * The path segment interface of the issue. + */ + export interface PathSegment { + /** + * The key representing a path segment. + */ + readonly key: PropertyKey; + } + /** + * The Standard Schema types interface. + */ + export interface Types { + /** + * The input type of the schema. + */ + readonly input: Input; + /** + * The output type of the schema. + */ + readonly output: Output; + } + /** + * Infers the input type of a Standard Schema. + */ + export type InferInput = NonNullable["input"]; + /** + * Infers the output type of a Standard Schema. + */ + export type InferOutput = NonNullable["output"]; + export {}; +} diff --git a/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/types.d.ts b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/types.d.ts new file mode 100644 index 000000000..301e09128 --- /dev/null +++ b/tests/fixtures/npm/cases/zod3/node_modules/zod/dist/types/v3/types.d.ts @@ -0,0 +1,1031 @@ +import { type IssueData, type StringValidation, type ZodCustomIssue, ZodError, type ZodErrorMap } from "./ZodError.js"; +import type { enumUtil } from "./helpers/enumUtil.js"; +import { errorUtil } from "./helpers/errorUtil.js"; +import { type AsyncParseReturnType, INVALID, type ParseContext, type ParseInput, type ParseParams, type ParseReturnType, ParseStatus, type SyncParseReturnType } from "./helpers/parseUtil.js"; +import type { partialUtil } from "./helpers/partialUtil.js"; +import type { Primitive } from "./helpers/typeAliases.js"; +import { util, type objectUtil } from "./helpers/util.js"; +import type { StandardSchemaV1 } from "./standard-schema.js"; +export interface RefinementCtx { + addIssue: (arg: IssueData) => void; + path: (string | number)[]; +} +export type ZodRawShape = { + [k: string]: ZodTypeAny; +}; +export type ZodTypeAny = ZodType; +export type TypeOf> = T["_output"]; +export type input> = T["_input"]; +export type output> = T["_output"]; +export type { TypeOf as infer }; +export type CustomErrorParams = Partial>; +export interface ZodTypeDef { + errorMap?: ZodErrorMap | undefined; + description?: string | undefined; +} +export type RawCreateParams = { + errorMap?: ZodErrorMap | undefined; + invalid_type_error?: string | undefined; + required_error?: string | undefined; + message?: string | undefined; + description?: string | undefined; +} | undefined; +export type ProcessedCreateParams = { + errorMap?: ZodErrorMap | undefined; + description?: string | undefined; +}; +export type SafeParseSuccess = { + success: true; + data: Output; + error?: never; +}; +export type SafeParseError = { + success: false; + error: ZodError; + data?: never; +}; +export type SafeParseReturnType = SafeParseSuccess | SafeParseError; +export declare abstract class ZodType { + readonly _type: Output; + readonly _output: Output; + readonly _input: Input; + readonly _def: Def; + get description(): string | undefined; + "~standard": StandardSchemaV1.Props; + abstract _parse(input: ParseInput): ParseReturnType; + _getType(input: ParseInput): string; + _getOrReturnCtx(input: ParseInput, ctx?: ParseContext | undefined): ParseContext; + _processInputParams(input: ParseInput): { + status: ParseStatus; + ctx: ParseContext; + }; + _parseSync(input: ParseInput): SyncParseReturnType; + _parseAsync(input: ParseInput): AsyncParseReturnType; + parse(data: unknown, params?: util.InexactPartial): Output; + safeParse(data: unknown, params?: util.InexactPartial): SafeParseReturnType; + "~validate"(data: unknown): StandardSchemaV1.Result | Promise>; + parseAsync(data: unknown, params?: util.InexactPartial): Promise; + safeParseAsync(data: unknown, params?: util.InexactPartial): Promise>; + /** Alias of safeParseAsync */ + spa: (data: unknown, params?: util.InexactPartial) => Promise>; + refine(check: (arg: Output) => arg is RefinedOutput, message?: string | CustomErrorParams | ((arg: Output) => CustomErrorParams)): ZodEffects; + refine(check: (arg: Output) => unknown | Promise, message?: string | CustomErrorParams | ((arg: Output) => CustomErrorParams)): ZodEffects; + refinement(check: (arg: Output) => arg is RefinedOutput, refinementData: IssueData | ((arg: Output, ctx: RefinementCtx) => IssueData)): ZodEffects; + refinement(check: (arg: Output) => boolean, refinementData: IssueData | ((arg: Output, ctx: RefinementCtx) => IssueData)): ZodEffects; + _refinement(refinement: RefinementEffect["refinement"]): ZodEffects; + superRefine(refinement: (arg: Output, ctx: RefinementCtx) => arg is RefinedOutput): ZodEffects; + superRefine(refinement: (arg: Output, ctx: RefinementCtx) => void): ZodEffects; + superRefine(refinement: (arg: Output, ctx: RefinementCtx) => Promise): ZodEffects; + constructor(def: Def); + optional(): ZodOptional; + nullable(): ZodNullable; + nullish(): ZodOptional>; + array(): ZodArray; + promise(): ZodPromise; + or(option: T): ZodUnion<[this, T]>; + and(incoming: T): ZodIntersection; + transform(transform: (arg: Output, ctx: RefinementCtx) => NewOut | Promise): ZodEffects; + default(def: util.noUndefined): ZodDefault; + default(def: () => util.noUndefined): ZodDefault; + brand(brand?: B): ZodBranded; + catch(def: Output): ZodCatch; + catch(def: (ctx: { + error: ZodError; + input: Input; + }) => Output): ZodCatch; + describe(description: string): this; + pipe(target: T): ZodPipeline; + readonly(): ZodReadonly; + isOptional(): boolean; + isNullable(): boolean; +} +export type IpVersion = "v4" | "v6"; +export type ZodStringCheck = { + kind: "min"; + value: number; + message?: string | undefined; +} | { + kind: "max"; + value: number; + message?: string | undefined; +} | { + kind: "length"; + value: number; + message?: string | undefined; +} | { + kind: "email"; + message?: string | undefined; +} | { + kind: "url"; + message?: string | undefined; +} | { + kind: "emoji"; + message?: string | undefined; +} | { + kind: "uuid"; + message?: string | undefined; +} | { + kind: "nanoid"; + message?: string | undefined; +} | { + kind: "cuid"; + message?: string | undefined; +} | { + kind: "includes"; + value: string; + position?: number | undefined; + message?: string | undefined; +} | { + kind: "cuid2"; + message?: string | undefined; +} | { + kind: "ulid"; + message?: string | undefined; +} | { + kind: "startsWith"; + value: string; + message?: string | undefined; +} | { + kind: "endsWith"; + value: string; + message?: string | undefined; +} | { + kind: "regex"; + regex: RegExp; + message?: string | undefined; +} | { + kind: "trim"; + message?: string | undefined; +} | { + kind: "toLowerCase"; + message?: string | undefined; +} | { + kind: "toUpperCase"; + message?: string | undefined; +} | { + kind: "jwt"; + alg?: string; + message?: string | undefined; +} | { + kind: "datetime"; + offset: boolean; + local: boolean; + precision: number | null; + message?: string | undefined; +} | { + kind: "date"; + message?: string | undefined; +} | { + kind: "time"; + precision: number | null; + message?: string | undefined; +} | { + kind: "duration"; + message?: string | undefined; +} | { + kind: "ip"; + version?: IpVersion | undefined; + message?: string | undefined; +} | { + kind: "cidr"; + version?: IpVersion | undefined; + message?: string | undefined; +} | { + kind: "base64"; + message?: string | undefined; +} | { + kind: "base64url"; + message?: string | undefined; +}; +export interface ZodStringDef extends ZodTypeDef { + checks: ZodStringCheck[]; + typeName: ZodFirstPartyTypeKind.ZodString; + coerce: boolean; +} +export declare function datetimeRegex(args: { + precision?: number | null; + offset?: boolean; + local?: boolean; +}): RegExp; +export declare class ZodString extends ZodType { + _parse(input: ParseInput): ParseReturnType; + protected _regex(regex: RegExp, validation: StringValidation, message?: errorUtil.ErrMessage): ZodEffects; + _addCheck(check: ZodStringCheck): ZodString; + email(message?: errorUtil.ErrMessage): ZodString; + url(message?: errorUtil.ErrMessage): ZodString; + emoji(message?: errorUtil.ErrMessage): ZodString; + uuid(message?: errorUtil.ErrMessage): ZodString; + nanoid(message?: errorUtil.ErrMessage): ZodString; + cuid(message?: errorUtil.ErrMessage): ZodString; + cuid2(message?: errorUtil.ErrMessage): ZodString; + ulid(message?: errorUtil.ErrMessage): ZodString; + base64(message?: errorUtil.ErrMessage): ZodString; + base64url(message?: errorUtil.ErrMessage): ZodString; + jwt(options?: { + alg?: string; + message?: string | undefined; + }): ZodString; + ip(options?: string | { + version?: IpVersion; + message?: string | undefined; + }): ZodString; + cidr(options?: string | { + version?: IpVersion; + message?: string | undefined; + }): ZodString; + datetime(options?: string | { + message?: string | undefined; + precision?: number | null; + offset?: boolean; + local?: boolean; + }): ZodString; + date(message?: string): ZodString; + time(options?: string | { + message?: string | undefined; + precision?: number | null; + }): ZodString; + duration(message?: errorUtil.ErrMessage): ZodString; + regex(regex: RegExp, message?: errorUtil.ErrMessage): ZodString; + includes(value: string, options?: { + message?: string; + position?: number; + }): ZodString; + startsWith(value: string, message?: errorUtil.ErrMessage): ZodString; + endsWith(value: string, message?: errorUtil.ErrMessage): ZodString; + min(minLength: number, message?: errorUtil.ErrMessage): ZodString; + max(maxLength: number, message?: errorUtil.ErrMessage): ZodString; + length(len: number, message?: errorUtil.ErrMessage): ZodString; + /** + * Equivalent to `.min(1)` + */ + nonempty(message?: errorUtil.ErrMessage): ZodString; + trim(): ZodString; + toLowerCase(): ZodString; + toUpperCase(): ZodString; + get isDatetime(): boolean; + get isDate(): boolean; + get isTime(): boolean; + get isDuration(): boolean; + get isEmail(): boolean; + get isURL(): boolean; + get isEmoji(): boolean; + get isUUID(): boolean; + get isNANOID(): boolean; + get isCUID(): boolean; + get isCUID2(): boolean; + get isULID(): boolean; + get isIP(): boolean; + get isCIDR(): boolean; + get isBase64(): boolean; + get isBase64url(): boolean; + get minLength(): number | null; + get maxLength(): number | null; + static create: (params?: RawCreateParams & { + coerce?: true; + }) => ZodString; +} +export type ZodNumberCheck = { + kind: "min"; + value: number; + inclusive: boolean; + message?: string | undefined; +} | { + kind: "max"; + value: number; + inclusive: boolean; + message?: string | undefined; +} | { + kind: "int"; + message?: string | undefined; +} | { + kind: "multipleOf"; + value: number; + message?: string | undefined; +} | { + kind: "finite"; + message?: string | undefined; +}; +export interface ZodNumberDef extends ZodTypeDef { + checks: ZodNumberCheck[]; + typeName: ZodFirstPartyTypeKind.ZodNumber; + coerce: boolean; +} +export declare class ZodNumber extends ZodType { + _parse(input: ParseInput): ParseReturnType; + static create: (params?: RawCreateParams & { + coerce?: boolean; + }) => ZodNumber; + gte(value: number, message?: errorUtil.ErrMessage): ZodNumber; + min: (value: number, message?: errorUtil.ErrMessage) => ZodNumber; + gt(value: number, message?: errorUtil.ErrMessage): ZodNumber; + lte(value: number, message?: errorUtil.ErrMessage): ZodNumber; + max: (value: number, message?: errorUtil.ErrMessage) => ZodNumber; + lt(value: number, message?: errorUtil.ErrMessage): ZodNumber; + protected setLimit(kind: "min" | "max", value: number, inclusive: boolean, message?: string): ZodNumber; + _addCheck(check: ZodNumberCheck): ZodNumber; + int(message?: errorUtil.ErrMessage): ZodNumber; + positive(message?: errorUtil.ErrMessage): ZodNumber; + negative(message?: errorUtil.ErrMessage): ZodNumber; + nonpositive(message?: errorUtil.ErrMessage): ZodNumber; + nonnegative(message?: errorUtil.ErrMessage): ZodNumber; + multipleOf(value: number, message?: errorUtil.ErrMessage): ZodNumber; + step: (value: number, message?: errorUtil.ErrMessage) => ZodNumber; + finite(message?: errorUtil.ErrMessage): ZodNumber; + safe(message?: errorUtil.ErrMessage): ZodNumber; + get minValue(): number | null; + get maxValue(): number | null; + get isInt(): boolean; + get isFinite(): boolean; +} +export type ZodBigIntCheck = { + kind: "min"; + value: bigint; + inclusive: boolean; + message?: string | undefined; +} | { + kind: "max"; + value: bigint; + inclusive: boolean; + message?: string | undefined; +} | { + kind: "multipleOf"; + value: bigint; + message?: string | undefined; +}; +export interface ZodBigIntDef extends ZodTypeDef { + checks: ZodBigIntCheck[]; + typeName: ZodFirstPartyTypeKind.ZodBigInt; + coerce: boolean; +} +export declare class ZodBigInt extends ZodType { + _parse(input: ParseInput): ParseReturnType; + _getInvalidInput(input: ParseInput): INVALID; + static create: (params?: RawCreateParams & { + coerce?: boolean; + }) => ZodBigInt; + gte(value: bigint, message?: errorUtil.ErrMessage): ZodBigInt; + min: (value: bigint, message?: errorUtil.ErrMessage) => ZodBigInt; + gt(value: bigint, message?: errorUtil.ErrMessage): ZodBigInt; + lte(value: bigint, message?: errorUtil.ErrMessage): ZodBigInt; + max: (value: bigint, message?: errorUtil.ErrMessage) => ZodBigInt; + lt(value: bigint, message?: errorUtil.ErrMessage): ZodBigInt; + protected setLimit(kind: "min" | "max", value: bigint, inclusive: boolean, message?: string): ZodBigInt; + _addCheck(check: ZodBigIntCheck): ZodBigInt; + positive(message?: errorUtil.ErrMessage): ZodBigInt; + negative(message?: errorUtil.ErrMessage): ZodBigInt; + nonpositive(message?: errorUtil.ErrMessage): ZodBigInt; + nonnegative(message?: errorUtil.ErrMessage): ZodBigInt; + multipleOf(value: bigint, message?: errorUtil.ErrMessage): ZodBigInt; + get minValue(): bigint | null; + get maxValue(): bigint | null; +} +export interface ZodBooleanDef extends ZodTypeDef { + typeName: ZodFirstPartyTypeKind.ZodBoolean; + coerce: boolean; +} +export declare class ZodBoolean extends ZodType { + _parse(input: ParseInput): ParseReturnType; + static create: (params?: RawCreateParams & { + coerce?: boolean; + }) => ZodBoolean; +} +export type ZodDateCheck = { + kind: "min"; + value: number; + message?: string | undefined; +} | { + kind: "max"; + value: number; + message?: string | undefined; +}; +export interface ZodDateDef extends ZodTypeDef { + checks: ZodDateCheck[]; + coerce: boolean; + typeName: ZodFirstPartyTypeKind.ZodDate; +} +export declare class ZodDate extends ZodType { + _parse(input: ParseInput): ParseReturnType; + _addCheck(check: ZodDateCheck): ZodDate; + min(minDate: Date, message?: errorUtil.ErrMessage): ZodDate; + max(maxDate: Date, message?: errorUtil.ErrMessage): ZodDate; + get minDate(): Date | null; + get maxDate(): Date | null; + static create: (params?: RawCreateParams & { + coerce?: boolean; + }) => ZodDate; +} +export interface ZodSymbolDef extends ZodTypeDef { + typeName: ZodFirstPartyTypeKind.ZodSymbol; +} +export declare class ZodSymbol extends ZodType { + _parse(input: ParseInput): ParseReturnType; + static create: (params?: RawCreateParams) => ZodSymbol; +} +export interface ZodUndefinedDef extends ZodTypeDef { + typeName: ZodFirstPartyTypeKind.ZodUndefined; +} +export declare class ZodUndefined extends ZodType { + _parse(input: ParseInput): ParseReturnType; + params?: RawCreateParams; + static create: (params?: RawCreateParams) => ZodUndefined; +} +export interface ZodNullDef extends ZodTypeDef { + typeName: ZodFirstPartyTypeKind.ZodNull; +} +export declare class ZodNull extends ZodType { + _parse(input: ParseInput): ParseReturnType; + static create: (params?: RawCreateParams) => ZodNull; +} +export interface ZodAnyDef extends ZodTypeDef { + typeName: ZodFirstPartyTypeKind.ZodAny; +} +export declare class ZodAny extends ZodType { + _any: true; + _parse(input: ParseInput): ParseReturnType; + static create: (params?: RawCreateParams) => ZodAny; +} +export interface ZodUnknownDef extends ZodTypeDef { + typeName: ZodFirstPartyTypeKind.ZodUnknown; +} +export declare class ZodUnknown extends ZodType { + _unknown: true; + _parse(input: ParseInput): ParseReturnType; + static create: (params?: RawCreateParams) => ZodUnknown; +} +export interface ZodNeverDef extends ZodTypeDef { + typeName: ZodFirstPartyTypeKind.ZodNever; +} +export declare class ZodNever extends ZodType { + _parse(input: ParseInput): ParseReturnType; + static create: (params?: RawCreateParams) => ZodNever; +} +export interface ZodVoidDef extends ZodTypeDef { + typeName: ZodFirstPartyTypeKind.ZodVoid; +} +export declare class ZodVoid extends ZodType { + _parse(input: ParseInput): ParseReturnType; + static create: (params?: RawCreateParams) => ZodVoid; +} +export interface ZodArrayDef extends ZodTypeDef { + type: T; + typeName: ZodFirstPartyTypeKind.ZodArray; + exactLength: { + value: number; + message?: string | undefined; + } | null; + minLength: { + value: number; + message?: string | undefined; + } | null; + maxLength: { + value: number; + message?: string | undefined; + } | null; +} +export type ArrayCardinality = "many" | "atleastone"; +export type arrayOutputType = Cardinality extends "atleastone" ? [T["_output"], ...T["_output"][]] : T["_output"][]; +export declare class ZodArray extends ZodType, ZodArrayDef, Cardinality extends "atleastone" ? [T["_input"], ...T["_input"][]] : T["_input"][]> { + _parse(input: ParseInput): ParseReturnType; + get element(): T; + min(minLength: number, message?: errorUtil.ErrMessage): this; + max(maxLength: number, message?: errorUtil.ErrMessage): this; + length(len: number, message?: errorUtil.ErrMessage): this; + nonempty(message?: errorUtil.ErrMessage): ZodArray; + static create: (schema: El, params?: RawCreateParams) => ZodArray; +} +export type ZodNonEmptyArray = ZodArray; +export type UnknownKeysParam = "passthrough" | "strict" | "strip"; +export interface ZodObjectDef extends ZodTypeDef { + typeName: ZodFirstPartyTypeKind.ZodObject; + shape: () => T; + catchall: Catchall; + unknownKeys: UnknownKeys; +} +export type mergeTypes = { + [k in keyof A | keyof B]: k extends keyof B ? B[k] : k extends keyof A ? A[k] : never; +}; +export type objectOutputType = objectUtil.flatten>> & CatchallOutput & PassthroughType; +export type baseObjectOutputType = { + [k in keyof Shape]: Shape[k]["_output"]; +}; +export type objectInputType = objectUtil.flatten> & CatchallInput & PassthroughType; +export type baseObjectInputType = objectUtil.addQuestionMarks<{ + [k in keyof Shape]: Shape[k]["_input"]; +}>; +export type CatchallOutput = ZodType extends T ? unknown : { + [k: string]: T["_output"]; +}; +export type CatchallInput = ZodType extends T ? unknown : { + [k: string]: T["_input"]; +}; +export type PassthroughType = T extends "passthrough" ? { + [k: string]: unknown; +} : unknown; +export type deoptional = T extends ZodOptional ? deoptional : T extends ZodNullable ? ZodNullable> : T; +export type SomeZodObject = ZodObject; +export type noUnrecognized = { + [k in keyof Obj]: k extends keyof Shape ? Obj[k] : never; +}; +export declare class ZodObject, Input = objectInputType> extends ZodType, Input> { + private _cached; + _getCached(): { + shape: T; + keys: string[]; + }; + _parse(input: ParseInput): ParseReturnType; + get shape(): T; + strict(message?: errorUtil.ErrMessage): ZodObject; + strip(): ZodObject; + passthrough(): ZodObject; + /** + * @deprecated In most cases, this is no longer needed - unknown properties are now silently stripped. + * If you want to pass through unknown properties, use `.passthrough()` instead. + */ + nonstrict: () => ZodObject; + extend(augmentation: Augmentation): ZodObject, UnknownKeys, Catchall>; + /** + * @deprecated Use `.extend` instead + * */ + augment: (augmentation: Augmentation) => ZodObject, UnknownKeys, Catchall>; + /** + * Prior to zod@1.0.12 there was a bug in the + * inferred type of merged objects. Please + * upgrade if you are experiencing issues. + */ + merge(merging: Incoming): ZodObject, Incoming["_def"]["unknownKeys"], Incoming["_def"]["catchall"]>; + setKey(key: Key, schema: Schema): ZodObject; + catchall(index: Index): ZodObject; + pick>(mask: Mask): ZodObject>, UnknownKeys, Catchall>; + omit>(mask: Mask): ZodObject, UnknownKeys, Catchall>; + /** + * @deprecated + */ + deepPartial(): partialUtil.DeepPartial; + partial(): ZodObject<{ + [k in keyof T]: ZodOptional; + }, UnknownKeys, Catchall>; + partial>(mask: Mask): ZodObject : T[k]; + }>, UnknownKeys, Catchall>; + required(): ZodObject<{ + [k in keyof T]: deoptional; + }, UnknownKeys, Catchall>; + required>(mask: Mask): ZodObject : T[k]; + }>, UnknownKeys, Catchall>; + keyof(): ZodEnum>; + static create: (shape: Shape, params?: RawCreateParams) => ZodObject, objectInputType>; + static strictCreate: (shape: Shape, params?: RawCreateParams) => ZodObject; + static lazycreate: (shape: () => Shape, params?: RawCreateParams) => ZodObject; +} +export type AnyZodObject = ZodObject; +export type ZodUnionOptions = Readonly<[ZodTypeAny, ...ZodTypeAny[]]>; +export interface ZodUnionDef> extends ZodTypeDef { + options: T; + typeName: ZodFirstPartyTypeKind.ZodUnion; +} +export declare class ZodUnion extends ZodType, T[number]["_input"]> { + _parse(input: ParseInput): ParseReturnType; + get options(): T; + static create: >(types: Options, params?: RawCreateParams) => ZodUnion; +} +export type ZodDiscriminatedUnionOption = ZodObject<{ + [key in Discriminator]: ZodTypeAny; +} & ZodRawShape, UnknownKeysParam, ZodTypeAny>; +export interface ZodDiscriminatedUnionDef[] = ZodDiscriminatedUnionOption[]> extends ZodTypeDef { + discriminator: Discriminator; + options: Options; + optionsMap: Map>; + typeName: ZodFirstPartyTypeKind.ZodDiscriminatedUnion; +} +export declare class ZodDiscriminatedUnion[]> extends ZodType, ZodDiscriminatedUnionDef, input> { + _parse(input: ParseInput): ParseReturnType; + get discriminator(): Discriminator; + get options(): Options; + get optionsMap(): Map>; + /** + * The constructor of the discriminated union schema. Its behaviour is very similar to that of the normal z.union() constructor. + * However, it only allows a union of objects, all of which need to share a discriminator property. This property must + * have a different value for each object in the union. + * @param discriminator the name of the discriminator property + * @param types an array of object schemas + * @param params + */ + static create, + ...ZodDiscriminatedUnionOption[] + ]>(discriminator: Discriminator, options: Types, params?: RawCreateParams): ZodDiscriminatedUnion; +} +export interface ZodIntersectionDef extends ZodTypeDef { + left: T; + right: U; + typeName: ZodFirstPartyTypeKind.ZodIntersection; +} +export declare class ZodIntersection extends ZodType, T["_input"] & U["_input"]> { + _parse(input: ParseInput): ParseReturnType; + static create: (left: TSchema, right: USchema, params?: RawCreateParams) => ZodIntersection; +} +export type ZodTupleItems = [ZodTypeAny, ...ZodTypeAny[]]; +export type AssertArray = T extends any[] ? T : never; +export type OutputTypeOfTuple = AssertArray<{ + [k in keyof T]: T[k] extends ZodType ? T[k]["_output"] : never; +}>; +export type OutputTypeOfTupleWithRest = Rest extends ZodTypeAny ? [...OutputTypeOfTuple, ...Rest["_output"][]] : OutputTypeOfTuple; +export type InputTypeOfTuple = AssertArray<{ + [k in keyof T]: T[k] extends ZodType ? T[k]["_input"] : never; +}>; +export type InputTypeOfTupleWithRest = Rest extends ZodTypeAny ? [...InputTypeOfTuple, ...Rest["_input"][]] : InputTypeOfTuple; +export interface ZodTupleDef extends ZodTypeDef { + items: T; + rest: Rest; + typeName: ZodFirstPartyTypeKind.ZodTuple; +} +export type AnyZodTuple = ZodTuple<[ZodTypeAny, ...ZodTypeAny[]] | [], ZodTypeAny | null>; +export declare class ZodTuple extends ZodType, ZodTupleDef, InputTypeOfTupleWithRest> { + _parse(input: ParseInput): ParseReturnType; + get items(): T; + rest(rest: RestSchema): ZodTuple; + static create: (schemas: Items, params?: RawCreateParams) => ZodTuple; +} +export interface ZodRecordDef extends ZodTypeDef { + valueType: Value; + keyType: Key; + typeName: ZodFirstPartyTypeKind.ZodRecord; +} +export type KeySchema = ZodType; +export type RecordType = [string] extends [K] ? Record : [number] extends [K] ? Record : [symbol] extends [K] ? Record : [BRAND] extends [K] ? Record : Partial>; +export declare class ZodRecord extends ZodType, ZodRecordDef, RecordType> { + get keySchema(): Key; + get valueSchema(): Value; + _parse(input: ParseInput): ParseReturnType; + get element(): Value; + static create(valueType: Value, params?: RawCreateParams): ZodRecord; + static create(keySchema: Keys, valueType: Value, params?: RawCreateParams): ZodRecord; +} +export interface ZodMapDef extends ZodTypeDef { + valueType: Value; + keyType: Key; + typeName: ZodFirstPartyTypeKind.ZodMap; +} +export declare class ZodMap extends ZodType, ZodMapDef, Map> { + get keySchema(): Key; + get valueSchema(): Value; + _parse(input: ParseInput): ParseReturnType; + static create: (keyType: KeySchema, valueType: ValueSchema, params?: RawCreateParams) => ZodMap; +} +export interface ZodSetDef extends ZodTypeDef { + valueType: Value; + typeName: ZodFirstPartyTypeKind.ZodSet; + minSize: { + value: number; + message?: string | undefined; + } | null; + maxSize: { + value: number; + message?: string | undefined; + } | null; +} +export declare class ZodSet extends ZodType, ZodSetDef, Set> { + _parse(input: ParseInput): ParseReturnType; + min(minSize: number, message?: errorUtil.ErrMessage): this; + max(maxSize: number, message?: errorUtil.ErrMessage): this; + size(size: number, message?: errorUtil.ErrMessage): this; + nonempty(message?: errorUtil.ErrMessage): ZodSet; + static create: (valueType: ValueSchema, params?: RawCreateParams) => ZodSet; +} +export interface ZodFunctionDef = ZodTuple, Returns extends ZodTypeAny = ZodTypeAny> extends ZodTypeDef { + args: Args; + returns: Returns; + typeName: ZodFirstPartyTypeKind.ZodFunction; +} +export type OuterTypeOfFunction, Returns extends ZodTypeAny> = Args["_input"] extends Array ? (...args: Args["_input"]) => Returns["_output"] : never; +export type InnerTypeOfFunction, Returns extends ZodTypeAny> = Args["_output"] extends Array ? (...args: Args["_output"]) => Returns["_input"] : never; +export declare class ZodFunction, Returns extends ZodTypeAny> extends ZodType, ZodFunctionDef, InnerTypeOfFunction> { + _parse(input: ParseInput): ParseReturnType; + parameters(): Args; + returnType(): Returns; + args[0]>(...items: Items): ZodFunction, Returns>; + returns>(returnType: NewReturnType): ZodFunction; + implement>(func: F): ReturnType extends Returns["_output"] ? (...args: Args["_input"]) => ReturnType : OuterTypeOfFunction; + strictImplement(func: InnerTypeOfFunction): InnerTypeOfFunction; + validate: >(func: F) => ReturnType extends Returns["_output"] ? (...args: Args["_input"]) => ReturnType : OuterTypeOfFunction; + static create(): ZodFunction, ZodUnknown>; + static create>(args: T): ZodFunction; + static create(args: T, returns: U): ZodFunction; + static create, U extends ZodTypeAny = ZodUnknown>(args: T, returns: U, params?: RawCreateParams): ZodFunction; +} +export interface ZodLazyDef extends ZodTypeDef { + getter: () => T; + typeName: ZodFirstPartyTypeKind.ZodLazy; +} +export declare class ZodLazy extends ZodType, ZodLazyDef, input> { + get schema(): T; + _parse(input: ParseInput): ParseReturnType; + static create: (getter: () => Inner, params?: RawCreateParams) => ZodLazy; +} +export interface ZodLiteralDef extends ZodTypeDef { + value: T; + typeName: ZodFirstPartyTypeKind.ZodLiteral; +} +export declare class ZodLiteral extends ZodType, T> { + _parse(input: ParseInput): ParseReturnType; + get value(): T; + static create: (value: Value, params?: RawCreateParams) => ZodLiteral; +} +export type ArrayKeys = keyof any[]; +export type Indices = Exclude; +export type EnumValues = readonly [T, ...T[]]; +export type Values = { + [k in T[number]]: k; +}; +export interface ZodEnumDef extends ZodTypeDef { + values: T; + typeName: ZodFirstPartyTypeKind.ZodEnum; +} +export type Writeable = { + -readonly [P in keyof T]: T[P]; +}; +export type FilterEnum = Values extends [] ? [] : Values extends [infer Head, ...infer Rest] ? Head extends ToExclude ? FilterEnum : [Head, ...FilterEnum] : never; +export type typecast = A extends T ? A : never; +declare function createZodEnum>(values: T, params?: RawCreateParams): ZodEnum>; +declare function createZodEnum(values: T, params?: RawCreateParams): ZodEnum; +export declare class ZodEnum extends ZodType, T[number]> { + _cache: Set | undefined; + _parse(input: ParseInput): ParseReturnType; + get options(): T; + get enum(): Values; + get Values(): Values; + get Enum(): Values; + extract(values: ToExtract, newDef?: RawCreateParams): ZodEnum>; + exclude(values: ToExclude, newDef?: RawCreateParams): ZodEnum>, [string, ...string[]]>>; + static create: typeof createZodEnum; +} +export interface ZodNativeEnumDef extends ZodTypeDef { + values: T; + typeName: ZodFirstPartyTypeKind.ZodNativeEnum; +} +export type EnumLike = { + [k: string]: string | number; + [nu: number]: string; +}; +export declare class ZodNativeEnum extends ZodType, T[keyof T]> { + _cache: Set | undefined; + _parse(input: ParseInput): ParseReturnType; + get enum(): T; + static create: (values: Elements, params?: RawCreateParams) => ZodNativeEnum; +} +export interface ZodPromiseDef extends ZodTypeDef { + type: T; + typeName: ZodFirstPartyTypeKind.ZodPromise; +} +export declare class ZodPromise extends ZodType, ZodPromiseDef, Promise> { + unwrap(): T; + _parse(input: ParseInput): ParseReturnType; + static create: (schema: Inner, params?: RawCreateParams) => ZodPromise; +} +export type Refinement = (arg: T, ctx: RefinementCtx) => any; +export type SuperRefinement = (arg: T, ctx: RefinementCtx) => void | Promise; +export type RefinementEffect = { + type: "refinement"; + refinement: (arg: T, ctx: RefinementCtx) => any; +}; +export type TransformEffect = { + type: "transform"; + transform: (arg: T, ctx: RefinementCtx) => any; +}; +export type PreprocessEffect = { + type: "preprocess"; + transform: (arg: T, ctx: RefinementCtx) => any; +}; +export type Effect = RefinementEffect | TransformEffect | PreprocessEffect; +export interface ZodEffectsDef extends ZodTypeDef { + schema: T; + typeName: ZodFirstPartyTypeKind.ZodEffects; + effect: Effect; +} +export declare class ZodEffects, Input = input> extends ZodType, Input> { + innerType(): T; + sourceType(): T; + _parse(input: ParseInput): ParseReturnType; + static create: (schema: I, effect: Effect, params?: RawCreateParams) => ZodEffects; + static createWithPreprocess: (preprocess: (arg: unknown, ctx: RefinementCtx) => unknown, schema: I, params?: RawCreateParams) => ZodEffects; +} +export { ZodEffects as ZodTransformer }; +export interface ZodOptionalDef extends ZodTypeDef { + innerType: T; + typeName: ZodFirstPartyTypeKind.ZodOptional; +} +export type ZodOptionalType = ZodOptional; +export declare class ZodOptional extends ZodType, T["_input"] | undefined> { + _parse(input: ParseInput): ParseReturnType; + unwrap(): T; + static create: (type: Inner, params?: RawCreateParams) => ZodOptional; +} +export interface ZodNullableDef extends ZodTypeDef { + innerType: T; + typeName: ZodFirstPartyTypeKind.ZodNullable; +} +export type ZodNullableType = ZodNullable; +export declare class ZodNullable extends ZodType, T["_input"] | null> { + _parse(input: ParseInput): ParseReturnType; + unwrap(): T; + static create: (type: Inner, params?: RawCreateParams) => ZodNullable; +} +export interface ZodDefaultDef extends ZodTypeDef { + innerType: T; + defaultValue: () => util.noUndefined; + typeName: ZodFirstPartyTypeKind.ZodDefault; +} +export declare class ZodDefault extends ZodType, ZodDefaultDef, T["_input"] | undefined> { + _parse(input: ParseInput): ParseReturnType; + removeDefault(): T; + static create: (type: Inner, params: RawCreateParams & { + default: Inner["_input"] | (() => util.noUndefined); + }) => ZodDefault; +} +export interface ZodCatchDef extends ZodTypeDef { + innerType: T; + catchValue: (ctx: { + error: ZodError; + input: unknown; + }) => T["_input"]; + typeName: ZodFirstPartyTypeKind.ZodCatch; +} +export declare class ZodCatch extends ZodType, unknown> { + _parse(input: ParseInput): ParseReturnType; + removeCatch(): T; + static create: (type: Inner, params: RawCreateParams & { + catch: Inner["_output"] | (() => Inner["_output"]); + }) => ZodCatch; +} +export interface ZodNaNDef extends ZodTypeDef { + typeName: ZodFirstPartyTypeKind.ZodNaN; +} +export declare class ZodNaN extends ZodType { + _parse(input: ParseInput): ParseReturnType; + static create: (params?: RawCreateParams) => ZodNaN; +} +export interface ZodBrandedDef extends ZodTypeDef { + type: T; + typeName: ZodFirstPartyTypeKind.ZodBranded; +} +export declare const BRAND: unique symbol; +export type BRAND = { + [BRAND]: { + [k in T]: true; + }; +}; +export declare class ZodBranded extends ZodType, ZodBrandedDef, T["_input"]> { + _parse(input: ParseInput): ParseReturnType; + unwrap(): T; +} +export interface ZodPipelineDef extends ZodTypeDef { + in: A; + out: B; + typeName: ZodFirstPartyTypeKind.ZodPipeline; +} +export declare class ZodPipeline extends ZodType, A["_input"]> { + _parse(input: ParseInput): ParseReturnType; + static create(a: ASchema, b: BSchema): ZodPipeline; +} +type BuiltIn = (((...args: any[]) => any) | (new (...args: any[]) => any)) | { + readonly [Symbol.toStringTag]: string; +} | Date | Error | Generator | Promise | RegExp; +type MakeReadonly = T extends Map ? ReadonlyMap : T extends Set ? ReadonlySet : T extends [infer Head, ...infer Tail] ? readonly [Head, ...Tail] : T extends Array ? ReadonlyArray : T extends BuiltIn ? T : Readonly; +export interface ZodReadonlyDef extends ZodTypeDef { + innerType: T; + typeName: ZodFirstPartyTypeKind.ZodReadonly; +} +export declare class ZodReadonly extends ZodType, ZodReadonlyDef, MakeReadonly> { + _parse(input: ParseInput): ParseReturnType; + static create: (type: Inner, params?: RawCreateParams) => ZodReadonly; + unwrap(): T; +} +type CustomParams = CustomErrorParams & { + fatal?: boolean; +}; +export declare function custom(check?: (data: any) => any, _params?: string | CustomParams | ((input: any) => CustomParams), +/** + * @deprecated + * + * Pass `fatal` into the params object instead: + * + * ```ts + * z.string().custom((val) => val.length > 5, { fatal: false }) + * ``` + * + */ +fatal?: boolean): ZodType; +export { ZodType as Schema, ZodType as ZodSchema }; +export declare const late: { + object: (shape: () => Shape, params?: RawCreateParams) => ZodObject; +}; +export declare enum ZodFirstPartyTypeKind { + ZodString = "ZodString", + ZodNumber = "ZodNumber", + ZodNaN = "ZodNaN", + ZodBigInt = "ZodBigInt", + ZodBoolean = "ZodBoolean", + ZodDate = "ZodDate", + ZodSymbol = "ZodSymbol", + ZodUndefined = "ZodUndefined", + ZodNull = "ZodNull", + ZodAny = "ZodAny", + ZodUnknown = "ZodUnknown", + ZodNever = "ZodNever", + ZodVoid = "ZodVoid", + ZodArray = "ZodArray", + ZodObject = "ZodObject", + ZodUnion = "ZodUnion", + ZodDiscriminatedUnion = "ZodDiscriminatedUnion", + ZodIntersection = "ZodIntersection", + ZodTuple = "ZodTuple", + ZodRecord = "ZodRecord", + ZodMap = "ZodMap", + ZodSet = "ZodSet", + ZodFunction = "ZodFunction", + ZodLazy = "ZodLazy", + ZodLiteral = "ZodLiteral", + ZodEnum = "ZodEnum", + ZodEffects = "ZodEffects", + ZodNativeEnum = "ZodNativeEnum", + ZodOptional = "ZodOptional", + ZodNullable = "ZodNullable", + ZodDefault = "ZodDefault", + ZodCatch = "ZodCatch", + ZodPromise = "ZodPromise", + ZodBranded = "ZodBranded", + ZodPipeline = "ZodPipeline", + ZodReadonly = "ZodReadonly" +} +export type ZodFirstPartySchemaTypes = ZodString | ZodNumber | ZodNaN | ZodBigInt | ZodBoolean | ZodDate | ZodUndefined | ZodNull | ZodAny | ZodUnknown | ZodNever | ZodVoid | ZodArray | ZodObject | ZodUnion | ZodDiscriminatedUnion | ZodIntersection | ZodTuple | ZodRecord | ZodMap | ZodSet | ZodFunction | ZodLazy | ZodLiteral | ZodEnum | ZodEffects | ZodNativeEnum | ZodOptional | ZodNullable | ZodDefault | ZodCatch | ZodPromise | ZodBranded | ZodPipeline | ZodReadonly | ZodSymbol; +declare abstract class Class { + constructor(..._: any[]); +} +declare const instanceOfType: (cls: T, params?: CustomParams) => ZodType, ZodTypeDef, InstanceType>; +declare const stringType: (params?: RawCreateParams & { + coerce?: true; +}) => ZodString; +declare const numberType: (params?: RawCreateParams & { + coerce?: boolean; +}) => ZodNumber; +declare const nanType: (params?: RawCreateParams) => ZodNaN; +declare const bigIntType: (params?: RawCreateParams & { + coerce?: boolean; +}) => ZodBigInt; +declare const booleanType: (params?: RawCreateParams & { + coerce?: boolean; +}) => ZodBoolean; +declare const dateType: (params?: RawCreateParams & { + coerce?: boolean; +}) => ZodDate; +declare const symbolType: (params?: RawCreateParams) => ZodSymbol; +declare const undefinedType: (params?: RawCreateParams) => ZodUndefined; +declare const nullType: (params?: RawCreateParams) => ZodNull; +declare const anyType: (params?: RawCreateParams) => ZodAny; +declare const unknownType: (params?: RawCreateParams) => ZodUnknown; +declare const neverType: (params?: RawCreateParams) => ZodNever; +declare const voidType: (params?: RawCreateParams) => ZodVoid; +declare const arrayType: (schema: El, params?: RawCreateParams) => ZodArray; +declare const objectType: (shape: Shape, params?: RawCreateParams) => ZodObject, objectInputType>; +declare const strictObjectType: (shape: Shape, params?: RawCreateParams) => ZodObject; +declare const unionType: >(types: Options, params?: RawCreateParams) => ZodUnion; +declare const discriminatedUnionType: typeof ZodDiscriminatedUnion.create; +declare const intersectionType: (left: TSchema, right: USchema, params?: RawCreateParams) => ZodIntersection; +declare const tupleType: (schemas: Items, params?: RawCreateParams) => ZodTuple; +declare const recordType: typeof ZodRecord.create; +declare const mapType: (keyType: KeySchema, valueType: ValueSchema, params?: RawCreateParams) => ZodMap; +declare const setType: (valueType: ValueSchema, params?: RawCreateParams) => ZodSet; +declare const functionType: typeof ZodFunction.create; +declare const lazyType: (getter: () => Inner, params?: RawCreateParams) => ZodLazy; +declare const literalType: (value: Value, params?: RawCreateParams) => ZodLiteral; +declare const enumType: typeof createZodEnum; +declare const nativeEnumType: (values: Elements, params?: RawCreateParams) => ZodNativeEnum; +declare const promiseType: (schema: Inner, params?: RawCreateParams) => ZodPromise; +declare const effectsType: (schema: I, effect: Effect, params?: RawCreateParams) => ZodEffects; +declare const optionalType: (type: Inner, params?: RawCreateParams) => ZodOptional; +declare const nullableType: (type: Inner, params?: RawCreateParams) => ZodNullable; +declare const preprocessType: (preprocess: (arg: unknown, ctx: RefinementCtx) => unknown, schema: I, params?: RawCreateParams) => ZodEffects; +declare const pipelineType: typeof ZodPipeline.create; +declare const ostring: () => ZodOptional; +declare const onumber: () => ZodOptional; +declare const oboolean: () => ZodOptional; +export declare const coerce: { + string: (typeof ZodString)["create"]; + number: (typeof ZodNumber)["create"]; + boolean: (typeof ZodBoolean)["create"]; + bigint: (typeof ZodBigInt)["create"]; + date: (typeof ZodDate)["create"]; +}; +export { anyType as any, arrayType as array, bigIntType as bigint, booleanType as boolean, dateType as date, discriminatedUnionType as discriminatedUnion, effectsType as effect, enumType as enum, functionType as function, instanceOfType as instanceof, intersectionType as intersection, lazyType as lazy, literalType as literal, mapType as map, nanType as nan, nativeEnumType as nativeEnum, neverType as never, nullType as null, nullableType as nullable, numberType as number, objectType as object, oboolean, onumber, optionalType as optional, ostring, pipelineType as pipeline, preprocessType as preprocess, promiseType as promise, recordType as record, setType as set, strictObjectType as strictObject, stringType as string, symbolType as symbol, effectsType as transformer, tupleType as tuple, undefinedType as undefined, unionType as union, unknownType as unknown, voidType as void, }; +export declare const NEVER: never; diff --git a/tests/fixtures/npm/cases/zod3/node_modules/zod/package.json b/tests/fixtures/npm/cases/zod3/node_modules/zod/package.json new file mode 100644 index 000000000..e02b87614 --- /dev/null +++ b/tests/fixtures/npm/cases/zod3/node_modules/zod/package.json @@ -0,0 +1,120 @@ +{ + "name": "zod", + "version": "3.25.67", + "type": "module", + "author": "Colin McDonnell ", + "description": "TypeScript-first schema declaration and validation library with static type inference", + "files": ["dist", "v3", "v4", "v4-mini"], + "funding": "https://github.com/sponsors/colinhacks", + "homepage": "https://zod.dev", + "keywords": ["typescript", "schema", "validation", "type", "inference"], + "license": "MIT", + "sideEffects": false, + "main": "./dist/cjs/index.js", + "types": "./dist/types/index.d.ts", + "module": "./dist/esm/index.js", + "exports": { + ".": { + "import": { + "@zod/source": "./src/index.ts", + "types": "./dist/types/index.d.ts", + "default": "./dist/esm/index.js" + }, + "require": { + "@zod/source": "./src/index.ts", + "types": "./dist/types/index.d.ts", + "default": "./dist/cjs/index.js" + } + }, + "./v3": { + "import": { + "@zod/source": "./src/index.ts", + "types": "./dist/types/index.d.ts", + "default": "./dist/esm/index.js" + }, + "require": { + "@zod/source": "./src/index.ts", + "types": "./dist/types/index.d.ts", + "default": "./dist/cjs/index.js" + } + }, + "./v4": { + "import": { + "@zod/source": "./src/v4/index.ts", + "types": "./dist/types/v4/index.d.ts", + "default": "./dist/esm/v4/index.js" + }, + "require": { + "@zod/source": "./src/v4/index.ts", + "types": "./dist/types/v4/index.d.ts", + "default": "./dist/cjs/v4/index.js" + } + }, + "./v4-mini": { + "@zod/source": "./src/v4/mini/index.ts", + "types": "./dist/types/v4/mini/index.d.ts", + "import": "./dist/esm/v4/mini/index.js", + "require": "./dist/cjs/v4/mini/index.js" + }, + "./v4/mini": { + "@zod/source": "./src/v4/mini/index.ts", + "types": "./dist/types/v4/mini/index.d.ts", + "import": "./dist/esm/v4/mini/index.js", + "require": "./dist/cjs/v4/mini/index.js" + }, + "./v4/core": { + "@zod/source": "./src/v4/core/index.ts", + "types": "./dist/types/v4/core/index.d.ts", + "import": "./dist/esm/v4/core/index.js", + "require": "./dist/cjs/v4/core/index.js" + }, + "./v4/locales": { + "@zod/source": "./src/v4/locales/index.ts", + "types": "./dist/types/v4/locales/index.d.ts", + "import": "./dist/esm/v4/locales/index.js", + "require": "./dist/cjs/v4/locales/index.js" + }, + "./v4/locales/*": { + "@zod/source": "./src/v4/locales/*", + "types": "./dist/types/v4/locales/*", + "import": "./dist/esm/v4/locales/*", + "require": "./dist/cjs/v4/locales/*" + }, + "./package.json": "./package.json" + }, + "tshy": { + "exports": { + "./package.json": "./package.json", + ".": "./src/index.ts", + "./v3": "./src/v3/index.ts", + "./v4": "./src/v4/index.ts", + "./v4-mini": "./src/v4/mini/index.ts", + "./v4/core": "./src/v4/core/index.ts", + "./v4/locales": "./src/v4/locales/index.ts", + "./v4/locales/*": "./src/v4/locales/*" + }, + "sourceDialects": ["@zod/source"], + "exclude": ["./src/**/*.test.ts"] + }, + "repository": { + "type": "git", + "url": "git+https://github.com/colinhacks/zod.git" + }, + "bugs": { + "url": "https://github.com/colinhacks/zod/issues" + }, + "support": { + "backing": { + "npm-funding": true + } + }, + "scripts": { + "clean": "rm -rf dist", + "build": "tsx ./build.mts", + "postbuild": "pnpm biome check --write .", + "test:watch": "pnpm vitest", + "test": "pnpm vitest run", + "bump:beta": "pnpm version \"v$(pnpm pkg get version | jq -r)-beta.$(date +%Y%m%dT%H%M%S)\"", + "pub:beta": "pnpm bump:beta && pnpm publish --tag next --publish-branch v4 --no-git-checks --dry-run" + } +} diff --git a/tests/fixtures/npm/cases/zod3/package.json b/tests/fixtures/npm/cases/zod3/package.json new file mode 100644 index 000000000..cda6a6168 --- /dev/null +++ b/tests/fixtures/npm/cases/zod3/package.json @@ -0,0 +1 @@ +{ "name": "zod3-case", "private": true, "type": "module" } From 45146d42562834b8b138a8a81c4fff049b0c7744 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Thu, 22 Oct 2026 05:48:46 -0700 Subject: [PATCH 025/197] Keep declaration order and override placeholders in record spreads A fixed-shape spread into an index-signature record copied its fields alphabetically; they now enter in declaration order, as the object's own keys do. A field whose type does not fit the index slot is valid when a later plain property overrides it ({ ...tags, status: String(tags.status) }): a placeholder of the slot's type holds the key's position until the override writes. New corpus program 6916. Co-Authored-By: Claude Opus 5.5 --- .../lowering/containers/indexed-objects.ts | 49 +++++++++++++++---- ...6916-index-record-spread-overrides.ts.json | 6 +++ .../6916-index-record-spread-overrides.ts | 43 ++++++++++++++++ ...6916-index-record-spread-overrides.ts.json | 1 + 4 files changed, 90 insertions(+), 9 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6916-index-record-spread-overrides.ts.json create mode 100644 tests/corpus/6916-index-record-spread-overrides.ts create mode 100644 tests/harness/corpus-costs/6916-index-record-spread-overrides.ts.json diff --git a/packages/compiler/src/frontend/lowering/containers/indexed-objects.ts b/packages/compiler/src/frontend/lowering/containers/indexed-objects.ts index b3faf649e..75d584e4b 100644 --- a/packages/compiler/src/frontend/lowering/containers/indexed-objects.ts +++ b/packages/compiler/src/frontend/lowering/containers/indexed-objects.ts @@ -1388,6 +1388,15 @@ export type IndexMergeContributor = * JS own-key order. Explicit values arrive pre-coerced to the target's * value slot. Null when a spread source is outside that matrix (the * caller keeps its fence). */ +/** A value of an index slot's type that holds a key's place until a later + * write replaces it, or null for slot types without a literal. */ +function indexSlotPlaceholder(slot: IrType, loc: SrcLoc): IrExpr | null { + if (slot.kind === "string") return { kind: "strLit", value: "", type: STRING, loc }; + if (slot.kind === "f64") return { kind: "numLit", value: 0, type: slot, loc }; + if (slot.kind === "bool") return { kind: "boolLit", value: false, type: slot, loc }; + return null; +} + export function lowerIndexMergeHelper( lowerer: Lowerer, toId: string, @@ -1410,9 +1419,9 @@ export function lowerIndexMergeHelper( shapeId: string; wrapTag: number; retag: string | null; - fields: Map | null; + fields: Map | null; }[] = []; - for (const c of contributors) { + for (const [ci, c] of contributors.entries()) { // A conditional spread needs the undefined arm as its "absent" // value — a target slot without one can't carry the empty arm. if ( @@ -1426,15 +1435,26 @@ export function lowerIndexMergeHelper( if (!from || from.tuple) return null; if (!from.indexValue) { if (from.fields.some((f) => f.name.startsWith("%"))) return null; - const fields = new Map(); + const fields = new Map(); for (const ff of from.fields) { const lift = typeEquals(ff.type, tIv) ? ({ how: "copy" } as WidthLift) : tIv.kind === "dyn" && (ff.type.kind === "dyn" || lowerer.dynConvertible(ff.type)) ? "dyn" : lowerer.widthLiftPlan(ff.type, tIv); - if (lift === null) return null; - fields.set(ff.name, lift); + if (lift !== null) { + fields.set(ff.name, lift); + continue; + } + // A field a later plain property overrides (`{ ...tags, status: + // String(tags.status) }`): its value never survives, but its key + // keeps the spread's position, so a placeholder of the slot's + // type holds the place until the override writes. + const overridden = contributors + .slice(ci + 1) + .some((later) => later.kind === "field" && later.name === ff.name); + if (!overridden || indexSlotPlaceholder(tIv, loc) === null) return null; + fields.set(ff.name, "placeholder"); } srcPlans.push({ shapeId: c.shapeId, wrapTag: -1, retag: null, fields }); continue; @@ -1569,7 +1589,16 @@ export function lowerIndexMergeHelper( // own planned lift (absent optionals skip — the unset convention); // no overflow exists to walk. if (plan.fields !== null) { - for (const ff of from.fields) { + // Keys enter in DECLARATION order, as the object's own keys do + // (declaredOrder omits internal '%'-fields, which never copy). + lowerer.keyOrder.observed.add(from.id); + const orderedFields = from.declaredOrder + ? from.declaredOrder.flatMap((n) => { + const f = from.fields.find((x) => x.name === n); + return f ? [f] : []; + }) + : from.fields; + for (const ff of orderedFields) { const lift = plan.fields.get(ff.name)!; const raw: IrExpr = { kind: "recordGet", @@ -1586,9 +1615,11 @@ export function lowerIndexMergeHelper( shapeId: toId, key: { kind: "strLit", value: ff.name, type: STRING, loc }, value: - lift === "dyn" - ? { kind: "dynFrom", value: raw, type: DYN, loc } - : lowerer.applyWidthLift(lift, raw, tIv, loc), + lift === "placeholder" + ? indexSlotPlaceholder(tIv, loc)! + : lift === "dyn" + ? { kind: "dynFrom", value: raw, type: DYN, loc } + : lowerer.applyWidthLift(lift, raw, tIv, loc), overflowOnly: true, loc, }; diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6916-index-record-spread-overrides.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6916-index-record-spread-overrides.ts.json new file mode 100644 index 000000000..db4bc557f --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6916-index-record-spread-overrides.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6916-index-record-spread-overrides.ts" + ], + "diags": [] +} diff --git a/tests/corpus/6916-index-record-spread-overrides.ts b/tests/corpus/6916-index-record-spread-overrides.ts new file mode 100644 index 000000000..094b17f02 --- /dev/null +++ b/tests/corpus/6916-index-record-spread-overrides.ts @@ -0,0 +1,43 @@ +// Spreading a fixed-shape record into an index-signature record: the +// spread's keys enter in declaration order, and a field whose type does +// not fit the index slot is valid when a later property overrides it +// (its key keeps the spread's position, the override supplies the value). +interface FailureTags { + phase: string; + httpStatus: number; + retried: boolean; + region: string; +} +interface Counts { + zeta: number; + alpha: number; + mid: number; +} + +function emit(name: string, tags: { [key: string]: string }): void { + const parts: string[] = []; + for (const key of Object.keys(tags)) parts.push(key + ":" + tags[key]); + console.log(name, parts.join(","), JSON.stringify(tags)); +} + +const tags: FailureTags = { phase: "fetch", httpStatus: 503, retried: true, region: "iad1" }; +for (let attempt = 1; attempt <= 2; attempt++) { + emit("attempt_failed", { + ...tags, + attempt: String(attempt), + httpStatus: String(tags.httpStatus), + retried: tags.retried ? "yes" : "no", + }); +} + +const base: Record = { team: "core", owner: "infra" }; +emit("queue", { ...base, owner: "builds", shard: "7" }); +emit("size", { ...base }); + +const counts: Counts = { zeta: 3, alpha: 1, mid: 2 }; +const scores: Record = { ...counts, alpha: 10 }; +console.log(Object.keys(scores).join(","), JSON.stringify(scores)); +const labels: Record = { first: "a", ...{ zeta: "z", alpha: "a" }, last: "l" }; +console.log(Object.entries(labels).map(([k, v]) => k + "=" + v).join(" ")); +const flags: { [key: string]: boolean } = { ...counts, mid: true, zeta: false, alpha: true }; +console.log(JSON.stringify(flags)); diff --git a/tests/harness/corpus-costs/6916-index-record-spread-overrides.ts.json b/tests/harness/corpus-costs/6916-index-record-spread-overrides.ts.json new file mode 100644 index 000000000..fae2adeb4 --- /dev/null +++ b/tests/harness/corpus-costs/6916-index-record-spread-overrides.ts.json @@ -0,0 +1 @@ +{ "plain": 1500, "san": 9000 } From 12563f720e0cee6474763a371ac360b1f58331b2 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 07:58:53 -0500 Subject: [PATCH 026/197] Compile conditional spreads of several properties and --- .../ambient/scriptc-node-fallback.d.ts | 24 +- .../builtins/child-process-methods.ts | 65 +++-- .../frontend/lowering/builtins/promises.ts | 97 ++++++- .../src/frontend/lowering/builtins/url.ts | 25 +- .../lowering/coercions/structural-plans.ts | 58 +++- .../frontend/lowering/coercions/structural.ts | 74 ++++- .../lowering/containers/array-callbacks.ts | 271 ++++++++++++++++++ .../lowering/expressions/array-literals.ts | 35 +++ .../lowering/expressions/object-literals.ts | 180 ++++++++---- .../src/frontend/lowering/lower-exprs.ts | 3 + .../src/frontend/lowering/lower-stmts.ts | 169 ++++++++++- .../compiler/src/frontend/lowering/lowerer.ts | 26 +- .../src/frontend/lowering/width-lift.ts | 11 + packages/compiler/src/frontend/type-mapper.ts | 21 ++ .../7020-conditional-spread-fields.ts.json | 6 + .../7021-promise-all-uniform-tuple.ts.json | 6 + .../7022-promise-all-dynamic-arrays.ts.json | 6 + .../corpus/7023-union-object-patterns.ts.json | 6 + ...024-async-return-fresh-collections.ts.json | 6 + .../7025-child-listener-chaining.ts.json | 6 + .../corpus/7026-url-object-patterns.ts.json | 6 + ...027-search-params-conditional-init.ts.json | 6 + .../corpus/7028-non-empty-rest-tuples.ts.json | 6 + .../corpus/7029-flatmap-union-results.ts.json | 6 + ...3-any-fields-into-declared-records.ts.json | 6 + .../7034-optional-field-union-routes.ts.json | 6 + ...engine-1123-any-conditional-spread.ts.json | 6 - .../corpus/7020-conditional-spread-fields.ts | 137 +++++++++ .../corpus/7021-promise-all-uniform-tuple.ts | 46 +++ .../corpus/7022-promise-all-dynamic-arrays.ts | 41 +++ tests/corpus/7023-union-object-patterns.ts | 63 ++++ .../7024-async-return-fresh-collections.ts | 48 ++++ tests/corpus/7025-child-listener-chaining.ts | 42 +++ tests/corpus/7026-url-object-patterns.ts | 27 ++ .../7027-search-params-conditional-init.ts | 21 ++ tests/corpus/7028-non-empty-rest-tuples.ts | 38 +++ tests/corpus/7029-flatmap-union-results.ts | 31 ++ .../7033-any-fields-into-declared-records.ts | 28 ++ .../7034-optional-field-union-routes.ts | 26 ++ tests/diagnostics/destructuring-fences.ts | 6 +- ...ired-engine-1123-any-conditional-spread.ts | 49 ---- .../__snapshots__/child-process.ts.txt | 2 +- .../__snapshots__/destructuring-fences.ts.txt | 16 +- ...-engine-1123-any-conditional-spread.ts.txt | 6 - .../__snapshots__/spread-fences.ts.txt | 2 +- .../7020-conditional-spread-fields.ts.json | 1 + .../7021-promise-all-uniform-tuple.ts.json | 1 + .../7022-promise-all-dynamic-arrays.ts.json | 1 + .../7023-union-object-patterns.ts.json | 1 + ...024-async-return-fresh-collections.ts.json | 1 + .../7025-child-listener-chaining.ts.json | 1 + .../7026-url-object-patterns.ts.json | 1 + ...027-search-params-conditional-init.ts.json | 1 + .../7028-non-empty-rest-tuples.ts.json | 1 + .../7029-flatmap-union-results.ts.json | 1 + ...3-any-fields-into-declared-records.ts.json | 1 + .../7034-optional-field-union-routes.ts.json | 1 + 57 files changed, 1609 insertions(+), 169 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7020-conditional-spread-fields.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7021-promise-all-uniform-tuple.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7022-promise-all-dynamic-arrays.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7023-union-object-patterns.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7024-async-return-fresh-collections.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7025-child-listener-chaining.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7026-url-object-patterns.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7027-search-params-conditional-init.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7028-non-empty-rest-tuples.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7029-flatmap-union-results.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7033-any-fields-into-declared-records.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7034-optional-field-union-routes.ts.json delete mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/retired-engine-1123-any-conditional-spread.ts.json create mode 100644 tests/corpus/7020-conditional-spread-fields.ts create mode 100644 tests/corpus/7021-promise-all-uniform-tuple.ts create mode 100644 tests/corpus/7022-promise-all-dynamic-arrays.ts create mode 100644 tests/corpus/7023-union-object-patterns.ts create mode 100644 tests/corpus/7024-async-return-fresh-collections.ts create mode 100644 tests/corpus/7025-child-listener-chaining.ts create mode 100644 tests/corpus/7026-url-object-patterns.ts create mode 100644 tests/corpus/7027-search-params-conditional-init.ts create mode 100644 tests/corpus/7028-non-empty-rest-tuples.ts create mode 100644 tests/corpus/7029-flatmap-union-results.ts create mode 100644 tests/corpus/7033-any-fields-into-declared-records.ts create mode 100644 tests/corpus/7034-optional-field-union-routes.ts delete mode 100644 tests/diagnostics/retired-engine-1123-any-conditional-spread.ts delete mode 100644 tests/harness/__snapshots__/retired-engine-1123-any-conditional-spread.ts.txt create mode 100644 tests/harness/corpus-costs/7020-conditional-spread-fields.ts.json create mode 100644 tests/harness/corpus-costs/7021-promise-all-uniform-tuple.ts.json create mode 100644 tests/harness/corpus-costs/7022-promise-all-dynamic-arrays.ts.json create mode 100644 tests/harness/corpus-costs/7023-union-object-patterns.ts.json create mode 100644 tests/harness/corpus-costs/7024-async-return-fresh-collections.ts.json create mode 100644 tests/harness/corpus-costs/7025-child-listener-chaining.ts.json create mode 100644 tests/harness/corpus-costs/7026-url-object-patterns.ts.json create mode 100644 tests/harness/corpus-costs/7027-search-params-conditional-init.ts.json create mode 100644 tests/harness/corpus-costs/7028-non-empty-rest-tuples.ts.json create mode 100644 tests/harness/corpus-costs/7029-flatmap-union-results.ts.json create mode 100644 tests/harness/corpus-costs/7033-any-fields-into-declared-records.ts.json create mode 100644 tests/harness/corpus-costs/7034-optional-field-union-routes.ts.json diff --git a/packages/compiler/ambient/scriptc-node-fallback.d.ts b/packages/compiler/ambient/scriptc-node-fallback.d.ts index 2a3f11be9..561dcc124 100644 --- a/packages/compiler/ambient/scriptc-node-fallback.d.ts +++ b/packages/compiler/ambient/scriptc-node-fallback.d.ts @@ -3554,20 +3554,20 @@ declare module "child_process" { export interface ChildProcess extends Disposable { ref(): void; - on(event: "spawn", listener: () => void): void; - once(event: "spawn", listener: () => void): void; + on(event: "spawn", listener: () => void): this; + once(event: "spawn", listener: () => void): this; /* The exit listener may also take Node's second parameter — the * terminating signal's name, null for a normal exit. */ - on(event: "exit", listener: (code: number | null, signal: string | null) => void): void; - on(event: "close", listener: (code: number | null, signal: string | null) => void): void; - on(event: "error", listener: (err: Error) => void): void; - once(event: "exit", listener: (code: number | null, signal: string | null) => void): void; - once(event: "close", listener: (code: number | null, signal: string | null) => void): void; - once(event: "error", listener: (err: Error) => void): void; - on(event: "message", listener: (message: any) => void): void; - on(event: "disconnect", listener: () => void): void; - once(event: "message", listener: (message: any) => void): void; - once(event: "disconnect", listener: () => void): void; + on(event: "exit", listener: (code: number | null, signal: string | null) => void): this; + on(event: "close", listener: (code: number | null, signal: string | null) => void): this; + on(event: "error", listener: (err: Error) => void): this; + once(event: "exit", listener: (code: number | null, signal: string | null) => void): this; + once(event: "close", listener: (code: number | null, signal: string | null) => void): this; + once(event: "error", listener: (err: Error) => void): this; + on(event: "message", listener: (message: any) => void): this; + on(event: "disconnect", listener: () => void): this; + once(event: "message", listener: (message: any) => void): this; + once(event: "disconnect", listener: () => void): this; /* The lifecycle members, Node's exact shapes: pid is undefined exactly * when the spawn failed; exitCode is null while running, the code * after a normal exit, null for a signal death, and -errno once a diff --git a/packages/compiler/src/frontend/lowering/builtins/child-process-methods.ts b/packages/compiler/src/frontend/lowering/builtins/child-process-methods.ts index af7270ea9..2e0f305a0 100644 --- a/packages/compiler/src/frontend/lowering/builtins/child-process-methods.ts +++ b/packages/compiler/src/frontend/lowering/builtins/child-process-methods.ts @@ -1,4 +1,4 @@ -import { boolLit, strLit } from "../../../ir/build.js"; +import { boolLit, strLit, varRef } from "../../../ir/build.js"; import * as ts from "../../ts7/adapter.js"; import { requestGeneralStreamSurface } from "../general-streams.js"; import { type Lowerer } from "../lowerer.js"; @@ -93,32 +93,47 @@ export function lowerChildMethodCall( '"exit", "close", "error", "message", and "disconnect" are the supported child events (as literals)', ); } - if (!ts.isExpressionStatement(call.parent)) { - lowerer.unsupported( - "SC1090", - call, - `chaining child.${name}(...) (the result is void here — register each listener as its own statement)`, - ); - } - const receiver = lowerer.lowerExpr(access.expression); + // `child.on(...)` returns the child: a chained call or a stored result + // reads the receiver, which evaluates once, before the listener. + let receiver = lowerer.lowerExpr(access.expression); + const chained = ts.isExpressionStatement(call.parent) + ? null + : { + local: lowerer.declareHiddenLocal("%childListenerTarget", receiver.type), + init: receiver, + }; + if (chained) receiver = varRef(chained.local.id, chained.local.type, loc); + const registered = (registration: IrExpr): IrExpr => + chained === null + ? registration + : { + kind: "seqExpr", + stmts: [ + { kind: "varDecl", localId: chained.local.id, init: chained.init, loc }, + { kind: "exprStmt", expr: registration, loc }, + ], + result: varRef(chained.local.id, chained.local.type, loc), + type: chained.local.type, + loc, + }; const once: IrExpr = { kind: "boolLit", value: name === "once", type: BOOL, loc }; if (event === "message") { - return { + return registered({ kind: "libCall", fn: "child.onMessage", args: [receiver, ipcMessageListener(lowerer, call.arguments.nodes[1]!), once], type: VOID, loc, - }; + }); } if (event === "disconnect") { - return { + return registered({ kind: "libCall", fn: "child.onDisconnect", args: [receiver, ipcDisconnectListener(lowerer, call.arguments.nodes[1]!), once], type: VOID, loc, - }; + }); } const listenerNode = stripTypeCasts(call.arguments.nodes[1]!); if ( @@ -163,7 +178,7 @@ export function lowerChildMethodCall( call.arguments.nodes[1]!, ); } - return { + return registered({ kind: "libCall", fn: "child.onDyn", args: [ @@ -173,7 +188,7 @@ export function lowerChildMethodCall( ], type: VOID, loc, - }; + }); } if (event === "spawn") { if (cb.type.kind !== "func" || cb.type.params.length !== 0 || cb.type.ret.kind !== "void") { @@ -182,7 +197,13 @@ export function lowerChildMethodCall( call.arguments.nodes[1]!, ); } - return { kind: "libCall", fn: "child.onSpawn", args: [receiver, cb], type: VOID, loc }; + return registered({ + kind: "libCall", + fn: "child.onSpawn", + args: [receiver, cb], + type: VOID, + loc, + }); } if (cb.type.kind !== "func" || cb.type.params.length > (event === "error" ? 1 : 2)) { lowerer.unsupported( @@ -243,13 +264,13 @@ export function lowerChildMethodCall( `${event} listeners whose signal parameter is not 'Signals | null' (got '${lowerer.fmt(sigParam!)}')`, ); } - return { + return registered({ kind: "libCall", fn: event === "close" ? "child.onClose" : "child.onExit", args: [receiver, cb], type: VOID, loc, - }; + }); } if (param !== undefined && !(param.kind === "object" && param.className === "%Error")) { lowerer.unsupported( @@ -258,7 +279,13 @@ export function lowerChildMethodCall( `error listeners whose parameter is not 'Error' (got '${lowerer.fmt(param)}')`, ); } - return { kind: "libCall", fn: "child.onError", args: [receiver, cb], type: VOID, loc }; + return registered({ + kind: "libCall", + fn: "child.onError", + args: [receiver, cb], + type: VOID, + loc, + }); } // child.kill(signal?) — Node's semantics exactly: the name resolves // through Node's signal table (unknown names throw the ERR_UNKNOWN_SIGNAL diff --git a/packages/compiler/src/frontend/lowering/builtins/promises.ts b/packages/compiler/src/frontend/lowering/builtins/promises.ts index 75e527aa5..129ac73a3 100644 --- a/packages/compiler/src/frontend/lowering/builtins/promises.ts +++ b/packages/compiler/src/frontend/lowering/builtins/promises.ts @@ -9,6 +9,7 @@ import { isJsSourceFile, locOf } from "../../program.js"; import { BOOL, DYN, + F64, type IrExpr, type IrFunction, type IrStmt, @@ -376,8 +377,14 @@ export function lowerPromiseStaticCall( ); } const entries = lowerer.lowerExpr(argNode); - if (isJsSourceFile(ts.getSourceFileOfNode(call)) && entries.type.kind === "dyn") - return checkedPromiseAll(lowerer, entries, loc); + if (entries.type.kind === "dyn") return checkedPromiseAll(lowerer, entries, loc); + // An 'any'-typed argument that holds a typed value (`const xs: any = + // []`): the combinator reads it as the runtime value it is. + if ( + (lowerer.typeOf(argNode).flags & ts.TypeFlags.Any) !== 0 && + lowerer.dynConvertible(entries.type) + ) + return checkedPromiseAll(lowerer, lowerer.coerceToExpected(entries, DYN), loc); // loadPlugin(p)))` — the loadPlugins shape, where the checker // spells `any[]`): the ENGINE's own Promise.all runs over the @@ -845,3 +852,89 @@ function lowerTypedSettledCombinator( } return { kind: "intrinsic", name: "promise.allSettled", args: [entries], type: resultT, loc }; } + +/** `const pair = await Promise.all([p, q])` where every entry is the same + * Promise and the binding's type is the checker's tuple `[T, T]`: the + * uniform entries settle through one runtime combinator into a T[] (the + * same combinator and turn as the array form), and the binding receives + * the tuple of its elements. Null for every other await. */ +export function lowerAwaitedUniformPromiseAll( + lowerer: Lowerer, + expr: ts.AwaitExpression, +): IrExpr | null { + let operand: ts.Expression = expr.expression; + while (ts.isParenthesizedExpression(operand)) operand = operand.expression; + if ( + !ts.isCallExpression(operand) || + operand.questionDotToken || + !ts.isPropertyAccessExpression(operand.expression) || + operand.arguments.nodes.length !== 1 || + isJsSourceFile(ts.getSourceFileOfNode(expr)) + ) + return null; + const argNode = operand.arguments.nodes[0]!; + if ( + !ts.isArrayLiteralExpression(argNode) || + argNode.elements.nodes.length === 0 || + argNode.elements.nodes.some(ts.isSpreadElement) + ) + return null; + let user: ts.Node = expr; + while (ts.isParenthesizedExpression(user.parent)) user = user.parent; + const decl = user.parent; + if (!ts.isVariableDeclaration(decl) || decl.initializer !== user || !ts.isIdentifier(decl.name)) + return null; + if (lowerer.stdlibGlobalMember(operand.expression, "Promise") !== "all") return null; + const tuple = lowerer.mapTypeOf(lowerer.typeOf(decl.name)); + if (tuple?.kind !== "record") return null; + const shape = lowerer.shapes.get(tuple.shapeId); + if (!shape?.tuple || shape.fields.length !== argNode.elements.nodes.length) return null; + const elems = argNode.elements.nodes.map((el) => lowerer.lowerExpr(el)); + const first = elems[0]!.type; + if ( + first.kind !== "promise" || + first.inner.kind === "void" || + !elems.every((e) => typeEquals(e.type, first)) || + !shape.fields.every((f) => typeEquals(f.type, first.inner)) + ) + return null; + const loc = locOf(expr); + const values = arrayOf(first.inner); + const local = lowerer.declareHiddenLocal("%allValues", values); + const all: IrExpr = { + kind: "intrinsic", + name: "promise.all", + args: [{ kind: "arrayLit", elems, type: arrayOf(first), loc }], + type: { kind: "promise", inner: values }, + loc, + }; + return { + kind: "seqExpr", + stmts: [ + { + kind: "varDecl", + localId: local.id, + init: { kind: "awaitExpr", value: all, type: values, loc }, + loc, + }, + ], + // Shape fields are sorted by name; a tuple field's name is its position. + result: { + kind: "recordLit", + fields: shape.fields.map((f) => ({ + name: f.name, + value: { + kind: "arrayGet", + arr: varRef(local.id, values, loc), + index: { kind: "numLit", value: Number(f.name), type: F64, loc }, + type: f.type, + loc, + }, + })), + type: tuple, + loc, + }, + type: tuple, + loc, + }; +} diff --git a/packages/compiler/src/frontend/lowering/builtins/url.ts b/packages/compiler/src/frontend/lowering/builtins/url.ts index 2a77fa85c..8619b4997 100644 --- a/packages/compiler/src/frontend/lowering/builtins/url.ts +++ b/packages/compiler/src/frontend/lowering/builtins/url.ts @@ -195,7 +195,30 @@ export function lowerSearchParamsNew( ); } const arg = args[0]; - if (arg === undefined || (ts.isIdentifier(arg) && arg.text === "undefined")) { + if (arg === undefined) { + return { kind: "libCall", fn: "sp.new", args: [], type: SEARCH_PARAMS_T, loc }; + } + return lowerSearchParamsInit(lowerer, arg, loc); +} + +/** The URLSearchParams built from one init expression. A conditional + * init (`new URLSearchParams(c ? { a: "1" } : {})`) evaluates its + * condition first and builds from the arm it selects, so each arm may be + * any supported init shape. */ +function lowerSearchParamsInit(lowerer: Lowerer, arg: ts.Expression, loc: SrcLoc): IrExpr { + let inner = arg; + while (ts.isParenthesizedExpression(inner)) inner = inner.expression; + if (ts.isConditionalExpression(inner)) { + return { + kind: "ternary", + cond: lowerer.lowerCondition(inner.condition), + then: lowerSearchParamsInit(lowerer, inner.whenTrue, loc), + else_: lowerSearchParamsInit(lowerer, inner.whenFalse, loc), + type: SEARCH_PARAMS_T, + loc, + }; + } + if (ts.isIdentifier(arg) && arg.text === "undefined") { return { kind: "libCall", fn: "sp.new", args: [], type: SEARCH_PARAMS_T, loc }; } // The object-literal init: `{ a: "1", b: "2" }` appends pairs in diff --git a/packages/compiler/src/frontend/lowering/coercions/structural-plans.ts b/packages/compiler/src/frontend/lowering/coercions/structural-plans.ts index 53b0748b4..f42663a13 100644 --- a/packages/compiler/src/frontend/lowering/coercions/structural-plans.ts +++ b/packages/compiler/src/frontend/lowering/coercions/structural-plans.ts @@ -1,7 +1,7 @@ import { planRecordUnionLift, planRecordUnionWrap } from "../union-retag.js"; import type { WidthLift } from "../width-lift.js"; import type { IrType } from "../../../ir/ir.js"; -import { isUnitType, typeEquals, UNDEFINED_T } from "../../../ir/ir.js"; +import { canDynCheckTo, isUnitType, typeEquals, UNDEFINED_T } from "../../../ir/ir.js"; import { type ClassInfo, findMethodOn, findGenericMethodOn } from "../lower-classes.js"; import type { Lowerer } from "../lowerer.js"; @@ -21,6 +21,22 @@ export function widthLiftPlan(lowerer: Lowerer, src: IrType, dst: IrType): Width if (dst.kind === "dyn" && src.kind !== "dyn" && lowerer.dynConvertible(src)) { return { how: "dynIn" }; } + // An 'unknown' (dyn) SOURCE field into a typed slot — a field TypeScript + // types `any` (a JSON.parse read), which the checker lets into any + // declared field type: the checked extraction, the same dynCheck a + // typed binding initialized from that value applies. + if ( + src.kind === "dyn" && + dst.kind !== "dyn" && + dst.kind !== "union" && + canDynCheckTo( + dst, + (id) => lowerer.shapes.get(id), + (id) => lowerer.unions.get(id), + ) + ) { + return { how: "dynOut" }; + } if (dst.kind === "union") { if (src.kind === "union") { if (lowerer.unionRetagMappable(src.unionId, dst.unionId)) return { how: "retag" }; @@ -108,6 +124,10 @@ export function widthLiftPlan(lowerer: Lowerer, src: IrType, dst: IrType): Width return { how: "discriminantRoute" }; } } + if (selected.length === 2 && src.kind === "record") { + const route = presenceRoute(lowerer, src.shapeId, selected[0]!, selected[1]!); + if (route) return route; + } if (selected.length !== 1) return null; return { how: "liftWrap", tag: selected[0]!.tag, arm: selected[0]!.arm }; } @@ -202,6 +222,42 @@ export function widthLiftPlan(lowerer: Lowerer, src: IrType, dst: IrType): Width return null; } +/** Two record arms a source record fits, where one arm declares a field + * the other lacks and the source types that field `T | undefined` + * (`{ count, errors: errs.length ? errs : undefined }` into `{ count; + * errors: string[] } | { count }`): the field's value picks the arm at + * run time. Null for any other pair. */ +function presenceRoute( + lowerer: Lowerer, + sourceId: string, + a: { tag: number; arm: IrType }, + b: { tag: number; arm: IrType }, +): WidthLift | null { + const source = lowerer.shapes.get(sourceId); + if (!source || source.tuple || source.indexValue || a.arm.kind !== "record") return null; + if (b.arm.kind !== "record") return null; + const shapeA = lowerer.shapes.get(a.arm.shapeId); + const shapeB = lowerer.shapes.get(b.arm.shapeId); + if (!shapeA || !shapeB) return null; + const optional = (t: IrType): boolean => + t.kind === "union" && lowerer.armTag(t.unionId, UNDEFINED_T) >= 0; + for (const [present, absent, presentShape, absentShape] of [ + [a, b, shapeA, shapeB], + [b, a, shapeB, shapeA], + ] as const) { + const extra = presentShape.fields.filter( + (f) => !absentShape.fields.some((g) => g.name === f.name), + ); + if (extra.length !== 1 || optional(extra[0]!.type)) continue; + if (absentShape.fields.some((g) => !presentShape.fields.some((f) => f.name === g.name))) + continue; + const field = source.fields.find((f) => f.name === extra[0]!.name); + if (!field || !optional(field.type)) continue; + return { how: "presenceRoute", field: field.name, present, absent }; + } + return null; +} + /** True for the unit-only element types (`(null | undefined)[]`, the * `readonly []` mapping): a union whose every arm is a unit. */ export function unitOnlyElem(lowerer: Lowerer, t: IrType): boolean { diff --git a/packages/compiler/src/frontend/lowering/coercions/structural.ts b/packages/compiler/src/frontend/lowering/coercions/structural.ts index 5b5e283b3..c3b4ad906 100644 --- a/packages/compiler/src/frontend/lowering/coercions/structural.ts +++ b/packages/compiler/src/frontend/lowering/coercions/structural.ts @@ -6,7 +6,7 @@ import { buildArrayConversion } from "./builders.js"; import type { WidthLift } from "../width-lift.js"; import { InternalCompilerError } from "../../../errors.js"; import type { IrExpr, IrFunction, IrRecordShape, IrStmt, IrType, SrcLoc } from "../../../ir/ir.js"; -import { BOOL, DYN, F64, STRING } from "../../../ir/ir.js"; +import { BOOL, DYN, F64, STRING, UNDEFINED_T } from "../../../ir/ir.js"; import { typeKey } from "../../type-mapper.js"; import { findStaticOn, findGenericStaticOn } from "../lower-classes.js"; import { lowerRecordOvfCaptureHelper } from "../containers/indexed-objects.js"; @@ -155,6 +155,76 @@ export function applyWidthLift( loc, }; } + case "presenceRoute": { + if (dst.kind !== "union" || value.type.kind !== "record") + throw new InternalCompilerError("lowerer bug: presenceRoute lift shape"); + const sourceType = value.type; + const fieldType = lowerer.shapes + .get(sourceType.shapeId) + ?.fields.find((f) => f.name === lift.field)?.type; + if (fieldType?.kind !== "union") + throw new InternalCompilerError("lowerer bug: presenceRoute field shape"); + // One helper per pair: the conversion reads the source twice. + const key = `presence:${typeKey(sourceType)}:${dst.unionId}:${lift.field}`; + let helper = lowerer.valueHelpers.get(key); + if (!helper) { + helper = `%union.presence.${lowerer.valueHelpers.size}`; + lowerer.valueHelpers.set(key, helper); + const source = (): IrExpr => varRef("v.0", sourceType, loc); + const arm = (route: { tag: number; arm: IrType }): IrExpr => { + const inner = lowerer.widthLiftPlan(sourceType, route.arm); + if (!inner) + throw new InternalCompilerError( + "lowerer bug: planned presenceRoute arm stopped lifting", + ); + return { + kind: "unionWrap", + unionId: dst.unionId, + tag: route.tag, + value: lowerer.applyWidthLift(inner, source(), route.arm, loc), + type: dst, + loc, + }; + }; + lowerer.liftedFns.push({ + name: helper, + params: [{ localId: "v.0", name: "v", type: sourceType }], + returnType: dst, + locals: [{ id: "v.0", name: "v", type: sourceType, mutable: false }], + body: [ + { + kind: "return", + value: { + kind: "ternary", + cond: { + kind: "unionIsTag", + unionId: fieldType.unionId, + tag: lowerer.armTag(fieldType.unionId, UNDEFINED_T), + value: { + kind: "recordGet", + obj: source(), + shapeId: sourceType.shapeId, + field: lift.field, + type: fieldType, + loc, + }, + negated: false, + type: BOOL, + loc, + }, + then: arm(lift.absent), + else_: arm(lift.present), + type: dst, + loc, + }, + loc, + }, + ], + loc, + }); + } + return { kind: "call", callee: helper, args: [value], type: dst, loc }; + } case "width": { if (dst.kind !== "record" || value.type.kind !== "record") throw new InternalCompilerError("lowerer bug: width lift shape"); @@ -220,6 +290,8 @@ export function applyWidthLift( throw new InternalCompilerError("lowerer bug: dynIn lift against a non-dyn slot"); return { kind: "dynFrom", value, type: DYN, loc }; } + case "dynOut": + return { kind: "dynCheck", value, type: dst, loc }; case "upcast": { if (dst.kind !== "object" || value.type.kind !== "object") throw new InternalCompilerError("lowerer bug: upcast lift shape"); diff --git a/packages/compiler/src/frontend/lowering/containers/array-callbacks.ts b/packages/compiler/src/frontend/lowering/containers/array-callbacks.ts index 612c47ea0..3de1a4e6b 100644 --- a/packages/compiler/src/frontend/lowering/containers/array-callbacks.ts +++ b/packages/compiler/src/frontend/lowering/containers/array-callbacks.ts @@ -1501,6 +1501,19 @@ export function lowerArrayFlatMapCall( } if (fnRet.kind === "union") { const def = lowerer.unions.get(fnRet.unionId); + const plan = def?.arms.some((a) => a.kind === "array") + ? flatMapUnionPlan(lowerer, call, argNode, fnRet) + : null; + if (plan) { + const helper = flatMapUnionHelper(lowerer, elem, fnRet, arity, plan, loc, present); + return { + kind: "call", + callee: helper, + args: [receiver, fnArg], + type: arrayOf(plan.outElem), + loc, + }; + } if (def?.arms.some((a) => a.kind === "array")) { lowerer.unsupported( "SC1090", @@ -1566,6 +1579,264 @@ export function lowerArrayFlatMapCall( }; } +/** How each arm of a flatMap callback's union result contributes to the + * output: an array arm appends its elements, an array arm the checker + * types `never[]` (an empty literal such as `s ? s : []`) appends nothing, + * and any other arm appends itself. */ +interface FlatMapUnionPlan { + outElem: IrType; + arms: ("spread" | "empty" | "value")[]; +} + +/** The plan for a callback returning `T | U[]`: every non-array arm and + * every array arm's element must convert into the result's element type + * without a check. Null when an arm does not. */ +function flatMapUnionPlan( + lowerer: Lowerer, + call: ts.CallExpression, + argNode: ts.Expression, + fnRet: IrType & { kind: "union" }, +): FlatMapUnionPlan | null { + const out = lowerer.mapTypeOf(lowerer.typeOf(call)); + if (out?.kind !== "array") return null; + const outElem = out.elem; + const arms = lowerer.unions.get(fnRet.unionId)?.arms; + if (!arms) return null; + // The `never[]` constituents of the callback's checked result. + const emptyArms: IrType[] = []; + const signature = lowerer.checker.getCallSignatures(lowerer.typeOf(argNode))[0]; + if (signature) { + const ret = lowerer.checker.getReturnTypeOfSignature(signature); + for (const part of ret.isUnionType() ? ts.constituentTypes(ret) : [ret]) { + if (!lowerer.checker.isArrayType(part)) continue; + const [element] = lowerer.checker.getTypeArguments(part as ts.TypeReference); + if (element === undefined || (element.flags & ts.TypeFlags.Never) === 0) continue; + const mapped = lowerer.mapTypeOf(part); + if (mapped) emptyArms.push(mapped); + } + } + const converts = (t: IrType): boolean => { + if (typeEquals(t, outElem)) return true; + if (outElem.kind !== "union") return false; + return lowerer.armTag(outElem.unionId, t) >= 0; + }; + const plan: FlatMapUnionPlan["arms"] = []; + for (const arm of arms) { + if (arm.kind === "array" && emptyArms.some((e) => typeEquals(e, arm))) plan.push("empty"); + else if (arm.kind === "array" && converts(arm.elem)) plan.push("spread"); + else if (arm.kind !== "array" && converts(arm)) plan.push("value"); + else return null; + } + return { outElem, arms: plan }; +} + +/** The flatMap loop over a union-returning callback: each result tests + * its arm and appends its elements, nothing, or itself (Node's + * IsArray decision per value). */ +function flatMapUnionHelper( + lowerer: Lowerer, + elem: IrType, + fnRet: IrType & { kind: "union" }, + arity: number, + plan: FlatMapUnionPlan, + loc: SrcLoc, + present: boolean, +): string { + const key = `flatMapUnion:${typeKey(elem)}:${typeKey(fnRet)}:${typeKey(plan.outElem)}:${arity}${present ? ":present" : ""}`; + const existing = lowerer.arrHofHelpers.get(key); + if (existing) return existing; + const name = `%arr.flatMap.${lowerer.arrHofHelpers.size}`; + lowerer.arrHofHelpers.set(key, name); + const arrT = arrayOf(elem); + const outT = arrayOf(plan.outElem); + const fnT = funcOf([callbackValueType(lowerer, elem, present), F64, arrT].slice(0, arity), fnRet); + const arms = lowerer.unions.get(fnRet.unionId)!.arms; + const outRef = (): IrExpr => varRef("out.0", outT, loc); + const resultRef = (): IrExpr => varRef("r.0", fnRet, loc); + const outLength = (): IrExpr => ({ + kind: "arrIntrinsic", + method: "length", + receiver: outRef(), + args: [], + type: F64, + loc, + }); + const locals: IrLocal[] = [ + ...arrayCallbackLocals(arrT, fnT), + { id: "out.0", name: "out", type: outT, mutable: false }, + { id: "r.0", name: "r", type: fnRet, mutable: false }, + ]; + const armBody = (tag: number): IrStmt[] => { + const arm = arms[tag]!; + const narrowed: IrExpr = { + kind: "unionNarrow", + unionId: fnRet.unionId, + tag, + value: resultRef(), + type: arm, + loc, + }; + if (plan.arms[tag] === "empty") return []; + if (plan.arms[tag] === "value") { + return [ + arrayValueStore( + lowerer, + outRef(), + outLength(), + lowerer.coerceToExpected(narrowed, plan.outElem), + plan.outElem, + loc, + ), + ]; + } + const innerT = arm as IrType & { kind: "array" }; + const listId = `list.${tag}`; + const lengthId = `m.${tag}`; + const indexId = `j.${tag}`; + const itemId = `inner.${tag}`; + locals.push( + { id: listId, name: "list", type: innerT, mutable: false }, + { id: lengthId, name: "m", type: F64, mutable: false }, + { id: indexId, name: "j", type: F64, mutable: true }, + { id: itemId, name: "inner", type: arrayValueType(lowerer, innerT.elem), mutable: false }, + ); + const list = (): IrExpr => varRef(listId, innerT, loc); + const index = (): IrExpr => varRef(indexId, F64, loc); + return [ + { kind: "varDecl", localId: listId, init: narrowed, loc }, + { + kind: "varDecl", + localId: lengthId, + init: { + kind: "arrIntrinsic", + method: "length", + receiver: list(), + args: [], + type: F64, + loc, + }, + loc, + }, + { + kind: "for", + init: { kind: "varDecl", localId: indexId, init: numLit(0, loc), loc }, + cond: { + kind: "bin", + op: "<", + left: index(), + right: varRef(lengthId, F64, loc), + type: BOOL, + loc, + }, + update: { + kind: "assign", + localId: indexId, + value: { kind: "bin", op: "+", left: index(), right: numLit(1, loc), type: F64, loc }, + loc, + }, + body: [ + { + kind: "if", + cond: arrayIndexPresent(list(), index(), loc), + then: [ + { + kind: "varDecl", + localId: itemId, + init: arrayValueRead(lowerer, list(), index(), innerT.elem, loc), + loc, + }, + arrayValueStore( + lowerer, + outRef(), + outLength(), + lowerer.coerceToExpected( + varRef(itemId, arrayValueType(lowerer, innerT.elem), loc), + plan.outElem, + ), + plan.outElem, + loc, + ), + ], + else_: null, + loc, + }, + ], + loc, + }, + ]; + }; + let dispatch: IrStmt[] = armBody(arms.length - 1); + for (let tag = arms.length - 2; tag >= 0; tag--) { + dispatch = [ + { + kind: "if", + cond: { + kind: "unionIsTag", + unionId: fnRet.unionId, + tag, + value: resultRef(), + negated: false, + type: BOOL, + loc, + }, + then: armBody(tag), + else_: dispatch, + loc, + }, + ]; + } + const body: IrStmt[] = [ + { + kind: "varDecl", + localId: "out.0", + init: { kind: "arrayLit", elems: [], type: outT, loc }, + loc, + }, + arrayLengthDeclaration(arrT, loc), + countedFor(loc, varRef("n.0", F64, loc), () => [ + { + kind: "if", + cond: currentArrayIndexPresent(arrT, loc), + then: [ + { + kind: "varDecl", + localId: "r.0", + init: callArrayCallback( + arrT, + fnT, + fnRet, + arity, + callbackValueRead( + lowerer, + varRef("a.0", arrT, loc), + varRef("i.0", F64, loc), + elem, + present, + loc, + ), + loc, + ), + loc, + }, + ...dispatch, + ], + else_: null, + loc, + }, + ]), + { kind: "return", value: outRef(), loc }, + ]; + lowerer.liftedFns.push({ + name, + params: arrayCallbackParams(arrT, fnT), + returnType: outT, + locals, + body, + loc, + }); + return name; +} + /** The flatMap loop (array-returning callback), from existing IR nodes: * * out = []; n = a.length; diff --git a/packages/compiler/src/frontend/lowering/expressions/array-literals.ts b/packages/compiler/src/frontend/lowering/expressions/array-literals.ts index 8f119504c..e5ccd1176 100644 --- a/packages/compiler/src/frontend/lowering/expressions/array-literals.ts +++ b/packages/compiler/src/frontend/lowering/expressions/array-literals.ts @@ -58,6 +58,25 @@ export function jsHoldsDynObject(lowerer: Lowerer, expr: ts.Expression): boolean return false; } +/** The literal is the source of an array destructuring: an assignment's + * right side or a declaration's initializer under an array pattern. */ +function isPatternSource(expr: ts.ArrayLiteralExpression): boolean { + let node: ts.Node = expr; + while (ts.isParenthesizedExpression(node.parent)) node = node.parent; + const parent = node.parent; + if (ts.isBinaryExpression(parent)) + return ( + parent.right === node && + parent.operatorToken.kind === ts.SyntaxKind.EqualsToken && + ts.isArrayLiteralExpression(parent.left) + ); + return ( + ts.isVariableDeclaration(parent) && + parent.initializer === node && + ts.isArrayBindingPattern(parent.name) + ); +} + /** `[a, b, c]`. The element type comes from the contextual type when tsc * has one (`const a: number[] = []`, arguments, nested literals) and from * the literal's own inferred type otherwise. A bare `[]` with no context @@ -107,6 +126,22 @@ export function lowerArrayLiteral( ) mapped = inferred; } + // A destructuring pattern's implied type ends in a rest element + // (`[h, ...t] = [1, 2, 3]` supplies `[number, ...number[]]`), which maps + // to an array. The literal keeps its own fixed tuple layout, read by + // position like any tuple source. + if ( + expected === undefined && + ctxType && + isPatternSource(expr) && + lowerer.checker.isTupleType(ctxType) && + ( + (ctxType as ts.TupleTypeReference).getTarget() as ts.TupleType | undefined + )?.elementFlags?.some((flag) => (flag & ts.ElementFlags.Rest) !== 0) + ) { + const own = lowerer.mapTypeOf(lowerer.typeOf(expr)); + if (own?.kind === "record" && lowerer.shapes.get(own.shapeId)?.tuple) mapped = own; + } // A JS literal whose OWN inferred type is never-tainted // (neverTaintedJsType — the evolving `const gb = []`, the mixed // command tuple `['pwd', []]`) carries no element information: route diff --git a/packages/compiler/src/frontend/lowering/expressions/object-literals.ts b/packages/compiler/src/frontend/lowering/expressions/object-literals.ts index 678b375fe..d0edc8e28 100644 --- a/packages/compiler/src/frontend/lowering/expressions/object-literals.ts +++ b/packages/compiler/src/frontend/lowering/expressions/object-literals.ts @@ -933,11 +933,11 @@ export function lowerObjectLiteral( // spread's own position (any position: they introduce one fresh // name, collision-fenced below). Everything else stays fenced. const cs = conditionalSpreadOf(prop.expression) ?? andSpreadOf(lowerer, prop.expression); - if (cs === "unsupported" || (cs && cs.props.length !== 1)) { + if (cs === "unsupported" && conditionalRecordSpreadOf(lowerer, prop.expression) === null) { lowerer.unsupported( "SC1090", prop, - "conditional spreads beyond `...(c ? { field: v } : {})` (exactly one property against an empty arm — spell other shapes as optional fields)", + "conditional spreads beyond `...(c ? { field: v } : {})` and `...(c ? record : {})` (one arm must be empty — spell other shapes as optional fields)", ); } // Spread ORDER is fenced in the field-by-field desugar below, not @@ -1721,64 +1721,71 @@ export function lowerObjectLiteral( if (ts.isSpreadAssignment(prop)) { const cs = conditionalSpreadOf(prop.expression) ?? andSpreadOf(lowerer, prop.expression); if (cs && cs !== "unsupported") { - // `...(c ? { k: v } : {})` — ONE conditional field at the - // spread's own position: cond evaluates exactly once, `v` only - // when the non-empty arm is taken (ternary arms are lazy), and - // the empty arm holds the interned undefined arm — exactly the - // omitted-optional-field representation, which is also why the - // target field must be optional (tsc types the idiom that way). - const csProp = cs.props[0]!; // single-prop-checked in the fence pass - const name = csProp.name.text; - const fieldType = fieldTypes.get(name); - if (!fieldType) { - if (shape.indexValue) { + // `...(c ? { k: v } : {})` — one conditional field per carried + // property at the spread's own position: cond evaluates exactly + // once (into a hidden local when several fields test it), each + // `v` only when the non-empty arm is taken (ternary arms are + // lazy), in property order, and the empty arm holds the interned + // undefined arm — exactly the omitted-optional-field + // representation, which is also why the target fields must be + // optional (tsc types the idiom that way). + let cond: IrExpr | null = null; + for (const csProp of cs.props) { + const name = csProp.name.text; + const fieldType = fieldTypes.get(name); + if (!fieldType) { + if (shape.indexValue) { + lowerer.unsupported( + "SC1090", + prop, + "conditional spreads into index-signature keys (overflow entries model presence — write the key in an if statement instead)", + ); + } + shapeMismatch(prop); + } + // An 'unknown' slot holds the checked-dynamic undefined for the + // empty arm, the absent-property read the width copy uses too. + const absent = + lowerer.absentFieldValue(fieldType, locOf(prop)) ?? + (fieldType.kind === "dyn" ? dynUndefinedExpr(locOf(prop)) : null); + if (!absent) { lowerer.unsupported( "SC1090", prop, - "conditional spreads into index-signature keys (overflow entries model presence — write the key in an if statement instead)", + `conditional spreads onto the required field '${name}' (the empty arm leaves it undefined — declare the field optional)`, ); } - shapeMismatch(prop); + if (fields.some((f) => f.name === name)) { + lowerer.unsupported( + "SC1090", + prop, + `conditional spread of '${name}' over an earlier '${name}' (the desugar keeps one entry per name — restructure so each name has one contributor)`, + ); + } + if (cond === null) { + cond = lowerer.lowerCondition(cs.cond); + if (cs.props.length > 1) cond = capture(cond); + } + const valueNode = ts.isPropertyAssignment(csProp) ? csProp.initializer : csProp; + let v = ts.isPropertyAssignment(csProp) + ? lowerer.lowerExpr(csProp.initializer) + : lowerer.lowerShorthandValue(csProp); + v = lowerer.coerceInto(valueNode, v, fieldType); + if (!typeEquals(v.type, fieldType)) lowerer.badType(valueNode, lowerer.typeOf(valueNode)); + conditionalNames.add(name); + orderSegments.push({ kind: "key", name, definite: false }); + fields.push({ + name, + value: { + kind: "ternary", + cond, + then: cs.whenTrue ? v : absent, + else_: cs.whenTrue ? absent : v, + type: fieldType, + loc: locOf(prop), + }, + }); } - // An 'unknown' slot holds the checked-dynamic undefined for the - // empty arm, the absent-property read the width copy uses too. - const absent = - lowerer.absentFieldValue(fieldType, locOf(prop)) ?? - (fieldType.kind === "dyn" ? dynUndefinedExpr(locOf(prop)) : null); - if (!absent) { - lowerer.unsupported( - "SC1090", - prop, - `conditional spreads onto the required field '${name}' (the empty arm leaves it undefined — declare the field optional)`, - ); - } - if (fields.some((f) => f.name === name)) { - lowerer.unsupported( - "SC1090", - prop, - `conditional spread of '${name}' over an earlier '${name}' (the desugar keeps one entry per name — restructure so each name has one contributor)`, - ); - } - const cond = lowerer.lowerCondition(cs.cond); - const valueNode = ts.isPropertyAssignment(csProp) ? csProp.initializer : csProp; - let v = ts.isPropertyAssignment(csProp) - ? lowerer.lowerExpr(csProp.initializer) - : lowerer.lowerShorthandValue(csProp); - v = lowerer.coerceInto(valueNode, v, fieldType); - if (!typeEquals(v.type, fieldType)) lowerer.badType(valueNode, lowerer.typeOf(valueNode)); - conditionalNames.add(name); - orderSegments.push({ kind: "key", name, definite: false }); - fields.push({ - name, - value: { - kind: "ternary", - cond, - then: cs.whenTrue ? v : absent, - else_: cs.whenTrue ? absent : v, - type: fieldType, - loc: locOf(prop), - }, - }); continue; } // `{ ...base, ... }` — field-by-field copy of a known record @@ -1790,10 +1797,13 @@ export function lowerObjectLiteral( // overridden. Field snapshots above preserve the copy order. let srcNode: ts.Expression = prop.expression; while (ts.isParenthesizedExpression(srcNode)) srcNode = srcNode.expression; + const condRecord = conditionalRecordSpreadOf(lowerer, prop.expression); const srcLowered = capture( - prop === expr.properties.nodes[0] && leadingSpreadLowered !== null - ? leadingSpreadLowered - : lowerer.lowerExpr(srcNode), + condRecord !== null + ? lowerConditionalRecordSpread(lowerer, condRecord, locOf(prop)) + : prop === expr.properties.nodes[0] && leadingSpreadLowered !== null + ? leadingSpreadLowered + : lowerer.lowerExpr(srcNode), ); // Array callbacks can receive a record-or-undefined ABI even when // TypeScript describes a required record. Select the copy strategy @@ -2731,6 +2741,62 @@ function syntacticAndSpread( return { cond, props, whenTrue: true }; } +/** `...(c ? source : {})` (either arm order; the empty arm may also be + * `undefined` or `null`) where `source` maps to a record and is not the + * literal the field desugar handles (conditionalSpreadOf). The spread + * copies the source's fields when the source arm is taken and nothing + * otherwise, which is the copy of a `Record | undefined` spread source. */ +export function conditionalRecordSpreadOf( + lowerer: Lowerer, + expr: ts.Expression, +): { cond: ts.Expression; source: ts.Expression; whenTrue: boolean } | null { + let e: ts.Expression = expr; + while (ts.isParenthesizedExpression(e)) e = e.expression; + if (!ts.isConditionalExpression(e)) return null; + const unwrap = (a: ts.Expression): ts.Expression => { + let x = a; + while (ts.isParenthesizedExpression(x)) x = x.expression; + return x; + }; + const isEmpty = (x: ts.Expression): boolean => + (ts.isObjectLiteralExpression(x) && x.properties.nodes.length === 0) || + x.kind === ts.SyntaxKind.NullKeyword || + (ts.isIdentifier(x) && + x.text === "undefined" && + (lowerer.typeOf(x).flags & ts.TypeFlags.Undefined) !== 0); + const whenTrue = unwrap(e.whenTrue); + const whenFalse = unwrap(e.whenFalse); + const sourceIsTrue = isEmpty(whenFalse) && !isEmpty(whenTrue); + if (!sourceIsTrue && !(isEmpty(whenTrue) && !isEmpty(whenFalse))) return null; + const source = sourceIsTrue ? whenTrue : whenFalse; + if (lowerer.mapTypeOf(lowerer.typeOf(source))?.kind !== "record") return null; + return { cond: e.condition, source, whenTrue: sourceIsTrue }; +} + +/** The spread source of a conditionalRecordSpreadOf match: `c ? source : + * undefined` as a `Record | undefined` value (cond first, the source only + * when its arm is taken). */ +function lowerConditionalRecordSpread( + lowerer: Lowerer, + spread: { cond: ts.Expression; source: ts.Expression; whenTrue: boolean }, + loc: SrcLoc, +): IrExpr { + const cond = lowerer.lowerCondition(spread.cond); + const source = lowerer.lowerExpr(spread.source); + if (source.type.kind !== "record") lowerer.badType(spread.source, lowerer.typeOf(spread.source)); + const type = lowerer.withUndefinedArm(source.type); + const present = lowerer.coerceToExpected(source, type); + const absent = lowerer.wrappedUndefined(type, loc)!; + return { + kind: "ternary", + cond, + then: spread.whenTrue ? present : absent, + else_: spread.whenTrue ? absent : present, + type, + loc, + }; +} + /** A conditional spread's carrier property: `name: value` or shorthand, * with an identifier/string-literal name. */ export type CondSpreadProp = (ts.PropertyAssignment | ts.ShorthandPropertyAssignment) & { diff --git a/packages/compiler/src/frontend/lowering/lower-exprs.ts b/packages/compiler/src/frontend/lowering/lower-exprs.ts index b5acaf9e6..6a80c551e 100644 --- a/packages/compiler/src/frontend/lowering/lower-exprs.ts +++ b/packages/compiler/src/frontend/lowering/lower-exprs.ts @@ -1,5 +1,6 @@ import { keyedHelperName } from "./stable-names.js"; import { lowerWebAssemblyValue, refuseWebAssemblyUse } from "./builtins/webassembly.js"; +import { lowerAwaitedUniformPromiseAll } from "./builtins/promises.js"; import { dynUndefinedExpr, nodeThrowExpr, numLit, strLit, varRef } from "../../ir/build.js"; import { InternalCompilerError } from "../../errors.js"; /* Expression lowering: the expression dispatch (lowerExpr), literals @@ -575,6 +576,8 @@ function lowerExprInner(lowerer: Lowerer, expr: ts.Expression): IrExpr { // boundary for malformed/upstream ASTs. lowerer.unsupported("SC1090", expr, "top-level await (await outside async functions)"); } + const uniformAll = lowerAwaitedUniformPromiseAll(lowerer, expr); + if (uniformAll) return uniformAll; const value = lowerer.lowerExpr(expr.expression); // A promise-or-absent union (`Promise | undefined` values, calls // of `(...) => Promise | void` callbacks): the await handles diff --git a/packages/compiler/src/frontend/lowering/lower-stmts.ts b/packages/compiler/src/frontend/lowering/lower-stmts.ts index 213abfec6..831fec83d 100644 --- a/packages/compiler/src/frontend/lowering/lower-stmts.ts +++ b/packages/compiler/src/frontend/lowering/lower-stmts.ts @@ -67,9 +67,11 @@ import { F64, type IrExpr, type IrGlobal, + type IrLibFn, type IrLocal, type IrStmt, type IrType, + SEARCH_PARAMS_T, STRING, type SrcLoc, UNDEFINED_T, @@ -106,6 +108,7 @@ import { COMPOUND_ASSIGN_OPS, type CompoundOp, STR_METHODS, + NARROW_FIRST, UNSUPPORTED_STMT, isStdlibMember, sideEffectFreeOptionValue, @@ -3210,6 +3213,20 @@ export function lowerBindingPattern( lowerModuleNamespacePattern(lowerer, pattern, nsSource, isLet, out); return; } + if (srcType.kind === "url") { + lowerUrlObjectPattern(lowerer, pattern, srcRef, isLet, out); + return; + } + if ( + srcType.kind === "union" && + !isJsSourceFile(ts.getSourceFileOfNode(pattern)) && + lowerer.unions + .get(srcType.unionId) + ?.arms.every((arm) => arm.kind === "record" || arm.kind === "object") + ) { + lowerUnionObjectPattern(lowerer, pattern, srcRef, srcType, isLet, out); + return; + } if (srcType.kind !== "record") { lowerer.unsupported( "SC1031", @@ -3424,7 +3441,10 @@ function patternSourceValue( !arms || !present?.length || present.length === arms.length || - (present.length > 1 && !(arrayPattern && present.every((arm) => arm.kind === "array"))) + (present.length > 1 && + !(arrayPattern + ? present.every((arm) => arm.kind === "array") + : present.every((arm) => arm.kind === "record" || arm.kind === "object"))) ) return value; for (const [tag, arm] of arms.entries()) { @@ -3541,6 +3561,153 @@ function lowerModuleNamespacePattern( } } +/** The URL getters an object pattern reads (`const { protocol, hostname } + * = new URL(text)`), each a read of the URL's current serialization. */ +const URL_PATTERN_GETTERS: Record = { + href: "url.href", + origin: "url.origin", + protocol: "url.protocol", + username: "url.username", + password: "url.password", + host: "url.host", + hostname: "url.hostname", + port: "url.port", + pathname: "url.pathname", + search: "url.search", + hash: "url.hash", +}; + +/** An object pattern over a URL: each element reads its getter in pattern + * order, as `url.name` does; `searchParams` binds the URL's live view. + * Methods and rest elements stay fenced. */ +function lowerUrlObjectPattern( + lowerer: Lowerer, + pattern: ts.ObjectBindingPattern, + srcRef: () => IrExpr, + isLet: boolean, + out: IrStmt[], +): void { + for (const el of pattern.elements.nodes) { + if (el.name === undefined) continue; + if (el.dotDotDotToken) { + lowerer.unsupported( + "SC1031", + el, + "rest bindings over URL sources (a URL's properties are prototype getters, which rest elements do not copy)", + ); + } + const prop = el.propertyName ?? el.name; + const propName = + ts.isIdentifier(prop) || + ts.isComputedPropertyName(prop) || + ts.isStringLiteralLike(prop) || + ts.isNumericLiteral(prop) + ? patternKeyNameOf(lowerer, prop as ts.PropertyName) + : null; + const loc = locOf(el); + let value: IrExpr; + if (propName !== null && Object.hasOwn(URL_PATTERN_GETTERS, propName)) { + value = { + kind: "libCall", + fn: URL_PATTERN_GETTERS[propName]!, + args: [srcRef()], + type: STRING, + loc, + }; + } else if (propName === "searchParams") { + value = { + kind: "libCall", + fn: "url.searchParams", + args: [srcRef()], + type: SEARCH_PARAMS_T, + loc, + }; + } else { + lowerer.unsupported( + "SC1031", + el, + propName === null + ? "destructuring with computed keys that do not fold to one property name" + : `destructuring '${propName}' from a URL (${Object.keys(URL_PATTERN_GETTERS).join(", ")} and searchParams are the supported URL properties)`, + ); + } + if (el.initializer) value = applyBindingDefault(lowerer, el, value); + lowerer.bindPatternTarget(el.name, value, isLet, out); + } +} + +/** An object pattern over a union of records and class instances + * (`function f({ kind, path }: Add | Remove)`): each element reads a data + * field every arm declares with one type, through the runtime tag, as + * `value.field` does. Rest elements and fields the arms type differently + * stay fenced. */ +function lowerUnionObjectPattern( + lowerer: Lowerer, + pattern: ts.ObjectBindingPattern, + srcRef: () => IrExpr, + srcType: IrType & { kind: "union" }, + isLet: boolean, + out: IrStmt[], +): void { + const arms = lowerer.unions.get(srcType.unionId)?.arms ?? []; + for (const el of pattern.elements.nodes) { + if (el.name === undefined) continue; + if (el.dotDotDotToken) { + lowerer.unsupported( + "SC1031", + el, + `rest bindings over '${lowerer.fmt(srcType)}' sources (${NARROW_FIRST})`, + ); + } + const prop = el.propertyName ?? el.name; + const propName = + ts.isIdentifier(prop) || + ts.isComputedPropertyName(prop) || + ts.isStringLiteralLike(prop) || + ts.isNumericLiteral(prop) + ? patternKeyNameOf(lowerer, prop as ts.PropertyName) + : null; + if (propName === null) { + lowerer.unsupported( + "SC1031", + el, + "destructuring with computed keys that do not fold to one property name", + ); + } + let common: IrType | null = null; + for (const arm of arms) { + let ft: IrType | undefined; + if (arm.kind === "record") { + ft = lowerer.shapes.get(arm.shapeId)?.fields.find((f) => f.name === propName)?.type; + } else if (arm.kind === "object") { + ft = lowerer.classes.get(arm.className)?.fields.get(propName); + } + if (!ft || ft.kind === "void" || isUnitType(ft) || (common && !typeEquals(common, ft))) { + common = null; + break; + } + common = ft; + } + if (common === null) { + lowerer.unsupported( + "SC1031", + el, + `destructuring '${propName}' from a '${lowerer.fmt(srcType)}' source (every arm must be a record or class with a same-typed field '${propName}' — ${NARROW_FIRST})`, + ); + } + let value: IrExpr = { + kind: "unionDisc", + unionId: srcType.unionId, + field: propName, + value: srcRef(), + type: common, + loc: locOf(el), + }; + if (el.initializer) value = applyBindingDefault(lowerer, el, value); + lowerer.bindPatternTarget(el.name, value, isLet, out); + } +} + /** Whether a pattern's first element has a computed key that does not * fold to one property name, so the key evaluates at run time. */ function startsWithRuntimeKey(lowerer: Lowerer, pattern: ts.ObjectBindingPattern): boolean { diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index b02a0a843..d87ace914 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -10784,9 +10784,29 @@ export class Lowerer { // Contextual types (such as Proxy descriptors' symbol-valued fields) // must not force an intermediate record with incompatible field slots. if (this.returnsLiteralIntoDyn(node, expected)) return this.lowerExprExpecting(node, expected); + if (this.returnsFreshCollection(node) && expected.kind !== "void") + return this.lowerExprExpecting(node, expected); return this.asyncReturnValueOf(node, this.lowerExpr(node), expected); } + /** An array literal or a `new Map(...)`, `new Set(...)` or `new + * Array(...)` of the builtins: a fresh collection, never a thenable, so + * an async return takes the declared result type as its construction + * layout the way a synchronous return does (`return []` in a function + * whose inferred result is `Promise`). */ + private returnsFreshCollection(node: ts.Expression): boolean { + let e = node; + while (ts.isParenthesizedExpression(e)) e = e.expression; + if (ts.isArrayLiteralExpression(e)) return true; + return ( + ts.isNewExpression(e) && + ts.isIdentifier(e.expression) && + (this.isStdlibGlobal(e.expression, "Map") || + this.isStdlibGlobal(e.expression, "Set") || + this.isStdlibGlobal(e.expression, "Array")) + ); + } + /** A fresh JS object literal returned into a checked-native (dyn) result, * which lowers straight into that storage. */ private returnsLiteralIntoDyn(node: ts.Expression, expected: IrType): boolean { @@ -10918,7 +10938,11 @@ export class Lowerer { loc, }; } - if (this.adoptsReturnedPromise() && !this.returnsLiteralIntoDyn(node, expected)) { + if ( + this.adoptsReturnedPromise() && + !this.returnsLiteralIntoDyn(node, expected) && + !this.returnsFreshCollection(node) + ) { // The async return lowers its operand untyped first (a promise // adopts before its payload converts); a maybe-promise union needs // the statement form, which branches between adoption and fulfillment. diff --git a/packages/compiler/src/frontend/lowering/width-lift.ts b/packages/compiler/src/frontend/lowering/width-lift.ts index e3722bc32..126f80f93 100644 --- a/packages/compiler/src/frontend/lowering/width-lift.ts +++ b/packages/compiler/src/frontend/lowering/width-lift.ts @@ -11,6 +11,16 @@ export type WidthLift = | { how: "retag" } | { how: "unitRetag" } | { how: "liftWrap"; tag: number; arm: IrType } + /** A record into a union of two record arms that differ by one field: + * the source field `field` (`T | undefined`) selects `present` when it + * holds a value and `absent`, which lacks the field, when it is + * undefined. */ + | { + how: "presenceRoute"; + field: string; + present: { tag: number; arm: IrType }; + absent: { tag: number; arm: IrType }; + } | { how: "width" } | { how: "unionWidth" } | { how: "arr" } @@ -20,5 +30,6 @@ export type WidthLift = | { how: "clsWidth" } | { how: "narrow" } | { how: "dynIn" } + | { how: "dynOut" } | { how: "upcast" } | { how: "funcAdapt" }; diff --git a/packages/compiler/src/frontend/type-mapper.ts b/packages/compiler/src/frontend/type-mapper.ts index bd2a32c37..aa275a27d 100644 --- a/packages/compiler/src/frontend/type-mapper.ts +++ b/packages/compiler/src/frontend/type-mapper.ts @@ -2361,6 +2361,27 @@ function mapTypeInner(type: ts.Type, ctx: TypeMapperCtx): IrType | null { const args = checker.getTypeArguments(ref); if (args.length === 0) return arrayOf(unitOnlyUnion(unions)); if (tupleShape?.elementFlags === undefined) return null; + // A leading-required tuple ending in a rest element whose positions + // all share one type (`[string, ...string[]]`, the non-empty list) + // is an array of that type: every read, write and method behaves as + // the array's, and tsc already proves the required positions exist. + const flags = tupleShape.elementFlags; + if ( + flags.length >= 2 && + (flags[flags.length - 1]! & ts.ElementFlags.Rest) !== 0 && + flags.slice(0, -1).every((f) => (f & ts.ElementFlags.Required) !== 0) + ) { + const elems = args.map((a) => mapType(a, ctx)); + const first = elems[0]; + if ( + first && + first.kind !== "void" && + isSupportedArrayElem(first) && + elems.every((e) => e !== null && typeEquals(e, first)) + ) + return arrayOf(first); + return null; + } // Optional/rest elements have no fixed native tuple layout. if (tupleShape.elementFlags.some((f) => !(f & ts.ElementFlags.Required))) { return null; diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7020-conditional-spread-fields.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7020-conditional-spread-fields.ts.json new file mode 100644 index 000000000..f6c5594a1 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7020-conditional-spread-fields.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7020-conditional-spread-fields.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7021-promise-all-uniform-tuple.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7021-promise-all-uniform-tuple.ts.json new file mode 100644 index 000000000..c1e5f1ae4 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7021-promise-all-uniform-tuple.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7021-promise-all-uniform-tuple.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7022-promise-all-dynamic-arrays.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7022-promise-all-dynamic-arrays.ts.json new file mode 100644 index 000000000..c1856ac5a --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7022-promise-all-dynamic-arrays.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7022-promise-all-dynamic-arrays.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7023-union-object-patterns.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7023-union-object-patterns.ts.json new file mode 100644 index 000000000..bae130837 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7023-union-object-patterns.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7023-union-object-patterns.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7024-async-return-fresh-collections.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7024-async-return-fresh-collections.ts.json new file mode 100644 index 000000000..c4b9855ef --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7024-async-return-fresh-collections.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7024-async-return-fresh-collections.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7025-child-listener-chaining.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7025-child-listener-chaining.ts.json new file mode 100644 index 000000000..ed0e7b650 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7025-child-listener-chaining.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7025-child-listener-chaining.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7026-url-object-patterns.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7026-url-object-patterns.ts.json new file mode 100644 index 000000000..f050c4d1a --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7026-url-object-patterns.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7026-url-object-patterns.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7027-search-params-conditional-init.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7027-search-params-conditional-init.ts.json new file mode 100644 index 000000000..9b69861b1 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7027-search-params-conditional-init.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7027-search-params-conditional-init.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7028-non-empty-rest-tuples.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7028-non-empty-rest-tuples.ts.json new file mode 100644 index 000000000..a40ef1180 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7028-non-empty-rest-tuples.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7028-non-empty-rest-tuples.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7029-flatmap-union-results.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7029-flatmap-union-results.ts.json new file mode 100644 index 000000000..72d564d3b --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7029-flatmap-union-results.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7029-flatmap-union-results.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7033-any-fields-into-declared-records.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7033-any-fields-into-declared-records.ts.json new file mode 100644 index 000000000..57e76b621 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7033-any-fields-into-declared-records.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7033-any-fields-into-declared-records.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7034-optional-field-union-routes.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7034-optional-field-union-routes.ts.json new file mode 100644 index 000000000..a78b23cb1 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7034-optional-field-union-routes.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7034-optional-field-union-routes.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/retired-engine-1123-any-conditional-spread.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/retired-engine-1123-any-conditional-spread.ts.json deleted file mode 100644 index 057d24d55..000000000 --- a/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/retired-engine-1123-any-conditional-spread.ts.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "order": [ - "/tests/diagnostics/retired-engine-1123-any-conditional-spread.ts" - ], - "diags": [] -} diff --git a/tests/corpus/7020-conditional-spread-fields.ts b/tests/corpus/7020-conditional-spread-fields.ts new file mode 100644 index 000000000..61c250174 --- /dev/null +++ b/tests/corpus/7020-conditional-spread-fields.ts @@ -0,0 +1,137 @@ +// Conditional spreads that carry several properties, and conditional +// spreads of a record value against an empty arm. The condition runs +// once, the carried values run only when their arm is taken and in +// property order, and the keys appear at the spread's position. +const log: string[] = []; +function note(label: string, value: T): T { + log.push(label); + return value; +} + +interface Extra { + region: string; + zone: number; +} + +function build(flag: boolean, model: string | undefined, extra: Extra | undefined) { + return { + name: note("name", "tool"), + ...(note("cond", flag) ? { verbose: note("verbose", true), depth: note("depth", 2) } : {}), + ...(model ? { model } : {}), + ...(extra ? extra : {}), + last: note("last", 0), + }; +} + +for (const [flag, model, extra] of [ + [true, "m1", { region: "eu", zone: 3 }], + [false, undefined, undefined], + [true, undefined, { region: "us", zone: 1 }], +] as const) { + log.length = 0; + const out = build(flag, model, extra); + console.log(JSON.stringify(out), Object.keys(out).join(","), log.join(" ")); + console.log("verbose" in out, out.depth ?? "no depth", out.region ?? "no region"); +} + +// The carrying arm can be the false arm, and `c && { ... }` carries +// several properties too. +function flags(quiet: boolean, color: boolean) { + return { + ...(quiet ? {} : { level: "info", prefix: "[app]" }), + ...(color && { fg: "green", bg: "black", bold: true }), + }; +} +console.log(JSON.stringify(flags(false, true)), JSON.stringify(flags(true, false))); + +// A record arm in either position, against `{}`, `undefined` or `null`. +interface Auth { + user: string; + token?: string; +} +const auth: Auth = { user: "u", token: "t" }; +const bare: Auth = { user: "v" }; +function request(useAuth: boolean, source: Auth) { + return { + url: "/x", + ...(useAuth ? source : {}), + ...(useAuth ? undefined : { anonymous: true }), + }; +} +console.log(JSON.stringify(request(true, auth)), JSON.stringify(request(false, auth))); +console.log(JSON.stringify(request(true, bare)), Object.keys(request(true, bare)).join(",")); +function fallback(primary: Auth | null, secondary: Auth) { + return { ...(primary === null ? secondary : null), id: 1 }; +} +console.log(JSON.stringify(fallback(null, auth)), JSON.stringify(fallback(bare, auth))); + +// A record arm read through a call runs only when its arm is taken. +let reads = 0; +function current(): Extra { + reads++; + return { region: "ap", zone: reads }; +} +const a = { ...(reads > 0 ? current() : {}), tag: "a" }; +const b = { ...(reads === 0 ? current() : {}), tag: "b" }; +console.log(JSON.stringify(a), JSON.stringify(b), reads); + +// Into a declared shape with optional fields. +interface Options { + cwd: string; + shell?: boolean; + timeout?: number; + env?: string; +} +function options(detached: boolean, env: string | undefined): Options { + return { cwd: "/", ...(detached ? { shell: false, timeout: 5 } : {}), ...(env ? { env } : {}) }; +} +console.log(JSON.stringify(options(true, "prod")), JSON.stringify(options(false, undefined))); + +// Inside 'any'-typed object literals, where the key is absent when the +// empty arm is taken. +function anyLiterals(): void { + function opts(aspect: string | undefined): any { + const o: any = { + mode: "M", + ...(aspect ? { cfg: { aspect } } : {}), + }; + return o; + } + const a = opts("1:1"); + console.log(`${a.mode} ${a.cfg.aspect} ${typeof a.cfg}`); + const b = opts(undefined); + console.log(`${b.mode} ${typeof b.cfg}`); + + // Several carried properties. + function multi(on: boolean): any { + return { base: 1, ...(on ? { x: "X", y: "Y" } : {}) }; + } + const m1 = multi(true); + const m2 = multi(false); + console.log(`${m1.base} ${m1.x} ${m1.y}`); + console.log(`${m2.base} ${typeof m2.x} ${typeof m2.y}`); + + // Reversed orientation and string-literal keys. + function rev(quiet: boolean): any { + return { kind: "r", ...(quiet ? {} : { "log-level": "loud" }) }; + } + console.log(`${rev(false)["log-level"]}`); + console.log(`${typeof rev(true)["log-level"]}`); + + // The spread mid-literal: properties around it evaluate exactly once and + // land in engine key order. + let sides = 0; + function side(v: number): number { + sides++; + return v; + } + function ordered(flag: boolean): any { + const o: any = { first: side(1), ...(flag ? { mid: 2 } : {}), last: side(3) }; + return o; + } + const o1 = ordered(true); + const o2 = ordered(false); + console.log(`${o1.first} ${o1.mid} ${o1.last}`); + console.log(`${o2.first} ${typeof o2.mid} ${o2.last} ${sides}`); +} +anyLiterals(); diff --git a/tests/corpus/7021-promise-all-uniform-tuple.ts b/tests/corpus/7021-promise-all-uniform-tuple.ts new file mode 100644 index 000000000..461351992 --- /dev/null +++ b/tests/corpus/7021-promise-all-uniform-tuple.ts @@ -0,0 +1,46 @@ +// `const pair = await Promise.all([p, q])` over entries of one promise +// type: the checker types the binding as a tuple, and the values arrive +// on the same microtask turn as Node's, in entry order whatever the +// settlement order. +const order: string[] = []; +function later(label: string, turns: number): Promise { + return new Promise((resolve) => { + let left = turns; + const step = (): void => { + if (left-- === 0) { + order.push(`settle ${label}`); + resolve(`${label}!`); + } else queueMicrotask(step); + }; + step(); + }); +} +async function count(n: number): Promise { + return n * 10; +} + +async function main(): Promise { + let ticking = true; + void (async () => { + for (let i = 0; i < 20 && ticking; i++) { + order.push(`t${i}`); + await null; + } + })(); + const bodies = await Promise.all([later("a", 3), later("b", 1)]); + order.push(`bodies ${bodies[0]} ${bodies[1]} ${bodies.length}`); + const numbers = await Promise.all([count(1), count(2), count(3)]); + const [x, y, z] = numbers; + order.push(`numbers ${x + y + z} ${numbers.length}`); + const annotated: [string, string] = await Promise.all([later("c", 0), later("d", 0)]); + order.push(`annotated ${annotated.join("+")}`); + ticking = false; + console.log(order.join("\n")); + try { + const failed = await Promise.all([later("e", 2), Promise.reject(new Error("no"))]); + console.log("unreachable", failed[0]); + } catch (e) { + console.log("caught", (e as Error).message); + } +} +void main(); diff --git a/tests/corpus/7022-promise-all-dynamic-arrays.ts b/tests/corpus/7022-promise-all-dynamic-arrays.ts new file mode 100644 index 000000000..dca5cdcde --- /dev/null +++ b/tests/corpus/7022-promise-all-dynamic-arrays.ts @@ -0,0 +1,41 @@ +// Promise.all over arrays built from 'any' values: the argument is a +// runtime value, so the combinator reads it with Node's iteration and +// resolves each entry (promises, thenables and plain values). +interface Check { + name: string; + ok: boolean; +} +async function check(name: string): Promise { + await null; + return { name, ok: name.length > 2 }; +} + +async function main(): Promise { + const raw: any = JSON.parse('{"items":[{"name":"abc"},{"name":"x"}],"ids":[1,2,3]}'); + const results = await Promise.all(raw.items.map((item: any) => check(item.name))); + for (const r of results) console.log(r.name, r.ok); + const doubled = await Promise.all(raw.ids.map(async (id: number) => id * 2)); + console.log(doubled.join(","), doubled.length); + const mixed = await Promise.all(raw.ids.map((id: number) => (id % 2 ? id : Promise.resolve(-id)))); + console.log(JSON.stringify(mixed)); + const parsed: unknown = JSON.parse('["a","bb","ccc"]'); + const list = Array.isArray(parsed) ? parsed : []; + const lengths = await Promise.all(list.map(async (s) => String(s).length)); + console.log(lengths, lengths.reduce((a, b) => a + b, 0)); + const empty: any = []; + console.log(JSON.stringify(await Promise.all(empty))); + try { + await Promise.all(raw.items.map(async (item: any) => { + if (item.name === "x") throw new Error(`bad ${item.name}`); + return item.name; + })); + } catch (e) { + console.log("rejected", (e as Error).message); + } + try { + await Promise.all(raw.missing); + } catch (e) { + console.log("not iterable", e instanceof TypeError); + } +} +void main(); diff --git a/tests/corpus/7023-union-object-patterns.ts b/tests/corpus/7023-union-object-patterns.ts new file mode 100644 index 000000000..8717e892a --- /dev/null +++ b/tests/corpus/7023-union-object-patterns.ts @@ -0,0 +1,63 @@ +// Object patterns over unions of records and classes read the fields +// every arm declares, in parameters, declarations and loops. +interface Add { + kind: "add"; + path: string; + content: string; + mode?: number; +} +interface Remove { + kind: "remove"; + path: string; + mode?: number; +} +type Change = Add | Remove; + +function describe({ kind, path: file, mode = 0o644 }: Change): string { + return `${kind}:${file}:${mode.toString(8)}`; +} +const changes: Change[] = [ + { kind: "add", path: "a.txt", content: "x" }, + { kind: "remove", path: "b.txt", mode: 0o600 }, +]; +for (const change of changes) console.log(describe(change)); +for (const { kind, path } of changes) console.log(kind, path.length); +function applyChange(change: Change): number { + const { path } = change; + let { kind } = change; + if (kind === "add") kind = "remove"; + return path.length + kind.length; +} +console.log(applyChange(changes[0]!), applyChange(changes[1]!)); + +// The source runs once, and the fields read in pattern order. +let reads = 0; +function next(): Change { + reads++; + return reads % 2 ? changes[0]! : changes[1]!; +} +const { path: p1, kind: k1 } = next(); +const { kind: k2 } = next(); +console.log(p1, k1, k2, reads); + +class Circle { + name: string; + size: number; + constructor(name: string, size: number) { + this.name = name; + this.size = size; + } +} +class Square { + name: string; + size: number; + corners = 4; + constructor(name: string, size: number) { + this.name = name; + this.size = size; + } +} +function label({ name, size }: Circle | Square): string { + return `${name}/${size}`; +} +console.log(label(new Circle("c", 2)), label(new Square("s", 3))); diff --git a/tests/corpus/7024-async-return-fresh-collections.ts b/tests/corpus/7024-async-return-fresh-collections.ts new file mode 100644 index 000000000..8343e028b --- /dev/null +++ b/tests/corpus/7024-async-return-fresh-collections.ts @@ -0,0 +1,48 @@ +// Async functions can return an empty array literal (where the result is +// inferred) or a new Map or Set: the value takes the result's element +// types, as a synchronous return does. +import { existsSync, readdirSync } from "node:fs"; + +async function listJson(dir: string) { + if (!existsSync(dir)) return []; + return readdirSync(dir).filter((name) => name.endsWith(".json")); +} + +async function childPaths(root: string) { + if (root === "") { + return []; + } + const found = [root, `${root}/b`]; + return found; +} + +async function index(names: string[]): Promise> { + if (names.length === 0) return new Map(); + const byName = new Map(); + for (const name of names) byName.set(name, name.length); + return byName; +} + +async function unique(words: string[]): Promise> { + if (words.length === 0) return new Set(); + return new Set(words); +} + +const pick = async (flag: boolean) => (flag ? [] : ["a", "b"]); + +async function main(): Promise { + const missing = await listJson("/nonexistent-dir-for-test"); + missing.push("pushed"); + console.log(missing, missing.length); + console.log(await childPaths(""), await childPaths("/root")); + const empty = await index([]); + empty.set("late", 4); + console.log(empty.size, empty.get("late"), (await index(["ab", "c"])).get("ab")); + const none = await unique([]); + none.add("z"); + console.log([...none], [...(await unique(["x", "x", "y"]))]); + const first = await pick(true); + first.push("c"); + console.log(first, await pick(false)); +} +void main(); diff --git a/tests/corpus/7025-child-listener-chaining.ts b/tests/corpus/7025-child-listener-chaining.ts new file mode 100644 index 000000000..317c87db1 --- /dev/null +++ b/tests/corpus/7025-child-listener-chaining.ts @@ -0,0 +1,42 @@ +// child.on() and child.once() return the child, so registrations chain +// and the result can be stored or passed on. The receiver evaluates once, +// before the listener. Each child starts after the previous one closed. +import { spawn } from "node:child_process"; + +const events: string[] = []; +let lookups = 0; +const children = [spawn("sh", ["-c", "exit 3"], { stdio: "ignore" })]; +function first() { + lookups++; + return children[0]!; +} + +const returned = first() + .on("exit", (code) => events.push(`exit ${code}`)) + .once("close", (code, signal) => { + events.push(`close ${code} ${signal}`); + console.log(events.join(", "), lookups); + second(); + }) + .on("error", (err) => events.push(`error ${err.message}`)); +console.log(returned === children[0], lookups); + +function second(): void { + spawn("sh", ["-c", "exit 0"], { stdio: "ignore" }) + .on("spawn", () => console.log("spawned")) + .on("exit", (code) => console.log("second exit", code)) + .on("close", () => third()); +} + +function third(): void { + const missing = spawn("/nonexistent/command-for-test", [], { stdio: "ignore" }).on( + "error", + (e) => console.log("missing", (e as NodeJS.ErrnoException).code), + ); + console.log(typeof missing.pid); + const detached = spawn("sh", ["-c", "exit 0"], { detached: true, stdio: "ignore" }).once( + "error", + (e) => console.log("spawn failed", e.message), + ); + detached.unref(); +} diff --git a/tests/corpus/7026-url-object-patterns.ts b/tests/corpus/7026-url-object-patterns.ts new file mode 100644 index 000000000..c2bc59958 --- /dev/null +++ b/tests/corpus/7026-url-object-patterns.ts @@ -0,0 +1,27 @@ +// Object patterns over URLs read the URL's getters in pattern order, +// with renames and defaults, in declarations and parameters. +let built = 0; +function parse(text: string): URL { + built++; + return new URL(text); +} +const { protocol, hostname, port } = parse("http://localhost:8080/path?q=1#top"); +console.log(protocol, hostname, port, built); +const { host: where, pathname: path = "/", search, hash, href } = parse("https://example.com/a/b?x=1&y=2#h"); +console.log(where, path, search, hash, href, built); + +function origin({ protocol: scheme, host, username, password }: URL): string { + return `${scheme}//${username}:${password}@${host}`; +} +console.log(origin(new URL("ftp://user:secret@files.example.org:2121/dir"))); + +const { searchParams } = new URL("http://h/?a=1&b=2"); +searchParams.append("c", "3"); +console.log(searchParams.toString(), [...searchParams.keys()].join(",")); + +const { origin: o, port: p } = new URL("https://[::1]:8443/"); +console.log(o, p); +for (const text of ["http://a.test:1", "http://b.test"]) { + const { hostname: name, port: listen } = new URL(text); + console.log(name, listen === "" ? "(default)" : listen); +} diff --git a/tests/corpus/7027-search-params-conditional-init.ts b/tests/corpus/7027-search-params-conditional-init.ts new file mode 100644 index 000000000..580d6c039 --- /dev/null +++ b/tests/corpus/7027-search-params-conditional-init.ts @@ -0,0 +1,21 @@ +// new URLSearchParams(c ? a : b) builds from the arm the condition +// selects; each arm may be a record literal, a string, pairs or another +// URLSearchParams. +let checks = 0; +function flag(value: boolean): boolean { + checks++; + return value; +} +for (const on of [true, false]) { + const params = new URLSearchParams(flag(on) ? { code: "x y", state: "s&t" } : {}); + console.log(params.toString(), params.size); + const query = new URLSearchParams(on ? "a=1&b=2" : [["c", "3"]]); + console.log(query.toString()); + const base = new URLSearchParams("keep=1"); + const copy = new URLSearchParams(on ? base : undefined); + copy.append("more", "2"); + console.log(copy.toString(), base.toString()); + const nested = new URLSearchParams(on ? (checks > 1 ? { n: "2" } : { n: "1" }) : "?empty="); + console.log(nested.toString()); +} +console.log(checks); diff --git a/tests/corpus/7028-non-empty-rest-tuples.ts b/tests/corpus/7028-non-empty-rest-tuples.ts new file mode 100644 index 000000000..5d00952ab --- /dev/null +++ b/tests/corpus/7028-non-empty-rest-tuples.ts @@ -0,0 +1,38 @@ +// Tuples of required positions followed by a rest element of the same +// type (`[string, ...string[]]`, the non-empty list) behave as arrays of +// that type. +type Hosts = [string, ...string[]]; +const LOCAL_HOSTS: Hosts = ["127.0.0.1", "::1", "localhost"]; +const single: [string, ...string[]] = ["only"]; +const pair: readonly [number, number, ...number[]] = [1, 2]; + +function first(hosts: Hosts): string { + return hosts[0]; +} +function isLocalHost(host: string): boolean { + return LOCAL_HOSTS.includes(host); +} +console.log(first(LOCAL_HOSTS), first(single), LOCAL_HOSTS.length, single.length); +console.log(isLocalHost("::1"), isLocalHost("10.0.0.1"), LOCAL_HOSTS.indexOf("localhost")); +console.log(LOCAL_HOSTS.slice(1).join(","), LOCAL_HOSTS.map((h) => h.length)); +for (const host of LOCAL_HOSTS) console.log(host); +const [head, ...rest] = LOCAL_HOSTS; +console.log(head, rest, JSON.stringify(LOCAL_HOSTS)); +single.push("more"); +console.log(single, single[1], pair[0] + pair[1], pair.length); + +function hostsFrom(text: string): Hosts { + const parts = text.split(","); + const out: Hosts = [parts[0]!]; + for (const part of parts.slice(1)) out.push(part); + return out; +} +const parsed = hostsFrom("a,b,c"); +console.log(parsed, parsed.length, `${parsed}`); + +function join(...names: [string, ...string[]]): string { + return names.join("+"); +} +console.log(join("x"), join("x", "y", "z"), join(...LOCAL_HOSTS)); +const lists: Hosts[] = [LOCAL_HOSTS, single]; +console.log(lists.map((l) => l[0]).join(" ")); diff --git a/tests/corpus/7029-flatmap-union-results.ts b/tests/corpus/7029-flatmap-union-results.ts new file mode 100644 index 000000000..83c9583de --- /dev/null +++ b/tests/corpus/7029-flatmap-union-results.ts @@ -0,0 +1,31 @@ +// flatMap callbacks that return a value or an array: an array result +// appends its elements (holes skipped), any other result appends itself, +// and an empty `[]` arm appends nothing. +const words = ["a", "", "c", "", "e"]; +console.log(words.flatMap((s) => (s ? s : []))); +console.log(words.flatMap((s, i) => (s ? [s, String(i)] : []))); + +function expand(line: string): string | string[] { + return line.includes(",") ? line.split(",") : line; +} +console.log(["x", "y,z", "w"].flatMap(expand), ["x", "y,z"].flatMap((l) => expand(l)).length); + +const numbers = [1, 2, 3, 4]; +console.log(numbers.flatMap((n) => (n % 2 ? n : [n, n * 10]))); +const sparse: number[] = []; +sparse[0] = 1; +sparse[2] = 3; +console.log(numbers.flatMap((n) => (n === 2 ? sparse : n))); + +interface Entry { + name: string; + tags?: string[]; +} +const entries: Entry[] = [{ name: "a", tags: ["t1", "t2"] }, { name: "b" }]; +console.log(entries.flatMap((e) => e.tags ?? []), entries.flatMap((e) => e.tags ?? e.name)); +let calls = 0; +const out = words.flatMap((s) => { + calls++; + return s.length > 0 ? s.toUpperCase() : []; +}); +console.log(out, calls); diff --git a/tests/corpus/7033-any-fields-into-declared-records.ts b/tests/corpus/7033-any-fields-into-declared-records.ts new file mode 100644 index 000000000..89c0f69e8 --- /dev/null +++ b/tests/corpus/7033-any-fields-into-declared-records.ts @@ -0,0 +1,28 @@ +// Records whose fields TypeScript types `any` convert into declared +// record types: each such field is checked against the declared type, +// as a typed binding initialized from the value is. +interface Transfer { + source: string; + targetId: string; + bytes: number; + tags: string[]; +} +const meta: any = JSON.parse('{"id":"x-1","size":4,"tags":["a","b"]}'); + +const mapped: Transfer[] = ["a", "b"].map((s, i) => ({ + source: s, + targetId: meta.id, + bytes: meta.size + i, + tags: meta.tags, +})); +console.log(JSON.stringify(mapped)); + +function build(): Transfer { + const record = { source: "c", targetId: meta.id, bytes: meta.size, tags: [] as string[] }; + return record; +} +console.log(JSON.stringify(build())); + +const list: Transfer[] = []; +for (const entry of [{ source: "d", targetId: meta.id, bytes: 1, tags: meta.tags }]) list.push(entry); +console.log(list.length, list[0]!.targetId.toUpperCase(), list[0]!.tags.length); diff --git a/tests/corpus/7034-optional-field-union-routes.ts b/tests/corpus/7034-optional-field-union-routes.ts new file mode 100644 index 000000000..0fb93ff8d --- /dev/null +++ b/tests/corpus/7034-optional-field-union-routes.ts @@ -0,0 +1,26 @@ +// A record whose `T | undefined` field is the one field that tells two +// arms of a union apart converts into that union: a value picks the arm +// that declares the field, and undefined picks the arm without it. +type Summary = + | { written: number; errors: string[]; ignored: number } + | { written: number; ignored: number }; + +function summarize(written: number, ignored: number, errors: string[]): Summary { + return { written, errors: errors.length > 0 ? errors : undefined, ignored }; +} +function errorCount(summary: Summary): number { + return "errors" in summary && summary.errors !== undefined ? summary.errors.length : 0; +} +for (const errors of [[], ["e1", "e2"]]) { + const summary = summarize(3, 1, errors); + console.log(JSON.stringify(summary), errorCount(summary), summary.written + summary.ignored); +} + +type Lookup = { name: string; value: number; source: string } | { name: string; source: string }; +const values = new Map([["a", 1]]); +const lookups: Lookup[] = ["a", "b"].map((name) => ({ + name, + value: values.get(name), + source: "map", +})); +console.log(JSON.stringify(lookups)); diff --git a/tests/diagnostics/destructuring-fences.ts b/tests/diagnostics/destructuring-fences.ts index e42f8a07a..0ee60579d 100644 --- a/tests/diagnostics/destructuring-fences.ts +++ b/tests/diagnostics/destructuring-fences.ts @@ -38,12 +38,12 @@ interface E1 { } interface E2 { kind: "two"; - b: string; + b: number; label: string; } function fromUnion(u: E1 | E2): string { - const { b } = u; // union source: narrow first - return b; + const { b } = u; // union source whose arms type the field differently: narrow first + return String(b); } console.log(fromUnion({ kind: "one", b: "s", count: 1 })); diff --git a/tests/diagnostics/retired-engine-1123-any-conditional-spread.ts b/tests/diagnostics/retired-engine-1123-any-conditional-spread.ts deleted file mode 100644 index 03043a32f..000000000 --- a/tests/diagnostics/retired-engine-1123-any-conditional-spread.ts +++ /dev/null @@ -1,49 +0,0 @@ -// The conditional-spread idiom inside `any`-typed object literals: -// `{ a, ...(c ? { k: v } : {}) }` builds one of two engine literals behind -// the test, so the key is truly ABSENT when the empty arm is taken — -// engine-exact (`typeof o.k` answers "undefined", JSON drops it), with the -// shared properties evaluating exactly once. - -function opts(aspect: string | undefined): any { - const o: any = { - mode: "M", - ...(aspect ? { cfg: { aspect } } : {}), - }; - return o; -} -const a = opts("1:1"); -console.log(`${a.mode} ${a.cfg.aspect} ${typeof a.cfg}`); -const b = opts(undefined); -console.log(`${b.mode} ${typeof b.cfg}`); - -// Multi-property carrier arms are fine in the island form. -function multi(on: boolean): any { - return { base: 1, ...(on ? { x: "X", y: "Y" } : {}) }; -} -const m1 = multi(true); -const m2 = multi(false); -console.log(`${m1.base} ${m1.x} ${m1.y}`); -console.log(`${m2.base} ${typeof m2.x} ${typeof m2.y}`); - -// Reversed orientation and string-literal keys. -function rev(quiet: boolean): any { - return { kind: "r", ...(quiet ? {} : { "log-level": "loud" }) }; -} -console.log(`${rev(false)["log-level"]}`); -console.log(`${typeof rev(true)["log-level"]}`); - -// The spread mid-literal: properties around it evaluate exactly once and -// land in engine key order. -let sides = 0; -function side(v: number): number { - sides++; - return v; -} -function ordered(flag: boolean): any { - const o: any = { first: side(1), ...(flag ? { mid: 2 } : {}), last: side(3) }; - return o; -} -const o1 = ordered(true); -const o2 = ordered(false); -console.log(`${o1.first} ${o1.mid} ${o1.last}`); -console.log(`${o2.first} ${typeof o2.mid} ${o2.last} ${sides}`); diff --git a/tests/harness/__snapshots__/child-process.ts.txt b/tests/harness/__snapshots__/child-process.ts.txt index f5f9f8f86..a1f2999b7 100644 --- a/tests/harness/__snapshots__/child-process.ts.txt +++ b/tests/harness/__snapshots__/child-process.ts.txt @@ -14,7 +14,7 @@ child-process.ts:27:14 - error SC1090: listeners returning a value (make the cal | ^~~~~~~~~~~~~~~ 28 | // Methods have no bound-value form — call on directly. -child-process.ts:29:7 - error SC2007: values of type '{ (event: "spawn", listener: () => void): void; (event: "exit", listener: (code: number | null, signal: string | null) => void): void; (event: "close", listener: (code: number | null, signal: string | null) => void): void; (event: "error", listener: (err: Error) => void): void; (event: "message", listener: (message:...' cannot be compiled: the type declares multiple call signatures (overloads), and a compiled function value is always one concrete signature +child-process.ts:29:7 - error SC2007: values of type '{ (event: "spawn", listener: () => void): ChildProcess; (event: "exit", listener: (code: number | null, signal: string | null) => void): ChildProcess; (event: "close", listener: (code: number | null, signal: string | null) => void): ChildProcess; (event: "error", listener: (err: Error) => void): ChildProcess; (event...' cannot be compiled: the type declares multiple call signatures (overloads), and a compiled function value is always one concrete signature 28 | // Methods have no bound-value form — call on directly. 29 | const f = c.on; diff --git a/tests/harness/__snapshots__/destructuring-fences.ts.txt b/tests/harness/__snapshots__/destructuring-fences.ts.txt index 743bb5657..2f6afe734 100644 --- a/tests/harness/__snapshots__/destructuring-fences.ts.txt +++ b/tests/harness/__snapshots__/destructuring-fences.ts.txt @@ -19,18 +19,18 @@ destructuring-fences.ts:31:9 - error SC1090: reading a property that has only a | ^~~~~~ 32 | console.log(onlyIn); -destructuring-fences.ts:45:9 - error SC1031: object destructuring of non-record values (the source is union-typed) are not supported yet +destructuring-fences.ts:45:11 - error SC1031: destructuring 'b' from a '{ b: string; count: number; kind: string } | { b: number; kind: string; label: string }' source (every arm must be a record or class with a same-typed field 'b' — narrow first: check a discriminant field, or compare with '!== undefined'/'!== null' for unit arms) is not supported yet 44 | function fromUnion(u: E1 | E2): string { - 45 | const { b } = u; // union source: narrow first - | ^~~~~ - 46 | return b; + 45 | const { b } = u; // union source whose arms type the field differently: narrow first + | ^ + 46 | return String(b); -destructuring-fences.ts:46:10 - error SC2004: uses of 'b' inherit the blocker on its declaration +destructuring-fences.ts:46:17 - error SC2004: uses of 'b' inherit the blocker on its declaration - 45 | const { b } = u; // union source: narrow first - 46 | return b; - | ^ + 45 | const { b } = u; // union source whose arms type the field differently: narrow first + 46 | return String(b); + | ^ 47 | } hint: the declaration of 'b' did not compile — fix the diagnostic reported there and these sites clear with it diff --git a/tests/harness/__snapshots__/retired-engine-1123-any-conditional-spread.ts.txt b/tests/harness/__snapshots__/retired-engine-1123-any-conditional-spread.ts.txt deleted file mode 100644 index 707d1dfd9..000000000 --- a/tests/harness/__snapshots__/retired-engine-1123-any-conditional-spread.ts.txt +++ /dev/null @@ -1,6 +0,0 @@ -retired-engine-1123-any-conditional-spread.ts:21:21 - error SC1090: conditional spreads beyond `...(c ? { field: v } : {})` (exactly one property against an empty arm — spell other shapes as optional fields) is not supported yet - - 20 | function multi(on: boolean): any { - 21 | return { base: 1, ...(on ? { x: "X", y: "Y" } : {}) }; - | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - 22 | } \ No newline at end of file diff --git a/tests/harness/__snapshots__/spread-fences.ts.txt b/tests/harness/__snapshots__/spread-fences.ts.txt index 56f8b81ee..eaced4979 100644 --- a/tests/harness/__snapshots__/spread-fences.ts.txt +++ b/tests/harness/__snapshots__/spread-fences.ts.txt @@ -1,4 +1,4 @@ -spread-fences.ts:14:21 - error SC1090: conditional spreads beyond `...(c ? { field: v } : {})` (exactly one property against an empty arm — spell other shapes as optional fields) is not supported yet +spread-fences.ts:14:21 - error SC1090: conditional spreads beyond `...(c ? { field: v } : {})` and `...(c ? record : {})` (one arm must be empty — spell other shapes as optional fields) is not supported yet 13 | const cond = true; 14 | const s1: Shape = { ...(cond ? { x: 1, y: 2 } : { x: 3, y: 4 }) }; diff --git a/tests/harness/corpus-costs/7020-conditional-spread-fields.ts.json b/tests/harness/corpus-costs/7020-conditional-spread-fields.ts.json new file mode 100644 index 000000000..ee564e79f --- /dev/null +++ b/tests/harness/corpus-costs/7020-conditional-spread-fields.ts.json @@ -0,0 +1 @@ +{ "plain": 3640, "san": 5580 } diff --git a/tests/harness/corpus-costs/7021-promise-all-uniform-tuple.ts.json b/tests/harness/corpus-costs/7021-promise-all-uniform-tuple.ts.json new file mode 100644 index 000000000..fba6fd530 --- /dev/null +++ b/tests/harness/corpus-costs/7021-promise-all-uniform-tuple.ts.json @@ -0,0 +1 @@ +{ "plain": 2510, "san": 3630 } diff --git a/tests/harness/corpus-costs/7022-promise-all-dynamic-arrays.ts.json b/tests/harness/corpus-costs/7022-promise-all-dynamic-arrays.ts.json new file mode 100644 index 000000000..c59bb483b --- /dev/null +++ b/tests/harness/corpus-costs/7022-promise-all-dynamic-arrays.ts.json @@ -0,0 +1 @@ +{ "plain": 3330, "san": 4690 } diff --git a/tests/harness/corpus-costs/7023-union-object-patterns.ts.json b/tests/harness/corpus-costs/7023-union-object-patterns.ts.json new file mode 100644 index 000000000..3f0a047e7 --- /dev/null +++ b/tests/harness/corpus-costs/7023-union-object-patterns.ts.json @@ -0,0 +1 @@ +{ "plain": 2440, "san": 3670 } diff --git a/tests/harness/corpus-costs/7024-async-return-fresh-collections.ts.json b/tests/harness/corpus-costs/7024-async-return-fresh-collections.ts.json new file mode 100644 index 000000000..6285a25c5 --- /dev/null +++ b/tests/harness/corpus-costs/7024-async-return-fresh-collections.ts.json @@ -0,0 +1 @@ +{ "plain": 2570, "san": 4360 } diff --git a/tests/harness/corpus-costs/7025-child-listener-chaining.ts.json b/tests/harness/corpus-costs/7025-child-listener-chaining.ts.json new file mode 100644 index 000000000..44843dc06 --- /dev/null +++ b/tests/harness/corpus-costs/7025-child-listener-chaining.ts.json @@ -0,0 +1 @@ +{ "plain": 2510, "san": 3950 } diff --git a/tests/harness/corpus-costs/7026-url-object-patterns.ts.json b/tests/harness/corpus-costs/7026-url-object-patterns.ts.json new file mode 100644 index 000000000..9d3f28ab1 --- /dev/null +++ b/tests/harness/corpus-costs/7026-url-object-patterns.ts.json @@ -0,0 +1 @@ +{ "plain": 2520, "san": 4000 } diff --git a/tests/harness/corpus-costs/7027-search-params-conditional-init.ts.json b/tests/harness/corpus-costs/7027-search-params-conditional-init.ts.json new file mode 100644 index 000000000..56c46a74b --- /dev/null +++ b/tests/harness/corpus-costs/7027-search-params-conditional-init.ts.json @@ -0,0 +1 @@ +{ "plain": 2240, "san": 3440 } diff --git a/tests/harness/corpus-costs/7028-non-empty-rest-tuples.ts.json b/tests/harness/corpus-costs/7028-non-empty-rest-tuples.ts.json new file mode 100644 index 000000000..147f70b9a --- /dev/null +++ b/tests/harness/corpus-costs/7028-non-empty-rest-tuples.ts.json @@ -0,0 +1 @@ +{ "plain": 2520, "san": 4030 } diff --git a/tests/harness/corpus-costs/7029-flatmap-union-results.ts.json b/tests/harness/corpus-costs/7029-flatmap-union-results.ts.json new file mode 100644 index 000000000..7efdd91bf --- /dev/null +++ b/tests/harness/corpus-costs/7029-flatmap-union-results.ts.json @@ -0,0 +1 @@ +{ "plain": 2640, "san": 4710 } diff --git a/tests/harness/corpus-costs/7033-any-fields-into-declared-records.ts.json b/tests/harness/corpus-costs/7033-any-fields-into-declared-records.ts.json new file mode 100644 index 000000000..1fd3ee053 --- /dev/null +++ b/tests/harness/corpus-costs/7033-any-fields-into-declared-records.ts.json @@ -0,0 +1 @@ +{ "plain": 2640, "san": 3950 } diff --git a/tests/harness/corpus-costs/7034-optional-field-union-routes.ts.json b/tests/harness/corpus-costs/7034-optional-field-union-routes.ts.json new file mode 100644 index 000000000..7e342cfd0 --- /dev/null +++ b/tests/harness/corpus-costs/7034-optional-field-union-routes.ts.json @@ -0,0 +1 @@ +{ "plain": 2420, "san": 3640 } From 712b4e84a1e52dca58f559a20123f935e68e54c5 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 08:12:12 -0500 Subject: [PATCH 027/197] Support the env option of spawnSync `spawnSync(cmd, args, { env })` replaces the child's environment with the given record, as spawn and execFileSync already do, alone or with the timeout, killSignal and stdio options. The fallback typings declare the option. --- .../ambient/scriptc-node-fallback.d.ts | 4 +- .../compiler/src/backend/llvm/lib-shared.ts | 1 + .../src/backend/llvm/reference-writes.ts | 1 + .../lowering/builtins/child-process.ts | 19 +++++++++- .../compiler/src/ir/builtin-signatures.ts | 6 +++ .../tests/corpus/7030-spawn-sync-env.ts.json | 6 +++ packages/runtime/src/scr_child.c | 38 +++++++++++++++---- packages/runtime/src/scr_child.h | 5 +++ tests/corpus/7030-spawn-sync-env.ts | 28 ++++++++++++++ .../corpus-costs/7030-spawn-sync-env.ts.json | 1 + 10 files changed, 98 insertions(+), 11 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7030-spawn-sync-env.ts.json create mode 100644 tests/corpus/7030-spawn-sync-env.ts create mode 100644 tests/harness/corpus-costs/7030-spawn-sync-env.ts.json diff --git a/packages/compiler/ambient/scriptc-node-fallback.d.ts b/packages/compiler/ambient/scriptc-node-fallback.d.ts index 2a3f11be9..c5fce2423 100644 --- a/packages/compiler/ambient/scriptc-node-fallback.d.ts +++ b/packages/compiler/ambient/scriptc-node-fallback.d.ts @@ -3512,7 +3512,8 @@ declare module "child_process" { * killSignal at the deadline and the result carries error: ETIMEDOUT + * the signal, never a throw), killSignal (a signal NAME literal), * stdio ("ignore"/"inherit"/"pipe" or a 3-tuple of those — captures - * read "" for non-piped streams), and windowsHide (a POSIX no-op). */ + * read "" for non-piped streams), env (replaces the child's + * environment), and windowsHide (a POSIX no-op). */ export function spawnSync( command: string, args?: string[], @@ -3521,6 +3522,7 @@ declare module "child_process" { timeout?: number; killSignal?: string; stdio?: "pipe" | "ignore" | "inherit" | ("pipe" | "ignore" | "inherit")[]; + env?: { [k: string]: string | undefined }; windowsHide?: boolean; }, ): SpawnSyncReturns; diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index 9a65d9a24..f18e4a7d5 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -754,6 +754,7 @@ export const LIB_FN_SYMS: Record = { "cp.spawnSync": "scr_spawn_sync", "cp.spawnSyncOpts": "scr_spawn_sync_opts", "cp.spawnSyncStdioStr": "scr_spawn_sync_stdio_str", + "cp.spawnSyncEnv": "scr_spawn_sync_env", "cp.execSync": "scr_exec_sync", "cp.execCapture": "scr_exec_capture", "spawnRes.stdout": "scr_spawn_res_stdout", diff --git a/packages/compiler/src/backend/llvm/reference-writes.ts b/packages/compiler/src/backend/llvm/reference-writes.ts index d31f14d12..6d1593a55 100644 --- a/packages/compiler/src/backend/llvm/reference-writes.ts +++ b/packages/compiler/src/backend/llvm/reference-writes.ts @@ -118,6 +118,7 @@ const INERT_LIB_CALLS = new Set([ "worker.parentPort", "worker.root", "cp.spawnSyncOpts", + "cp.spawnSyncEnv", ]); /** jsval conversions (ir/jsval.ts): no user code, no stores. */ diff --git a/packages/compiler/src/frontend/lowering/builtins/child-process.ts b/packages/compiler/src/frontend/lowering/builtins/child-process.ts index ea2dd379d..9096316a3 100644 --- a/packages/compiler/src/frontend/lowering/builtins/child-process.ts +++ b/packages/compiler/src/frontend/lowering/builtins/child-process.ts @@ -115,6 +115,8 @@ export function lowerSpawnSyncCall(lowerer: Lowerer, expr: ts.CallExpression, lo // "pipe"`) — accepted when its TYPE proves every arm is a supported // literal; the runtime maps the value to the modes at the call. let stdioStr: IrExpr | null = null; + // The `env` option's [k,v,...] pairs, which replace the child's environment. + let envPairs: IrExpr | null = null; let plain = true; // no behavior-changing option: the historical libCall if (optsNode) { @@ -227,11 +229,16 @@ export function lowerSpawnSyncCall(lowerer: Lowerer, expr: ts.CallExpression, lo case "windowsHide": lowerer.lowerExpr(m.value); // Node no-op on POSIX break; + case "env": + // A REPLACEMENT environment, as for spawn and execFileSync. + envPairs = recordToEnvPairs(lowerer, m.value); + plain = false; + break; default: lowerer.noLowering( `spawnSync option '${m.name}'`, p, - "encoding, timeout, killSignal, stdio, and windowsHide are the supported options", + "encoding, timeout, killSignal, stdio, env, and windowsHide are the supported options", ); } } @@ -239,6 +246,13 @@ export function lowerSpawnSyncCall(lowerer: Lowerer, expr: ts.CallExpression, lo if (plain) { return { kind: "libCall", fn: "cp.spawnSync", args: [cmd, argv], type: SPAWNRES_T, loc }; } + if (stdioStr !== null && envPairs !== null) { + lowerer.noLowering( + "spawnSync with a runtime stdio value and env", + optsNode!, + 'pass stdio as a "pipe"/"ignore"/"inherit" literal when env is set', + ); + } if (stdioStr !== null) { return { kind: "libCall", @@ -250,7 +264,7 @@ export function lowerSpawnSyncCall(lowerer: Lowerer, expr: ts.CallExpression, lo } return { kind: "libCall", - fn: "cp.spawnSyncOpts", + fn: envPairs !== null ? "cp.spawnSyncEnv" : "cp.spawnSyncOpts", args: [ cmd, argv, @@ -259,6 +273,7 @@ export function lowerSpawnSyncCall(lowerer: Lowerer, expr: ts.CallExpression, lo numLit(inMode, loc), numLit(outMode, loc), numLit(errMode, loc), + ...(envPairs !== null ? [envPairs] : []), ], type: SPAWNRES_T, loc, diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index fcdbe8694..6b7dc1741 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -1318,6 +1318,12 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ argTypes: [STRING, arrayOf(STRING), F64, STRING, F64, F64, F64], result: SPAWNRES_T, }, + /** cp.spawnSyncOpts with the `env` option: the trailing [k,v,...] + * pairs replace the child's environment (recordToEnvPairs). */ + "cp.spawnSyncEnv": { + argTypes: [STRING, arrayOf(STRING), F64, STRING, F64, F64, F64, arrayOf(STRING)], + result: SPAWNRES_T, + }, /** cp.spawnSyncOpts with the stdio carried as a RUNTIME string — * "pipe" | "ignore" | "inherit", proven by the call site's TYPE (the * defaultRunner idiom `stdio: options?.stdio ?? "pipe"`); the runtime diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7030-spawn-sync-env.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7030-spawn-sync-env.ts.json new file mode 100644 index 000000000..f8afbe00b --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7030-spawn-sync-env.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7030-spawn-sync-env.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_child.c b/packages/runtime/src/scr_child.c index ec685656d..387885c42 100644 --- a/packages/runtime/src/scr_child.c +++ b/packages/runtime/src/scr_child.c @@ -1105,9 +1105,9 @@ char **scr_child_argv(ScrStr *cmd, ScrArr *args) { static ScrSpawnRes *scr_spawn_sync_core(ScrStr *cmd, ScrArr *args, double timeout_ms, int killsig, int in_mode, int out_mode, - int err_mode) { + int err_mode, ScrArr *env_pairs) { ScrWinSyncRes w; - scr_win_run_sync(cmd, args, NULL, in_mode, out_mode, err_mode, NULL, NULL, + scr_win_run_sync(cmd, args, NULL, in_mode, out_mode, err_mode, NULL, env_pairs, timeout_ms, &w); if (w.spawn_errname != NULL) { free(w.out.data); @@ -1136,19 +1136,26 @@ static ScrSpawnRes *scr_spawn_sync_core(ScrStr *cmd, ScrArr *args, double timeou } ScrSpawnRes *scr_spawn_sync(ScrStr *cmd, ScrArr *args) { - return scr_spawn_sync_core(cmd, args, 0, SIGTERM, 0, 0, 0); + return scr_spawn_sync_core(cmd, args, 0, SIGTERM, 0, 0, 0, NULL); } ScrSpawnRes *scr_spawn_sync_opts(ScrStr *cmd, ScrArr *args, double timeout_ms, ScrStr *killsignal, double in_mode, double out_mode, double err_mode) { + return scr_spawn_sync_env(cmd, args, timeout_ms, killsignal, in_mode, out_mode, err_mode, + NULL); +} + +ScrSpawnRes *scr_spawn_sync_env(ScrStr *cmd, ScrArr *args, double timeout_ms, + ScrStr *killsignal, double in_mode, double out_mode, + double err_mode, ScrArr *env_pairs) { int killsig = SIGTERM; if (killsignal->len > 0) { int resolved = scr_signal_from_name(killsignal); if (resolved > 0) killsig = resolved; } return scr_spawn_sync_core(cmd, args, timeout_ms, killsig, (int)in_mode, - (int)out_mode, (int)err_mode); + (int)out_mode, (int)err_mode, env_pairs); } ScrSpawnRes *scr_spawn_sync_stdio_str(ScrStr *cmd, ScrArr *args, double timeout_ms, @@ -3101,9 +3108,14 @@ static ScrSpawnRes *scr_spawn_res_new(bool has_status, double status, * documented divergence). timeout_ms > 0 sends `killsig` at the deadline * and the result carries error: ETIMEDOUT + the signal — never a throw, * Node's spawnSync shape. */ +static char **scr_exec_envp(ScrArr *pairs); +static void scr_exec_envp_free(char **envp); + +/* env_pairs: NULL inherits the parent's environment; else [k,v,...] + * replaces it, as Node's `env` option does. */ static ScrSpawnRes *scr_spawn_sync_core(ScrStr *cmd, ScrArr *args, double timeout_ms, int killsig, int in_mode, int out_mode, - int err_mode) { + int err_mode, ScrArr *env_pairs) { if (out_mode == 2 || err_mode == 2) { /* Inherited output fds: flush the parent's buffers first so earlier * logs precede the child's writes (Node's practical ordering). */ @@ -3138,10 +3150,13 @@ static ScrSpawnRes *scr_spawn_sync_core(ScrStr *cmd, ScrArr *args, double timeou } char **argv = scr_child_argv(cmd, args); + char **envp = env_pairs != NULL ? scr_exec_envp(env_pairs) : NULL; pid_t pid = -1; - int spawn_err = posix_spawnp(&pid, cmd->data, &fa, NULL, argv, environ); + int spawn_err = posix_spawnp(&pid, cmd->data, &fa, NULL, argv, + envp != NULL ? envp : environ); posix_spawn_file_actions_destroy(&fa); free(argv); + scr_exec_envp_free(envp); if (cap_out) close(outfd[1]); if (cap_err) close(errfd[1]); @@ -3242,7 +3257,7 @@ static ScrSpawnRes *scr_spawn_sync_core(ScrStr *cmd, ScrArr *args, double timeou } ScrSpawnRes *scr_spawn_sync(ScrStr *cmd, ScrArr *args) { - return scr_spawn_sync_core(cmd, args, 0, SIGTERM, 0, 0, 0); + return scr_spawn_sync_core(cmd, args, 0, SIGTERM, 0, 0, 0, NULL); } /* The options entry (cp.spawnSyncOpts): killSignal as a NAME ("" = the @@ -3252,13 +3267,20 @@ ScrSpawnRes *scr_spawn_sync(ScrStr *cmd, ScrArr *args) { ScrSpawnRes *scr_spawn_sync_opts(ScrStr *cmd, ScrArr *args, double timeout_ms, ScrStr *killsignal, double in_mode, double out_mode, double err_mode) { + return scr_spawn_sync_env(cmd, args, timeout_ms, killsignal, in_mode, out_mode, err_mode, + NULL); +} + +ScrSpawnRes *scr_spawn_sync_env(ScrStr *cmd, ScrArr *args, double timeout_ms, + ScrStr *killsignal, double in_mode, double out_mode, + double err_mode, ScrArr *env_pairs) { int killsig = SIGTERM; if (killsignal->len > 0) { int resolved = scr_signal_from_name(killsignal); if (resolved > 0) killsig = resolved; } return scr_spawn_sync_core(cmd, args, timeout_ms, killsig, (int)in_mode, - (int)out_mode, (int)err_mode); + (int)out_mode, (int)err_mode, env_pairs); } /* The runtime-string stdio entry (cp.spawnSyncStdioStr): the compiler diff --git a/packages/runtime/src/scr_child.h b/packages/runtime/src/scr_child.h index e899130d8..a5853ad59 100644 --- a/packages/runtime/src/scr_child.h +++ b/packages/runtime/src/scr_child.h @@ -23,6 +23,11 @@ ScrSpawnRes *scr_spawn_sync(ScrStr *cmd, ScrArr *args); /* +1, never throws */ ScrSpawnRes *scr_spawn_sync_opts(ScrStr *cmd, ScrArr *args, double timeout_ms, ScrStr *killsignal, double in_mode, double out_mode, double err_mode); +/* scr_spawn_sync_opts with an `env` option: [k,v,...] pairs replace the + * child's environment (NULL inherits the parent's). */ +ScrSpawnRes *scr_spawn_sync_env(ScrStr *cmd, ScrArr *args, double timeout_ms, + ScrStr *killsignal, double in_mode, double out_mode, + double err_mode, ScrArr *env_pairs); /* The runtime-string stdio entry: the compiler proved the value is * "pipe" | "ignore" | "inherit" by type; the mode mapping happens here. */ ScrSpawnRes *scr_spawn_sync_stdio_str(ScrStr *cmd, ScrArr *args, double timeout_ms, diff --git a/tests/corpus/7030-spawn-sync-env.ts b/tests/corpus/7030-spawn-sync-env.ts new file mode 100644 index 000000000..e613c81f0 --- /dev/null +++ b/tests/corpus/7030-spawn-sync-env.ts @@ -0,0 +1,28 @@ +// spawnSync's env option replaces the child's environment, alone or with +// the other options. +import { spawnSync } from "node:child_process"; + +const merged = spawnSync("sh", ["-c", 'echo "$GREETING $HOME_SET"'], { + env: { ...process.env, GREETING: "hello", HOME_SET: process.env.HOME ? "yes" : "no" }, + encoding: "utf8", +}); +console.log(merged.stdout.trim(), merged.status); + +const only = spawnSync("/bin/sh", ["-c", 'echo "${GREETING:-unset} ${HOME:-nohome}"'], { + env: { GREETING: "only" }, + encoding: "utf8", +}); +console.log(only.stdout.trim()); + +const empty = spawnSync("/bin/sh", ["-c", "env | grep -c ."], { env: {}, encoding: "utf8" }); +console.log(Number(empty.stdout.trim()) <= 3, empty.status); + +const quiet = spawnSync("sh", ["-c", 'echo "$MODE" >&2; exit 4'], { + env: { PATH: process.env.PATH ?? "/usr/bin:/bin", MODE: "loud" }, + stdio: ["ignore", "pipe", "pipe"], + timeout: 5000, +}); +console.log(quiet.status, quiet.stderr.toString().trim(), quiet.stdout.toString() === ""); + +const missing = spawnSync("/nonexistent/tool", [], { env: { A: "1" } }); +console.log(missing.status, missing.error?.message); diff --git a/tests/harness/corpus-costs/7030-spawn-sync-env.ts.json b/tests/harness/corpus-costs/7030-spawn-sync-env.ts.json new file mode 100644 index 000000000..cf8ef4d26 --- /dev/null +++ b/tests/harness/corpus-costs/7030-spawn-sync-env.ts.json @@ -0,0 +1 @@ +{ "plain": 2460, "san": 4060 } From 7259f10ad47c79d5f201502eb88208990ad82d08 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 08:16:36 -0500 Subject: [PATCH 028/197] Keep running maximum and minimum updates as branches With a running maximum in a register, LLVM turns `if (v > max) { max = v; at = i; }` into selects, so each element waits on the previous one's `maxsd`. The update gets rarer as a scan goes on and its branch predicts well, so such an `if` now carries branch weights that say it is rarely taken, and LLVM keeps the branch. An `if` qualifies when it has no `else`, its condition is one numeric `<`, `<=`, `>` or `>=` between a local and another expression, and it assigns that expression to the local. The local must be an accumulator: inside a loop the `if` is a statement of the innermost loop's body, the local is declared outside that loop, and the loop writes it only in this `if`; outside a loop the `if` is a statement of the function body and the local is a binding the function captures and writes only there, as in a callback. A clamp of a value read in the same iteration, or a sum clamped after each addition, is left alone: its branch goes whichever way the data goes. So is an update nested in another branch, whose kept branch can stop LLVM from turning the enclosing, data-dependent branch into selects. --- packages/compiler/src/backend/llvm/blocks.ts | 4 +- .../src/backend/llvm/emit-function.ts | 2 + .../src/backend/llvm/emit-statements.ts | 7 +- packages/compiler/src/backend/llvm/emitter.ts | 3 + .../src/backend/llvm/running-extrema.test.ts | 97 ++++++++++ .../src/backend/llvm/running-extrema.ts | 110 ++++++++++++ .../test/running-extrema-branches.test.ts | 75 ++++++++ .../7301-running-extrema-branches.ts.json | 6 + tests/corpus/7301-running-extrema-branches.ts | 169 ++++++++++++++++++ .../7301-running-extrema-branches.ts.json | 1 + 10 files changed, 471 insertions(+), 3 deletions(-) create mode 100644 packages/compiler/src/backend/llvm/running-extrema.test.ts create mode 100644 packages/compiler/src/backend/llvm/running-extrema.ts create mode 100644 packages/compiler/test/running-extrema-branches.test.ts create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7301-running-extrema-branches.ts.json create mode 100644 tests/corpus/7301-running-extrema-branches.ts create mode 100644 tests/harness/corpus-costs/7301-running-extrema-branches.ts.json diff --git a/packages/compiler/src/backend/llvm/blocks.ts b/packages/compiler/src/backend/llvm/blocks.ts index 6d22bf3bf..354e631c6 100644 --- a/packages/compiler/src/backend/llvm/blocks.ts +++ b/packages/compiler/src/backend/llvm/blocks.ts @@ -95,8 +95,8 @@ export class BlockBuilder { this.terminate(`br label %${label}`); } - condBr(v: string, t: string, f: string): void { - this.terminate(`br i1 ${v}, label %${t}, label %${f}`); + condBr(v: string, t: string, f: string, attachment = ""): void { + this.terminate(`br i1 ${v}, label %${t}, label %${f}${attachment}`); } /** Emit a double-indexed `for (i = 0; i < len; i++)` loop. `next` is diff --git a/packages/compiler/src/backend/llvm/emit-function.ts b/packages/compiler/src/backend/llvm/emit-function.ts index 09e6551c3..3aa9a28a7 100644 --- a/packages/compiler/src/backend/llvm/emit-function.ts +++ b/packages/compiler/src/backend/llvm/emit-function.ts @@ -55,6 +55,7 @@ import { INT32_ABI_SUFFIX, SCALAR_UNION_RESULT } from "./int32-abi.js"; import { f64Lit } from "./common.js"; import { RESULT_ENTRY_PREFIX } from "./record-results.js"; import { emitIntegerClone, emitIntegerDispatcher } from "./integer-clones.js"; +import { runningExtremumUpdates } from "./running-extrema.js"; export function emitFunction(emitter: LlEmitter, fn: IrFunction): string { const B = new BlockBuilder(); @@ -81,6 +82,7 @@ export function emitFunction(emitter: LlEmitter, fn: IrFunction): string { emitter.currentLocals.set(id, plain); emitter.lazyCaptureBoxes.set(id, ""); } + emitter.runningExtrema = emitter.debug === null ? runningExtremumUpdates(fn) : new Set(); emitter.currentConstantCallbacks = emitter.debug === null ? (emitter.constantCallbacks.get(fn.name) ?? new Map()) : new Map(); // Preserve concrete source bindings for debugger inspection. Suspended diff --git a/packages/compiler/src/backend/llvm/emit-statements.ts b/packages/compiler/src/backend/llvm/emit-statements.ts index 7239db7d8..0227776ab 100644 --- a/packages/compiler/src/backend/llvm/emit-statements.ts +++ b/packages/compiler/src/backend/llvm/emit-statements.ts @@ -782,7 +782,12 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { const trueLabel = B.newLabel("if.t"); const joinLabel = B.newLabel("if.j"); const falseLabel = s.else_ ? B.newLabel("if.f") : joinLabel; - B.condBr(cond, trueLabel, falseLabel); + // A running maximum or minimum is rarely updated; the weights keep + // the branch, where a select would make each element wait on the last. + const weights = emitter.runningExtrema.has(s) + ? `, !prof !{!"branch_weights", i32 1, i32 1000}` + : ""; + B.condBr(cond, trueLabel, falseLabel, weights); B.startBlock(trueLabel); emitter.emitBlock(s.then); B.br(joinLabel); diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index ed1785912..ed1029b85 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -831,6 +831,9 @@ export class LlEmitter { readonly callLifetimes: CallLifetimes; readonly constantCallbacks: Map>; currentConstantCallbacks: ReadonlyMap = new Map(); + /** The current function's running maximum and minimum updates, whose + * branches are weighted as rarely taken (running-extrema.ts). */ + runningExtrema: ReadonlySet = new Set(); readonly stackCallbacks: StackCallbacks; readonly receiverReaders: AmbientReceiverReaders; readonly stackCaptures: StackCaptures; diff --git a/packages/compiler/src/backend/llvm/running-extrema.test.ts b/packages/compiler/src/backend/llvm/running-extrema.test.ts new file mode 100644 index 000000000..4ce307c5c --- /dev/null +++ b/packages/compiler/src/backend/llvm/running-extrema.test.ts @@ -0,0 +1,97 @@ +import { expect, test } from "vitest"; +import { BOOL, F64, type IrExpr, type IrFunction, type IrStmt } from "../../ir/ir.js"; +import { runningExtremumUpdates } from "./running-extrema.js"; + +const loc = { file: "extrema.ts", start: 0, end: 0 }; +const ref = (localId: string): IrExpr => ({ kind: "varRef", localId, type: F64, loc }); +const num = (value: number): IrExpr => ({ kind: "numLit", value, type: F64, loc }); +const assign = (localId: string, value: IrExpr): IrStmt => ({ + kind: "assign", + localId, + value, + loc, +}); +const decl = (localId: string): IrStmt => ({ kind: "varDecl", localId, init: num(0), loc }); +const compare = (op: "<" | ">", left: IrExpr, right: IrExpr): IrExpr => ({ + kind: "bin", + op, + left, + right, + type: BOOL, + loc, +}); +/** `if (v > max) max = v;` */ +const update = (): IrStmt & { kind: "if" } => ({ + kind: "if", + cond: compare(">", ref("v"), ref("max")), + then: [assign("max", ref("v"))], + else_: null, + loc, +}); +const loop = (body: IrStmt[]): IrStmt => ({ + kind: "while", + cond: compare(">", ref("n"), num(0)), + body, + loc, +}); +const fn = (body: IrStmt[], captures: string[] = []): IrFunction => ({ + name: "scan", + params: [], + locals: ["v", "max", "n"].map((id) => ({ id, name: id, type: F64, mutable: true })), + captures: captures.map((localId) => ({ localId, name: localId, type: F64 })), + returnType: F64, + body, + loc, +}); + +test("a loop's update of an accumulator declared outside it qualifies", () => { + const s = update(); + expect(runningExtremumUpdates(fn([decl("max"), loop([decl("v"), s])])).has(s)).toBe(true); +}); + +test("the local may be on either side of the comparison", () => { + const s = update(); + s.cond = compare("<", ref("max"), ref("v")); + expect(runningExtremumUpdates(fn([decl("max"), loop([s])])).has(s)).toBe(true); +}); + +test("a local declared in the loop is not an accumulator", () => { + const s = update(); + expect(runningExtremumUpdates(fn([loop([decl("max"), s])])).size).toBe(0); +}); + +test("another write in the loop disqualifies the update", () => { + const s = update(); + const body = [decl("max"), loop([assign("max", num(1)), s])]; + expect(runningExtremumUpdates(fn(body)).size).toBe(0); +}); + +test("the assigned value must be the compared expression", () => { + const s = update(); + s.then = [assign("max", num(5))]; + expect(runningExtremumUpdates(fn([decl("max"), loop([s])])).size).toBe(0); +}); + +test("outside a loop only a captured binding qualifies", () => { + const captured = update(); + expect(runningExtremumUpdates(fn([captured], ["max"])).has(captured)).toBe(true); + expect(runningExtremumUpdates(fn([update()])).size).toBe(0); +}); + +test("an else branch disqualifies the update", () => { + const s = update(); + s.else_ = [assign("v", num(0))]; + expect(runningExtremumUpdates(fn([decl("max"), loop([s])])).size).toBe(0); +}); + +test("an update nested in another branch does not qualify", () => { + const s = update(); + const guard: IrStmt = { + kind: "if", + cond: compare(">", ref("v"), num(0)), + then: [s], + else_: null, + loc, + }; + expect(runningExtremumUpdates(fn([decl("max"), loop([guard])])).size).toBe(0); +}); diff --git a/packages/compiler/src/backend/llvm/running-extrema.ts b/packages/compiler/src/backend/llvm/running-extrema.ts new file mode 100644 index 000000000..1ddea603d --- /dev/null +++ b/packages/compiler/src/backend/llvm/running-extrema.ts @@ -0,0 +1,110 @@ +// Running maximum and minimum updates, `if (v > max) { max = v; ... }`. +// With the accumulator in a register LLVM turns such an update into a +// select (`maxsd`, or a compare and mask), so every element waits on the +// previous one's. The update gets rarer as a scan goes on, so a branch +// predicts well; emitStmtBody gives these `if`s branch weights that say the +// update is rarely taken, and LLVM keeps the branch. A clamp of a value read +// in the same iteration is not an accumulator and keeps its select: its +// branch would go whichever way the data goes. An update nested in another +// branch keeps its select too: kept, its branch can stop LLVM from turning +// the enclosing branch into selects, and that branch may follow the data. + +import type { IrExpr, IrFunction, IrStmt } from "../../ir/ir.js"; +import { everyExprChild, everyStmtChild } from "../../ir/traverse.js"; + +type Loop = IrStmt & { kind: "for" | "while" | "doWhile" | "forOf" }; + +const RELATIONAL = new Set(["<", "<=", ">", ">="]); + +/** The `if` statements of `fn` that update a running maximum or minimum: + * no `else`, a numeric `<`, `<=`, `>` or + * `>=` between a local and another expression, and a then-block that + * assigns that expression to the local. The local must carry across + * iterations: inside a loop the `if` is a statement of the innermost loop's + * body, the local is declared outside that loop, and the loop writes it only + * in this `if`; outside any loop the `if` is a statement of the function + * body and the local is a binding the function captures and writes only in + * this `if` (a callback that a loop elsewhere calls). */ +export function runningExtremumUpdates(fn: IrFunction): Set { + const found = new Set(); + const captured = new Set((fn.captures ?? []).map((c) => c.localId)); + const loops: Loop[] = []; + const candidates: { s: IrStmt; local: string; loop: Loop | undefined }[] = []; + const expr = (e: IrExpr): boolean => everyExprChild(e, expr, stmt); + const stmt = (s: IrStmt): boolean => { + if (s.kind === "if") { + const local = updatedExtremum(s); + if (local !== undefined) candidates.push({ s, local, loop: loops[loops.length - 1] }); + } + const loop = + s.kind === "for" || s.kind === "while" || s.kind === "doWhile" || s.kind === "forOf"; + if (loop) loops.push(s); + everyStmtChild(s, expr, stmt); + if (loop) loops.pop(); + return true; + }; + for (const s of fn.body) stmt(s); + for (const { s, local, loop } of candidates) { + const scope = loop ?? (captured.has(local) ? fn.body : undefined); + const body = loop ? loop.body : fn.body; + if (scope !== undefined && body.includes(s) && writes(scope, local) === 1) found.add(s); + } + return found; +} + +/** The local an `if` updates to a new maximum or minimum, if it is one. */ +function updatedExtremum(s: IrStmt & { kind: "if" }): string | undefined { + if (s.else_ !== null && s.else_.length > 0) return undefined; + const c = s.cond; + if (c.kind !== "bin" || !RELATIONAL.has(c.op)) return undefined; + if (c.left.type.kind !== "f64" || c.right.type.kind !== "f64") return undefined; + for (const [local, other] of [ + [c.left, c.right], + [c.right, c.left], + ]) { + if (local!.kind !== "varRef") continue; + const id = local!.localId; + const assigns = s.then.some( + (x) => x.kind === "assign" && x.localId === id && sameExpr(x.value, other!), + ); + if (assigns) return id; + } + return undefined; +} + +/** Declarations and writes of `local` under `scope`. A declaration counts + * as a write too, so a local declared inside a loop does not qualify. */ +function writes(scope: IrStmt | IrStmt[], local: string): number { + let n = 0; + const expr = (e: IrExpr): boolean => { + if ((e.kind === "assignExpr" || e.kind === "incDec") && e.localId === local) n++; + return everyExprChild(e, expr, stmt); + }; + const stmt = (s: IrStmt): boolean => { + if ((s.kind === "assign" || s.kind === "varDecl" || s.kind === "forOf") && s.localId === local) + n++; + else if (s.kind === "tryCatch" && s.catchLocalId === local) n++; + return everyStmtChild(s, expr, stmt); + }; + if (Array.isArray(scope)) for (const s of scope) stmt(s); + else stmt(scope); + return n; +} + +/** Structural equality of two small expressions (`v`, `a[i]`), ignoring + * source locations and types; larger expressions do not compare equal. */ +function sameExpr(a: IrExpr, b: IrExpr): boolean { + let budget = 64; + const same = (x: unknown, y: unknown): boolean => { + if (--budget < 0) return false; + if (typeof x !== "object" || x === null || typeof y !== "object" || y === null) return x === y; + if (Array.isArray(x) !== Array.isArray(y)) return false; + const keys = Object.keys(x).filter((k) => k !== "loc" && k !== "type"); + if (keys.length !== Object.keys(y).filter((k) => k !== "loc" && k !== "type").length) + return false; + return keys.every((k) => + same((x as Record)[k], (y as Record)[k]), + ); + }; + return same(a, b); +} diff --git a/packages/compiler/test/running-extrema-branches.test.ts b/packages/compiler/test/running-extrema-branches.test.ts new file mode 100644 index 000000000..e1f5a7837 --- /dev/null +++ b/packages/compiler/test/running-extrema-branches.test.ts @@ -0,0 +1,75 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { compile } from "../src/index.js"; + +async function emitLlvm(source: string): Promise { + const dir = await mkdtemp(join(tmpdir(), "scriptc-running-extrema-")); + try { + const entry = join(dir, "main.ts"); + const outPath = join(dir, "main.ll"); + await writeFile(entry, source); + const result = await compile(entry, { outDir: dir, outPath, outputKind: "llvm" }); + if (!result.ok) throw new Error(result.diagnostics.map((d) => d.message).join("\n")); + return await readFile(outPath, "utf8"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +} + +const RARE = + /br i1 %\S+, label %if\.t\d+, label %if\.j\d+, !prof !\{!"branch_weights", i32 1, i32 1000\}/; + +test("a running maximum's update is weighted as rarely taken", async () => { + const llvm = await emitLlvm(` + function argmax(a: Float64Array): number { + let max = -Infinity; + let at = -1; + for (let i = 0; i < a.length; i++) { + const v = a[i]!; + if (v > max) { max = v; at = i; } + } + return at; + } + console.log(argmax(new Float64Array([1, 3, 2]))); + `); + expect(llvm).toMatch(RARE); +}); + +test("a callback's update of a captured minimum is weighted", async () => { + const llvm = await emitLlvm(` + function least(values: number[]): number { + let min = Infinity; + values.forEach((v) => { if (min > v) min = v; }); + return min; + } + console.log(least([3, 1, 2])); + `); + expect(llvm).toMatch(RARE); +}); + +test("clamps and saturating sums keep their selects", async () => { + const llvm = await emitLlvm(` + function clampSum(a: Float64Array, hi: number): number { + let s = 0; + for (let i = 0; i < a.length; i++) { + let v = a[i]!; + if (hi < v) v = hi; + s += v; + } + return s; + } + function saturate(a: Float64Array, hi: number): number { + let v = 0; + for (let i = 0; i < a.length; i++) { + v += a[i]!; + if (hi < v) v = hi; + } + return v; + } + const a = new Float64Array([0.5, 2, 0.25]); + console.log(clampSum(a, 1), saturate(a, 1)); + `); + expect(llvm).not.toContain("branch_weights"); +}); diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7301-running-extrema-branches.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7301-running-extrema-branches.ts.json new file mode 100644 index 000000000..6dcb4b57d --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7301-running-extrema-branches.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7301-running-extrema-branches.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7301-running-extrema-branches.ts b/tests/corpus/7301-running-extrema-branches.ts new file mode 100644 index 000000000..9941ce538 --- /dev/null +++ b/tests/corpus/7301-running-extrema-branches.ts @@ -0,0 +1,169 @@ +// Running maximum and minimum scans over ascending, descending, alternating, +// pseudo-random and NaN-holding data: plain for, while, do-while and for-of +// loops, `<=` and `>=` ties, an argmax that records its index, typed array +// and array element reads, callbacks that update captured bindings, a scan +// nested in an outer loop, an update nested in another branch, and the clamp +// and saturating updates that are not running extrema. + +function shapes(n: number): [string, Float64Array][] { + const ascending = new Float64Array(n); + const descending = new Float64Array(n); + const alternating = new Float64Array(n); + const random = new Float64Array(n); + const holes = new Float64Array(n); + let x = 12345; + for (let i = 0; i < n; i++) { + ascending[i] = i; + descending[i] = n - i; + alternating[i] = i % 2 === 0 ? 0 : i; + x = (x * 1103515245 + 12345) % 2147483648; + random[i] = (x % 1000) - 500; + holes[i] = i % 7 === 3 ? NaN : (x % 97) - 48; + } + return [ + ["ascending", ascending], + ["descending", descending], + ["alternating", alternating], + ["random", random], + ["nan", holes], + ]; +} + +function argmax(a: Float64Array): string { + let max = -Infinity; + let at = -1; + for (let i = 0; i < a.length; i++) { + const v = a[i]!; + if (v > max) { + max = v; + at = i; + } + } + return `${max}@${at}`; +} + +function lastMin(a: Float64Array): string { + let min = Infinity; + let at = -1; + let i = 0; + while (i < a.length) { + if (a[i]! <= min) { + min = a[i]!; + at = i; + } + i++; + } + return `${min}@${at}`; +} + +function bothEnds(values: number[]): string { + let lo = Infinity; + let hi = -Infinity; + for (const v of values) { + if (lo > v) lo = v; + if (v >= hi) hi = v; + } + return `${lo}..${hi}`; +} + +function firstDrop(a: Float64Array): number { + let floor = a[0]!; + let i = 0; + do { + if (a[i]! < floor) floor = a[i]!; + i++; + } while (i < a.length); + return floor; +} + +function callbackPeak(a: Float64Array): string { + let max = -Infinity; + let at = -1; + a.forEach((v, i) => { + if (v > max) { + max = v; + at = i; + } + }); + return `${max}@${at}`; +} + +function arrayCallbackMin(values: number[]): number { + let min = Infinity; + values.forEach((v) => { + if (v < min) min = v; + }); + return min; +} + +function rowMaxima(a: Float64Array, width: number): string { + const out: number[] = []; + for (let row = 0; row * width < a.length; row++) { + let best = -Infinity; + for (let j = row * width; j < Math.min(a.length, (row + 1) * width); j++) { + if (a[j]! > best) best = a[j]!; + } + out.push(best); + } + return out.join(","); +} + +function longestNonNegativeRun(a: Float64Array): number { + let longest = 0; + let run = 0; + for (let i = 0; i < a.length; i++) { + if (a[i]! >= 0) { + run++; + if (run > longest) longest = run; + } else run = 0; + } + return longest; +} + +function clampSum(a: Float64Array, hi: number): number { + let s = 0; + for (let i = 0; i < a.length; i++) { + let v = a[i]!; + if (hi < v) v = hi; + s += v; + } + return s; +} + +function saturate(a: Float64Array, hi: number): number { + let v = 0; + for (let i = 0; i < a.length; i++) { + v += a[i]!; + if (hi < v) v = hi; + } + return v; +} + +for (const [name, a] of shapes(40)) { + const values = Array.from(a); + console.log( + name, + argmax(a), + lastMin(a), + bothEnds(values), + firstDrop(a), + callbackPeak(a), + arrayCallbackMin(values), + ); + console.log( + " rows", + rowMaxima(a, 7), + "run", + longestNonNegativeRun(a), + "clamp", + clampSum(a, 5), + "saturate", + saturate(a, 30), + ); +} + +const signed = new Float64Array([0, -0, 0, -0]); +let top = -Infinity; +for (let i = 0; i < signed.length; i++) if (signed[i]! >= top) top = signed[i]!; +console.log("signed zero", Object.is(top, -0), argmax(signed), lastMin(signed)); +console.log("empty", argmax(new Float64Array(0)), bothEnds([]), arrayCallbackMin([])); diff --git a/tests/harness/corpus-costs/7301-running-extrema-branches.ts.json b/tests/harness/corpus-costs/7301-running-extrema-branches.ts.json new file mode 100644 index 000000000..74df88f0c --- /dev/null +++ b/tests/harness/corpus-costs/7301-running-extrema-branches.ts.json @@ -0,0 +1 @@ +{ "plain": 5300, "san": 5800 } From 14694925c7d2edfa5c681c2d623f39b0ce3494af Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 08:32:24 -0500 Subject: [PATCH 029/197] Bind a class expression's name used only in a shorthand property, and stop keeping local class prototypes alive - A named function-local class expression whose only reference to its own name is a shorthand property (`static wrap() { return { Packed }; }`) now gets its name binding. The self-reference check read the shorthand name's property symbol, so the class lowered without the binding and the property held a placeholder function instead of the class. - A function-local class's prototype takes its name from the class object's interned name instead of a registered copy. The registry kept every such prototype alive until the process exited, so evaluating a local class repeatedly kept each evaluation's prototype and the method state it captures: a program that evaluates a local class 200,000 times and prints each instance peaked at 121 MB, and now at 2 MB. Reusing a freed object clears its name. --- .../compiler/src/frontend/lowering/lower-classes.ts | 12 +++++------- packages/runtime/src/scr_dyn.h | 12 +++++++----- packages/runtime/src/scr_json.c | 13 ++++++------- .../6917-local-class-expression-self-reference.ts | 12 ++++++++++++ .../6918-local-class-expression-self-reference.mjs | 11 +++++++++++ 5 files changed, 41 insertions(+), 19 deletions(-) diff --git a/packages/compiler/src/frontend/lowering/lower-classes.ts b/packages/compiler/src/frontend/lowering/lower-classes.ts index c74969d10..f16d5ee5d 100644 --- a/packages/compiler/src/frontend/lowering/lower-classes.ts +++ b/packages/compiler/src/frontend/lowering/lower-classes.ts @@ -184,6 +184,7 @@ import { constructionEscape } from "./construction-escapes.js"; import { checkedClassConstruction, checkedClassConstructionPacked } from "./class-construction.js"; import { initializeRuntimeStatics, hasRuntimeStatics } from "./class-runtime-statics.js"; import { reflectedClassStaticMethodValue } from "./class-method-values.js"; +import { identifierReadSymbol } from "./surfaces.js"; export function returnsOnlyThis(member: ts.MethodDeclaration): boolean { const body = member.body; @@ -6035,8 +6036,9 @@ export function lowerClassExpressionInfo( } /** True when a named class expression's body refers to the class value by - * its own name (`class Box { static of() { return new Box(); } }`); type - * annotations naming it (`again(): Box`) do not count. */ + * its own name (`class Box { static of() { return new Box(); } }`, a + * shorthand property `{ Box }` included); type annotations naming it + * (`again(): Box`) do not count. */ function classExpressionNamesItself(lowerer: Lowerer, expr: ts.ClassExpression): boolean { const name = expr.name; if (!name) return false; @@ -6046,11 +6048,7 @@ function classExpressionNamesItself(lowerer: Lowerer, expr: ts.ClassExpression): for (const member of expr.members.nodes) { ts.walkPreorder(member, (n) => { if (found || ts.isTypeNode(n)) return "skip"; - if ( - ts.isIdentifier(n) && - n.text === name.text && - lowerer.checker.getSymbolAtLocation(n) === symbol - ) + if (ts.isIdentifier(n) && n.text === name.text && identifierReadSymbol(lowerer, n) === symbol) found = true; return undefined; }); diff --git a/packages/runtime/src/scr_dyn.h b/packages/runtime/src/scr_dyn.h index 8a29e7c5d..7c62aa05e 100644 --- a/packages/runtime/src/scr_dyn.h +++ b/packages/runtime/src/scr_dyn.h @@ -370,11 +370,13 @@ struct ScrDyn { void *source_identity; ScrDyn *(*source_access)(void *, bool materialize); struct ScrDynObjectIndex *index; /* optional wide-object lookup; owns no values */ - /* On a runtime-built class prototype (e.g. url.Url.prototype): the - * constructor name (static text) util.inspect prints for objects - * inheriting from it (scr_dyn_class_name). NULL otherwise. Only - * prototypes that live until their thread exits set it, and they - * clear it before release, so freelist reuse never sees it. */ + /* On a runtime-built class prototype (e.g. url.Url.prototype) and + * on a compiled class's prototype object: the constructor name + * (static or immortal text) util.inspect prints for objects + * inheriting from it (scr_dyn_class_name). NULL otherwise. Names + * that are copies live on prototypes that live until their thread + * exits, and those clear it before release; reuse of a freed object + * resets it (scr_dyn_alloc). */ const char *class_name; struct ScrDynRegion *region; /* see ScrDynRegion */ /* The dynamic tier's hidden class (scr_js_object.c): the transition diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index 6986330f5..007f8a766 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -1021,6 +1021,7 @@ static ScrDyn *scr_dyn_alloc(ScrDynKind kind) { } else if (kind == SCR_DYN_OBJ) { d->v.obj.len = 0; /* cap/entries preserved */ d->v.obj.region = NULL; + d->v.obj.class_name = NULL; /* a released class prototype's name */ SCR_JS_SHAPE_RESET(d); if (d->v.obj.cap <= SCR_DYN_KEY_CACHE_SLOTS) { for (size_t i = 0; i < d->v.obj.cap; i++) { @@ -4185,13 +4186,11 @@ ScrDyn *scr_dyn_class_prototype(ScrDyn *value, ScrDyn *base) { if (!cls->prototype_data) { ScrDyn *data = base->kind == SCR_DYN_UNDEF ? scr_dyn_new_obj() : scr_dyn_obj_create(base); if (!data) return NULL; - /* util.inspect prints an instance with its class's name. */ - const char *name = value->v.fn.name; - if (name && name[0]) { - ScrStr *class_name = scr_str_new(name, strlen(name)); - scr_dyn_name_class_prototype(data, class_name); - scr_str_release(class_name); - } + /* util.inspect prints an instance with its class's name: the class + * object's interned name, which is immortal. The prototype is owned by + * the class object and no registry keeps it alive, so evaluating a + * function-local class again releases the previous one's prototype. */ + if (cls->name && cls->name->len) data->v.obj.class_name = cls->name->data; cls->prototype_data = scr_box_new_obj(scr_dyn_retain_v, scr_dyn_release_v, scr_dyn_trace_v); scr_box_set_ref(cls->prototype_data, data); } diff --git a/tests/corpus/6917-local-class-expression-self-reference.ts b/tests/corpus/6917-local-class-expression-self-reference.ts index f11746eda..cff612d29 100644 --- a/tests/corpus/6917-local-class-expression-self-reference.ts +++ b/tests/corpus/6917-local-class-expression-self-reference.ts @@ -39,3 +39,15 @@ function nested(n: number) { return Outer.build(n).size(); } console.log(nested(3)); + +// The only reference to the name is a shorthand property. +function registry(tag: string) { + return class Entry { + readonly tag = tag; + static table(): Record { + return { Entry }; + } + }; +} +const Entry1 = registry("e"); +console.log(Entry1.table().Entry === Entry1, new Entry1().tag); diff --git a/tests/corpus/6918-local-class-expression-self-reference.mjs b/tests/corpus/6918-local-class-expression-self-reference.mjs index cfc8de806..4b2440674 100644 --- a/tests/corpus/6918-local-class-expression-self-reference.mjs +++ b/tests/corpus/6918-local-class-expression-self-reference.mjs @@ -48,3 +48,14 @@ function factory(tag) { const T = factory("t"); T.make(); console.log(T.make().tag, T.count, factory("u").count); + +// The only reference to the name is a shorthand property. +function packed() { + const K = class Packed { + static wrap() { + return { Packed }; + } + }; + return [K.wrap().Packed === K, K.wrap().Packed.name]; +} +console.log(packed()); From 01939bc0721d00fe519fc23a66026a7f96193ab8 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 08:41:08 -0500 Subject: [PATCH 030/197] Give arrays state bytes only once they hold a hole or undefined An array starts without per-slot state bytes. While it has none, every index below its length is a present value in dense storage (len <= cap, no sparse entries) and its hole-free prefix bound `filled` equals its length, so its slots take 8 bytes per element (1 for booleans) instead of 9 (2). The first hole or present undefined below the length, a length set past the elements, or a sparse entry gives it states (scr_arr_add_states), which it keeps; SCR_ARR_STATES in `spare` records it, and the element ops id moves up one bit. Runtime readers take scr_arr_dense_state or scr_arr_scan_states, and writers keep the elements contiguous from 0 or give the array states before raising its length. A length set on an empty array with inline slots reserves its slots and states in one allocation. Emitted reads keep their shape: they load a state only at or past `filled`, which an array without states never reaches. A store past `filled` tests the flag and, without states, appends at the length; push and pop write a state byte only when the array has states. map, fused map and slice pipelines that keep holes, and concat's element copy reserve the result's storage, fill it, and set its length afterwards, so a result without holes is filled by appends and gets no states. The typed-view cache refresh moves the flag with the storage it swaps, and tagged-template arrays carry no state bytes. --- packages/compiler/src/backend/llvm/common.ts | 11 +- .../backend/llvm/dense-array-access.test.ts | 49 ++ .../src/backend/llvm/dense-array-access.ts | 85 ++- packages/compiler/src/backend/llvm/dyn.ts | 28 +- .../compiler/src/backend/llvm/emit-module.ts | 16 +- .../src/backend/llvm/emit-statements.ts | 5 + .../src/backend/llvm/expr-primitives.test.ts | 15 +- .../lowering/containers/array-callbacks.ts | 15 + .../lowering/containers/array-concat.ts | 10 + .../lowering/containers/array-pipelines.ts | 27 +- packages/compiler/src/ir/ir.ts | 14 +- packages/runtime/src/scr_array.c | 531 +++++++++++++----- packages/runtime/src/scr_array.h | 77 ++- packages/runtime/src/scr_bytes.c | 2 +- packages/runtime/src/scr_copying.c | 28 +- packages/runtime/src/scr_cycle.c | 4 +- packages/runtime/src/scr_itable.c | 2 +- packages/runtime/src/scr_itable.test.c | 2 + packages/runtime/src/scr_lib.c | 24 +- packages/runtime/src/scr_map.c | 2 +- packages/runtime/src/scr_publish.c | 2 +- packages/runtime/src/scr_regex.c | 24 +- packages/runtime/src/scr_region.c | 2 +- packages/runtime/src/scr_string.c | 12 +- packages/runtime/test/test_array.c | 191 ++++++- 25 files changed, 903 insertions(+), 275 deletions(-) diff --git a/packages/compiler/src/backend/llvm/common.ts b/packages/compiler/src/backend/llvm/common.ts index 822ee29e2..ac4e28708 100644 --- a/packages/compiler/src/backend/llvm/common.ts +++ b/packages/compiler/src/backend/llvm/common.ts @@ -10,9 +10,10 @@ import type { import type { LlvmEmitterContext } from "./expr-context.js"; /** Field numbers of %ScrArr, which mirrors scr_runtime.h's ScrArr field for - * field (its static assertions pin `len`, `cap`, `cyc`, `filled` and - * `data`). `len` and `cap` are i32 on every target; the dense states end - * the allocation after the `cap` value slots at `data` (scrArrStates). */ + * field (its static assertions pin `len`, `cap`, `cyc`, `spare`, `filled` + * and `data`). `len` and `cap` are i32 on every target; the dense states of + * an array with states (SCR_ARR_STATES in `spare`) end the allocation after + * the `cap` value slots at `data` (scrArrStates). */ export const SCR_ARR_FIELD = { rc: 0, len: 1, @@ -24,6 +25,10 @@ export const SCR_ARR_FIELD = { data: 7, } as const; +/** The `spare` flag of a %ScrArr whose dense slots carry state bytes + * (SCR_ARR_STATES in scr_array.h). */ +export const SCR_ARR_STATES = 16; + /** Loads the i32 `len` or `cap` of a %ScrArr at `ptr` into `dest` as a * size-type count. */ export function loadArrCount( diff --git a/packages/compiler/src/backend/llvm/dense-array-access.test.ts b/packages/compiler/src/backend/llvm/dense-array-access.test.ts index 1b84cf0a0..6a0123559 100644 --- a/packages/compiler/src/backend/llvm/dense-array-access.test.ts +++ b/packages/compiler/src/backend/llvm/dense-array-access.test.ts @@ -316,6 +316,55 @@ test.each([32, 64] as const)( }, ); +test("arrays without state bytes skip the state stores", () => { + const read: IrExpr = { kind: "arrayGet", arr: ref("s", words), index, type: STRING, loc }; + const write: IrStmt = { + kind: "arraySet", + arr: ref("s", words), + index, + value: { kind: "strLit", value: "x", type: STRING, loc }, + loc, + }; + const push: IrStmt = { + kind: "exprStmt", + expr: { + kind: "arrIntrinsic", + method: "push", + receiver: ref("a", nums), + args: [{ kind: "numLit", value: 2.5, type: F64, loc }], + type: F64, + loc, + }, + loc, + }; + const llvm = emitLlvmModule( + module("plain", STRING, [push, write, { kind: "return", value: read, loc }]), + ); + const fn = body(llvm, "plain"); + // SCR_ARR_STATES (16) in `spare`, field 5. + const flag = String.raw`getelementptr inbounds %ScrArr, ptr %t\d+, i32 0, i32 5\n %t\d+ = load i16, ptr %t\d+, !tbaa !\d+\n %t\d+ = and i16 %t\d+, 16\n %t\d+ = icmp ne i16 %t\d+, 0\n`; + // A push marks the state only when the array has states. + expect(fn).toMatch( + new RegExp( + `${flag} br i1 %t\\d+, label %arr\\.push\\.state\\d+, label %arr\\.push\\.marked\\d+\\narr\\.push\\.state\\d+:\\n store i8 1, ptr `, + ), + ); + // Above the prefix, an array without states has no elements (its prefix + // is its length): a store there appends at the length and replaces no + // value. + expect(fn).toMatch( + new RegExp( + `${flag} br i1 %t\\d+, label %arr\\.set\\.states\\d+, label %arr\\.set\\.plain\\d+`, + ), + ); + expect(fn).toMatch( + /^arr\.set\.plain\d+:\n %t\d+ = icmp eq i64 .*\n br i1 %t\d+, label %arr\.set\.values\d+, label %arr\.set\.slow\d+$/m, + ); + expect(fn).toMatch(/phi i1 \[ %t\d+, %arr\.set\.states\d+ \], \[ false, %arr\.set\.values\d+ \]/); + // Reads test no flag: they reach the state check only above the prefix. + expect(fn).not.toMatch(/label %arr\.get\.state\d+/); +}); + test("state and presence queries answer dense slots inline", () => { const state: IrExpr = { kind: "arrayState", arr: ref("b", flags), index, type: F64, loc }; const has: IrExpr = { kind: "arrayHas", arr: ref("s", words), index, type: BOOL, loc }; diff --git a/packages/compiler/src/backend/llvm/dense-array-access.ts b/packages/compiler/src/backend/llvm/dense-array-access.ts index 4488f90cb..4d8d5fecf 100644 --- a/packages/compiler/src/backend/llvm/dense-array-access.ts +++ b/packages/compiler/src/backend/llvm/dense-array-access.ts @@ -14,13 +14,17 @@ * state bytes altogether: the runtime keeps every state below it a present * value, so reads load the slot directly and writes replace it without * touching its state. Other dense indices take the state-checked path, - * whose writes extend the bound when they continue it. */ + * whose writes extend the bound when they continue it. An array without + * state bytes (SCR_ARR_STATES clear in `spare`) keeps its bound at its + * length, so reads never reach the state check for it; a write past the + * bound tests the flag and, without states, appends at the length. */ import { isRefCounted, type IrExpr, type IrType } from "../../ir/ir.js"; import { f64Lit, loadArrCount, SCR_ARR_FIELD, + SCR_ARR_STATES, scrArrStatePtr, scrArrStates, sizeToDouble, @@ -31,6 +35,24 @@ import { exactInteger, widenInteger } from "./integer-values.js"; const SCR_ARR_VALUE = 1; +/** Whether the array at `array` has state bytes, as an i1. `spare` changes + * only in runtime calls, so its loads take their own alias tag. */ +export function emitArrayHasStates(host: LlvmEmitterContext, array: string): string { + const B = host.B; + const sparePtr = B.tmp(), + spare = B.tmp(), + bit = B.tmp(), + has = B.tmp(); + B.line( + `${sparePtr} = getelementptr inbounds %ScrArr, ptr ${array}, i32 0, i32 ${SCR_ARR_FIELD.spare}`, + ); + host.markMemoryPointer(sparePtr, "array:spare"); + B.line(`${spare} = load i16, ptr ${sparePtr}${host.fieldAliasAttachment(sparePtr)}`); + B.line(`${bit} = and i16 ${spare}, ${SCR_ARR_STATES}`); + B.line(`${has} = icmp ne i16 ${bit}, 0`); + return has; +} + /** Call-site effects of the non-retaining read fallbacks: they only read * module-visible memory, or trap without returning to the caller (the trap * writes runtime-private stderr/TLS state). Declaring this lets LLVM keep @@ -482,18 +504,48 @@ export function emitDenseArraySet( B.line(`store ptr ${value}, ptr ${valuePtr}${host.fieldAliasAttachment(valuePtr)}`); } }; + const stated = B.newLabel("arr.set.states"), + plain = B.newLabel("arr.set.plain"), + plainStore = B.newLabel("arr.set.values"), + stored = B.newLabel("arr.set.stored.slot"); + B.condBr(emitArrayHasStates(host, array), stated, plain); + B.startBlock(stated); const statePtr = statePointer(host, guard); const valuePtr = valuePointer(host, guard, acc); - let oldState: string | null = null; - let old: string | null = null; + let statedOld: string | null = null; + let statedHad: string | null = null; if (acc === "ref") { - oldState = B.tmp(); - old = B.tmp(); + const oldState = B.tmp(); + statedOld = B.tmp(); + statedHad = B.tmp(); B.line(`${oldState} = load i8, ptr ${statePtr}${host.fieldAliasAttachment(statePtr)}`); - B.line(`${old} = load ptr, ptr ${valuePtr}${host.fieldAliasAttachment(valuePtr)}`); + B.line(`${statedOld} = load ptr, ptr ${valuePtr}${host.fieldAliasAttachment(valuePtr)}`); + B.line(`${statedHad} = icmp eq i8 ${oldState}, ${SCR_ARR_VALUE}`); } storeValue(valuePtr); B.line(`store i8 ${SCR_ARR_VALUE}, ptr ${statePtr}${host.fieldAliasAttachment(statePtr)}`); + const statedEnd = B.currentLabel(); + B.br(stored); + // Without states the bound is the length, so a store here appends at + // the length (it replaces no value) or leaves holes, which the runtime + // gives states. + B.startBlock(plain); + const appends = B.tmp(); + B.line(`${appends} = icmp eq ${host.sizeType} ${offset}, ${len}`); + B.condBr(appends, plainStore, slow); + B.startBlock(plainStore); + storeValue(valuePointer(host, guard, acc)); + const plainEnd = B.currentLabel(); + B.br(stored); + B.startBlock(stored); + let old: string | null = null; + let hadValue: string | null = null; + if (acc === "ref") { + old = B.tmp(); + hadValue = B.tmp(); + B.line(`${old} = phi ptr [ ${statedOld}, %${statedEnd} ], [ null, %${plainEnd} ]`); + B.line(`${hadValue} = phi i1 [ ${statedHad}, %${statedEnd} ], [ false, %${plainEnd} ]`); + } // A store at the bound continues the hole-free prefix. const filledWide = sizedFilled(host, guard.filled), continues = B.tmp(); @@ -537,9 +589,7 @@ export function emitDenseArraySet( storeArrCount(host, nextLen, lenPtr, host.fieldAliasAttachment(lenPtr)); B.br(storedLabel); B.startBlock(storedLabel); - if (acc === "ref" && oldState && old) { - const hadValue = B.tmp(); - B.line(`${hadValue} = icmp eq i8 ${oldState}, ${SCR_ARR_VALUE}`); + if (acc === "ref" && hadValue && old) { const releaseLabel = B.newLabel("arr.set.release"); B.condBr(hadValue, releaseLabel, join); B.startBlock(releaseLabel); @@ -627,7 +677,14 @@ export function emitDenseArrayPop( raw = wide; } B.line(`store ${slotTy} 0, ptr ${valuePtr}${host.fieldAliasAttachment(valuePtr)}`); + // Only an array with states has a state byte to clear. + const clear = B.newLabel("arr.pop.state"), + cleared = B.newLabel("arr.pop.cleared"); + B.condBr(emitArrayHasStates(host, array), clear, cleared); + B.startBlock(clear); B.line(`store i8 0, ptr ${statePtr}${host.fieldAliasAttachment(statePtr)}`); + B.br(cleared); + B.startBlock(cleared); let last32 = last; if (size === "i64") { last32 = B.tmp(); @@ -698,7 +755,14 @@ export function emitDenseArrayPush( } else { B.line(`store ptr ${value}, ptr ${valuePtr}${host.fieldAliasAttachment(valuePtr)}`); } + // Only an array with states has a state byte to mark. + const mark = B.newLabel("arr.push.state"), + marked = B.newLabel("arr.push.marked"); + B.condBr(emitArrayHasStates(host, array), mark, marked); + B.startBlock(mark); B.line(`store i8 ${SCR_ARR_VALUE}, ptr ${statePtr}${host.fieldAliasAttachment(statePtr)}`); + B.br(marked); + B.startBlock(marked); const next = B.tmp(), fastValue = B.tmp(); B.line(`${next} = add nuw ${size} ${len}, 1`); @@ -716,6 +780,7 @@ export function emitDenseArrayPush( B.line(`${bumped} = select i1 ${packed}, i32 ${nextFilled}, i32 ${filled}`); B.line(`store i32 ${bumped}, ptr ${filledPtr}${host.fieldAliasAttachment(filledPtr)}`); B.line(`${fastValue} = ${sizeToDouble(size)} ${size} ${next} to double`); + const fastEnd = B.currentLabel(); B.br(join); B.startBlock(slow); const slowValue = B.tmp(); @@ -723,6 +788,6 @@ export function emitDenseArrayPush( B.br(join); B.startBlock(join); const result = B.tmp(); - B.line(`${result} = phi double [ ${fastValue}, %${fast} ], [ ${slowValue}, %${slow} ]`); + B.line(`${result} = phi double [ ${fastValue}, %${fastEnd} ], [ ${slowValue}, %${slow} ]`); return result; } diff --git a/packages/compiler/src/backend/llvm/dyn.ts b/packages/compiler/src/backend/llvm/dyn.ts index 6911ef98b..86ea79907 100644 --- a/packages/compiler/src/backend/llvm/dyn.ts +++ b/packages/compiler/src/backend/llvm/dyn.ts @@ -1,6 +1,6 @@ import { typedRefConstructor } from "./shapes.js"; import { bindLiveArrayOps } from "./expr-stream-bridges.js"; -import { BYTES_ELEM_NUM, SCR_ARR_FIELD, f64Lit, sizeToDouble } from "./common.js"; +import { BYTES_ELEM_NUM, SCR_ARR_FIELD, SCR_ARR_STATES, f64Lit, sizeToDouble } from "./common.js"; import { emitUnionPeek, emitUnionTag } from "./union-repr.js"; import { INLINE_UNION_FIELD, @@ -1708,6 +1708,32 @@ export class LlDyn { B.line(`store ${member.type} ${oldValue}, ptr ${checkedPtr}`); }); if (t.kind !== "record") { + // Whether the dense slots carry states (SCR_ARR_STATES in + // `spare`) moves with them; the rest of `spare` stays. + const cachedSpare = B.tmp(), + checkedSpare = B.tmp(); + const sparePtr = (array: string): string => { + const ptr = B.tmp(); + B.line( + `${ptr} = getelementptr inbounds %ScrArr, ptr ${array}, i64 0, i32 ${SCR_ARR_FIELD.spare}`, + ); + return ptr; + }; + const cachedSparePtr = sparePtr(cached), + checkedSparePtr = sparePtr(checked); + B.line(`${cachedSpare} = load i16, ptr ${cachedSparePtr}`); + B.line(`${checkedSpare} = load i16, ptr ${checkedSparePtr}`); + const exchange = (own: string, other: string, ptr: string): void => { + const kept = B.tmp(), + moved = B.tmp(), + next = B.tmp(); + B.line(`${kept} = and i16 ${own}, ${~SCR_ARR_STATES}`); + B.line(`${moved} = and i16 ${other}, ${SCR_ARR_STATES}`); + B.line(`${next} = or i16 ${kept}, ${moved}`); + B.line(`store i16 ${next}, ptr ${ptr}`); + }; + exchange(cachedSpare, checkedSpare, cachedSparePtr); + exchange(checkedSpare, cachedSpare, checkedSparePtr); // Sparse slots and numeric properties live out of line; each // array keeps its own match metadata. host.declare("declare void @scr_arr_swap_indexed_ext(ptr, ptr)"); diff --git a/packages/compiler/src/backend/llvm/emit-module.ts b/packages/compiler/src/backend/llvm/emit-module.ts index 3d9d41072..bf89784b7 100644 --- a/packages/compiler/src/backend/llvm/emit-module.ts +++ b/packages/compiler/src/backend/llvm/emit-module.ts @@ -699,21 +699,19 @@ export function emitModuleParts(emitter: LlEmitter): string[] { for (const code of regexMatchers.values()) out.push(...code.lines, ``); for (const [, inst] of emitter.templateStringsInstances) { // One immortal ScrArr per tagged-template site: a data global of - // interned cooked-string literals followed by their states (the - // runtime finds them before data + cap * 9), and the ScrArr header over - // it (rc == SIZE_MAX, len == cap == filled, SCR_ELEM_STR = 2, no REF - // entry points). Every dense slot is present; reads retain immortal - // strings — a no-op. Slots are 8 bytes on every target: on 32-bit - // targets each holds the pointer in its low half and zero in its high - // half, so the states start where the runtime looks for them. + // interned cooked-string literals, and the ScrArr header over it + // (rc == SIZE_MAX, len == cap == filled, SCR_ELEM_STR = 2, no REF entry + // points, no state bytes: every dense slot is present). Reads retain + // immortal strings — a no-op. Slots are 8 bytes on every target: on + // 32-bit targets each holds the pointer in its low half and zero in its + // high half. const n = inst.slots.length; - const present = n === 0 ? "zeroinitializer" : `[ ${inst.slots.map(() => "i8 1").join(", ")} ]`; const wide = emitter.sizeType === "i64"; const slot = wide ? "ptr" : "<{ ptr, i32 }>"; const slotValue = (s: string): string => wide ? `ptr ${s}` : `<{ ptr, i32 }> <{ ptr ${s}, i32 0 }>`; out.push( - `@${inst.sym}_data = internal constant <{ [${n} x ${slot}], [${n} x i8] }> <{ [${n} x ${slot}] [ ${inst.slots.map(slotValue).join(", ")} ], [${n} x i8] ${present} }>, align 8`, + `@${inst.sym}_data = internal constant [${n} x ${slot}] [ ${inst.slots.map(slotValue).join(", ")} ], align 8`, `@${inst.sym} = internal global %ScrArr { ${emitter.sizeType} -1, i32 ${n}, i32 ${n}, i8 2, i8 0, i16 0, i32 ${n}, ptr @${inst.sym}_data }`, ); } diff --git a/packages/compiler/src/backend/llvm/emit-statements.ts b/packages/compiler/src/backend/llvm/emit-statements.ts index 7239db7d8..5376d4f53 100644 --- a/packages/compiler/src/backend/llvm/emit-statements.ts +++ b/packages/compiler/src/backend/llvm/emit-statements.ts @@ -435,6 +435,11 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { const length = emitter.emitExpr(s.length); if (s.arr.type.kind !== "array") throw new InternalCompilerError("llvm emitter bug: arraySetLength on non-array"); + if (s.reserve === true) { + emitter.declare(`declare void @scr_arr_reserve_len(ptr, double)`); + B.line(`call void @scr_arr_reserve_len(ptr ${arr.name}, double ${length.name})`); + break; + } emitter.emitPublishedArrayGuard(arr.name, s.arr.type, s.pop === true ? 2 : 3); emitter.declare(`declare void @scr_arr_set_len(ptr, double)`); // Assigning the current length (a reused work array reset to where diff --git a/packages/compiler/src/backend/llvm/expr-primitives.test.ts b/packages/compiler/src/backend/llvm/expr-primitives.test.ts index 41ecd189e..726ec20c5 100644 --- a/packages/compiler/src/backend/llvm/expr-primitives.test.ts +++ b/packages/compiler/src/backend/llvm/expr-primitives.test.ts @@ -308,7 +308,7 @@ test("sums that only feed a bitwise conversion of int32 operands add in 32 bits" }); test.each([32, 64] as const)( - "tagged-template string arrays keep 8-byte slots before their states on %i-bit targets", + "tagged-template string arrays keep 8-byte slots without states on %i-bit targets", (bits) => { const strings: IrExpr = { kind: "templateStrings", @@ -321,14 +321,17 @@ test.each([32, 64] as const)( mod.functions[0]!.body = [{ kind: "exprStmt", expr: strings, loc }]; const llvm = emitLlvmModule(mod, { pointerBits: bits }); const data = /^@sc_tsa_0_data = internal constant .*$/m.exec(llvm)![0]; - // The runtime reads each slot as 8 bytes and finds the states after - // cap * 8 bytes of slots (scr_arr_states_end). - if (bits === 64) expect(data).toContain("<{ [3 x ptr], [3 x i8] }>"); + // The runtime reads each slot as 8 bytes; an array without states + // (`spare` 0 in its header) has no state bytes after them. + if (bits === 64) expect(data).toMatch(/^\S+ = internal constant \[3 x ptr\] /); else { - expect(data).toContain("<{ [3 x <{ ptr, i32 }>], [3 x i8] }>"); + expect(data).toMatch(/^\S+ = internal constant \[3 x <\{ ptr, i32 \}>\] /); expect(data.match(/<\{ ptr @\S+, i32 0 \}>/g)).toHaveLength(3); } - expect(data).toContain("[3 x i8] [ i8 1, i8 1, i8 1 ]"); + expect(data).not.toContain("x i8]"); + expect(llvm).toMatch( + /^@sc_tsa_0 = internal global %ScrArr \{ i\d+ -1, i32 3, i32 3, i8 2, i8 0, i16 0, i32 3, /m, + ); }, ); diff --git a/packages/compiler/src/frontend/lowering/containers/array-callbacks.ts b/packages/compiler/src/frontend/lowering/containers/array-callbacks.ts index 612c47ea0..cc22594d5 100644 --- a/packages/compiler/src/frontend/lowering/containers/array-callbacks.ts +++ b/packages/compiler/src/frontend/lowering/containers/array-callbacks.ts @@ -778,6 +778,10 @@ function buildArrayHofFn( loc, }, readLen, + // map's result has the receiver's length, holes where the receiver + // has them. Storage for it is reserved first and the length is set + // after the loop, so a result without holes is filled by appends and + // gets no state bytes. ...(method === "map" ? [ { @@ -785,6 +789,7 @@ function buildArrayHofFn( arr: varRef("out.0", outT, loc), length: varRef("n.0", F64, loc), loc, + reserve: true, } satisfies IrStmt, ] : []), @@ -816,6 +821,16 @@ function buildArrayHofFn( loc, }, ]), + ...(method === "map" + ? [ + { + kind: "arraySetLength", + arr: varRef("out.0", outT, loc), + length: varRef("n.0", F64, loc), + loc, + } satisfies IrStmt, + ] + : []), { kind: "return", value: varRef("out.0", outT, loc), loc }, ]; } else if (method === "filter" && fastT) { diff --git a/packages/compiler/src/frontend/lowering/containers/array-concat.ts b/packages/compiler/src/frontend/lowering/containers/array-concat.ts index b4e285ee9..a7ce0a106 100644 --- a/packages/compiler/src/frontend/lowering/containers/array-concat.ts +++ b/packages/compiler/src/frontend/lowering/containers/array-concat.ts @@ -97,11 +97,15 @@ export function arrayConcatHelper( return [ { kind: "varDecl", localId: "offset.0", init: length(out), loc }, { kind: "varDecl", localId: "count.0", init: length(source), loc }, + // Storage is reserved first and the length is set after the copy, so + // a source without holes is copied by appends and the result gets no + // state bytes. { kind: "arraySetLength", arr: out, length: { kind: "bin", op: "+", left: offset, right: count, type: F64, loc }, loc, + reserve: true, }, countedFor(loc, count, () => [ { @@ -134,6 +138,12 @@ export function arrayConcatHelper( loc, }, ]), + { + kind: "arraySetLength", + arr: out, + length: { kind: "bin", op: "+", left: offset, right: count, type: F64, loc }, + loc, + }, ]; }; const body: IrStmt[] = [ diff --git a/packages/compiler/src/frontend/lowering/containers/array-pipelines.ts b/packages/compiler/src/frontend/lowering/containers/array-pipelines.ts index 180f40435..2f839a20b 100644 --- a/packages/compiler/src/frontend/lowering/containers/array-pipelines.ts +++ b/packages/compiler/src/frontend/lowering/containers/array-pipelines.ts @@ -1263,17 +1263,28 @@ class FusedLoop { loc, }); if (holes) { + // The result keeps the source's holes: storage for its length is + // reserved first and the length is set after the loop, so a result + // without holes is filled by appends and gets no state bytes. + const length = (): IrExpr => ({ + kind: "ternary", + cond: { kind: "bin", op: "<", left: lo, right: hi, type: BOOL, loc }, + then: { kind: "bin", op: "-", left: hi, right: lo, type: F64, loc }, + else_: numLit(0, loc), + type: F64, + loc, + }); stateDecls.push({ kind: "arraySetLength", arr: varRef(out, resultT, loc), - length: { - kind: "ternary", - cond: { kind: "bin", op: "<", left: lo, right: hi, type: BOOL, loc }, - then: { kind: "bin", op: "-", left: hi, right: lo, type: F64, loc }, - else_: numLit(0, loc), - type: F64, - loc, - }, + length: length(), + loc, + reserve: true, + }); + epilogue.push({ + kind: "arraySetLength", + arr: varRef(out, resultT, loc), + length: length(), loc, }); } diff --git a/packages/compiler/src/ir/ir.ts b/packages/compiler/src/ir/ir.ts index 4a395feb2..8008008f9 100644 --- a/packages/compiler/src/ir/ir.ts +++ b/packages/compiler/src/ir/ir.ts @@ -1943,8 +1943,18 @@ export type IrStmt = * RangeError from the runtime. `pop` marks the stores that implement * `pop()`: a frozen (published) array fails with pop's TypeError rather * than the length assignment's, and pop's `length = 0` store on an empty - * array is dropped unless the array may be frozen. */ - | { kind: "arraySetLength"; arr: IrExpr; length: IrExpr; loc: SrcLoc; pop?: boolean } + * array is dropped unless the array may be frozen. `reserve` marks a + * capacity hint instead: dense storage for `length` elements of a fresh + * array that statements fill next, with no change to its length and no + * observable effect. */ + | { + kind: "arraySetLength"; + arr: IrExpr; + length: IrExpr; + loc: SrcLoc; + pop?: boolean; + reserve?: boolean; + } /** Write an explicit present `undefined` state without shrinking length. */ | { kind: "arraySetUndefined"; arr: IrExpr; index: IrExpr; loc: SrcLoc } /** Delete an indexed/property slot without changing Array.length. */ diff --git a/packages/runtime/src/scr_array.c b/packages/runtime/src/scr_array.c index c00d1b4f6..c63bc4cc9 100644 --- a/packages/runtime/src/scr_array.c +++ b/packages/runtime/src/scr_array.c @@ -48,7 +48,7 @@ static inline const ScrArrExt *scr_arr_ax(const ScrArr *a) { } #define SCR_AX(a) scr_arr_ax(a) -static void scr_arr_move_inline(ScrArr *a, size_t cap, size_t unit); +static void scr_arr_move_inline(ScrArr *a, size_t cap); /* Bytes before the dense slots' allocation start: the ext pointer's word. */ static size_t scr_arr_prefix(const ScrArr *a) { @@ -60,9 +60,10 @@ static void *scr_arr_alloc_start(const ScrArr *a) { return (char *)scr_arr_block(a) - scr_arr_prefix(a); } -/* Bytes per dense slot with its state: an 8-byte value, or a boolean byte. */ +/* Bytes per dense slot: an 8-byte value or a boolean byte, and its state + * byte when the array has states. */ static size_t scr_arr_slot_unit(const ScrArr *a) { - return a->elem == SCR_ELEM_BOOL ? 2 * sizeof(uint8_t) : sizeof(uint64_t) + sizeof(uint8_t); + return scr_arr_slot_width(a) + (scr_arr_has_states(a) ? 1 : 0); } /* Bytes of dense storage (head room included) after the prefix. */ @@ -78,7 +79,7 @@ ScrArrExt *scr_arr_ext(ScrArr *a) { /* The pointer goes in the word before the dense allocation, which an * array with ext always has: inline slots move out, and an array without * storage gets an allocation of the word alone. */ - if (a->spare & SCR_ARR_INLINE_SLOTS) scr_arr_move_inline(a, a->cap, scr_arr_slot_unit(a)); + if (a->spare & SCR_ARR_INLINE_SLOTS) scr_arr_move_inline(a, a->cap); char *start; if (!a->data) { start = scr_mem_alloc(SCR_ARR_EXT_PREFIX); @@ -387,18 +388,14 @@ static void scr_arr_sparse_drop_prefix(ScrArr *a, size_t count) { x->sparse_len -= count; } -/* Bytes per dense slot: the value and its state byte. Boolean arrays - * store a one-byte value. */ -#define SCR_ARR_SLOT_BYTES (sizeof(uint64_t) + sizeof(uint8_t)) -#define SCR_ARR_BOOL_SLOT_BYTES (2 * sizeof(uint8_t)) /* Appends to an array smaller than this grow straight to it. */ #define SCR_ARR_MIN_GROWTH ((size_t)4) /* Inline slots: an array built by scr_arr_new_small / scr_arr_new_ops_small * (empty literals the compiler sees appended to), or constructed with a * known element count of up to SCR_ARR_KNOWN_INLINE_MAX, keeps its first - * dense slots and states in its own block, right after the ScrArr, so it - * costs one allocation and its first elements share the header's cache lines. + * dense slots in its own block, right after the ScrArr, so it costs one + * allocation and its first elements share the header's cache lines. * `spare` records it. Growth past the inline capacity moves the slots to an * ordinary allocation; the inline room then stays unused until the array is * freed. */ @@ -449,8 +446,14 @@ static void scr_arr_drop_head(ScrArr *a) { a->spare &= (uint16_t)~SCR_ARR_HEAD_ROOM; if (head == 0) return; size_t cap = a->cap, total = cap + head, live = a->len < cap ? a->len : cap; - uint8_t *end = scr_arr_states_end(a); uint64_t *data = a->data - head; + if (!scr_arr_has_states(a)) { + if (live) memmove(data, a->data, live * sizeof(uint64_t)); + a->data = data; + a->cap = total; + return; + } + uint8_t *end = scr_arr_states_end(a); /* The states end moves to the end of the allocation, past the `head` * vacated states: the live states move up with it, and every other state * becomes a hole. */ @@ -472,7 +475,7 @@ static bool scr_arr_drop_front(ScrArr *a, size_t count) { if (a->elem == SCR_ELEM_BOOL || scr_arr_inline(a)) return false; if (count == 0) return true; size_t head = scr_arr_head(a); - memset(scr_arr_state_run(a, 0, count), SCR_ARR_HOLE, count); + if (scr_arr_has_states(a)) memset(scr_arr_state_run(a, 0, count), SCR_ARR_HOLE, count); memset(a->data, 0, count * sizeof(uint64_t)); if (head) a->data[-1] = 0; a->data += count; @@ -492,9 +495,9 @@ static bool scr_arr_drop_front(ScrArr *a, size_t count) { } /* Slots requested by the small constructions. The small allocator rounds - * them up to fill its block: 3 eight-byte slots after a plain header in a - * 64-byte block (72 after a collector header), or 4 booleans in a 40-byte - * block. Most small arrays hold up to three elements; two slots make + * them up to fill its block: 4 eight-byte slots after a plain header in a + * 64-byte block (3 in 72 bytes after a collector header), or 8 booleans in + * a 40-byte block. Most small arrays hold up to three elements; two slots make * arrays that grow past them move their storage often enough to slow * programs that build many short lists. */ #if SCR_SMALL_ALLOC @@ -504,18 +507,14 @@ static bool scr_arr_drop_front(ScrArr *a, size_t count) { #endif /* Inline capacity of a small construction with `prefix` bytes of block - * before the slots. */ -static size_t scr_arr_small_cap(size_t prefix, bool bools) { + * before slots of `unit` bytes. Inlined, so it folds to a constant for a + * constant unit. */ +static inline size_t scr_arr_small_cap(size_t prefix, size_t unit) { #if SCR_SMALL_ALLOC - /* One return per slot unit, so each folds to a constant at its callers. */ - if (bools) - return (scr_sa_class_size(scr_sa_class(prefix + SCR_ARR_SMALL_SLOTS * SCR_ARR_BOOL_SLOT_BYTES)) - prefix) / - SCR_ARR_BOOL_SLOT_BYTES; - return (scr_sa_class_size(scr_sa_class(prefix + SCR_ARR_SMALL_SLOTS * SCR_ARR_SLOT_BYTES)) - prefix) / - SCR_ARR_SLOT_BYTES; + return (scr_sa_class_size(scr_sa_class(prefix + SCR_ARR_SMALL_SLOTS * unit)) - prefix) / unit; #else (void)prefix; - (void)bools; + (void)unit; return SCR_ARR_SMALL_SLOTS; #endif } @@ -528,18 +527,17 @@ static size_t scr_arr_small_cap(size_t prefix, bool bools) { #define SCR_ARR_KNOWN_INLINE_MAX ((size_t)16) /* Inline capacity of a construction of `need` elements with `prefix` bytes - * of block before the slots, rounded up to fill the block; 0 when its slots - * stay separate. */ -static size_t scr_arr_known_cap(size_t prefix, size_t need, bool bools) { + * of block before slots of `unit` bytes, rounded up to fill the block; 0 + * when its slots stay separate. */ +static size_t scr_arr_known_cap(size_t prefix, size_t need, size_t unit) { if (need == 0 || need > SCR_ARR_KNOWN_INLINE_MAX) return 0; #if SCR_SMALL_ALLOC - size_t unit = bools ? SCR_ARR_BOOL_SLOT_BYTES : SCR_ARR_SLOT_BYTES; size_t bytes = prefix + need * unit; if (bytes > SCR_SA_MAX) return 0; return (scr_sa_class_size(scr_sa_class(bytes)) - prefix) / unit; #else (void)prefix; - (void)bools; + (void)unit; return need; #endif } @@ -547,20 +545,22 @@ static size_t scr_arr_known_cap(size_t prefix, size_t need, bool bools) { static void scr_arr_use_inline(ScrArr *a, size_t cap) { a->data = (uint64_t *)((char *)a + SCR_ARR_INLINE_OFFSET); a->cap = (uint32_t)cap; - memset(scr_arr_state_run(a, 0, cap), SCR_ARR_HOLE, cap); + if (scr_arr_has_states(a)) memset(scr_arr_state_run(a, 0, cap), SCR_ARR_HOLE, cap); a->spare |= SCR_ARR_INLINE_SLOTS; } /* Move inline slots to their own allocation of `cap` (>= a->cap) slots; the * array's block keeps the unused room until it is freed. */ -static void scr_arr_move_inline(ScrArr *a, size_t cap, size_t unit) { +static void scr_arr_move_inline(ScrArr *a, size_t cap) { size_t width = scr_arr_slot_width(a); - uint64_t *data = scr_mem_alloc(cap * unit); + uint64_t *data = scr_mem_alloc(cap * scr_arr_slot_unit(a)); if (!data) scr_arr_oom(); - uint8_t *end = (uint8_t *)data + cap * (width + 1); memcpy(data, a->data, a->cap * width); - memcpy(end - a->cap, scr_arr_state_run(a, 0, a->cap), a->cap); - memset(end - cap, SCR_ARR_HOLE, cap - a->cap); + if (scr_arr_has_states(a)) { + uint8_t *end = (uint8_t *)data + cap * (width + 1); + memcpy(end - a->cap, scr_arr_state_run(a, 0, a->cap), a->cap); + memset(end - cap, SCR_ARR_HOLE, cap - a->cap); + } a->data = data; a->cap = cap; a->spare &= (uint16_t)~SCR_ARR_INLINE_SLOTS; @@ -582,15 +582,15 @@ typedef enum { * reallocation. Growth keeps its own sequence (4, 8, 16, ...): rounding it * as well moves array storage into other size classes and measured * slower. */ -static size_t scr_arr_block_cap(size_t cap, bool bools) { +static size_t scr_arr_block_cap(size_t cap, size_t unit) { #if SCR_SMALL_ALLOC - size_t bytes = cap * (bools ? SCR_ARR_BOOL_SLOT_BYTES : SCR_ARR_SLOT_BYTES); + size_t bytes = cap * unit; if (bytes <= SCR_SA_MAX) { size_t block = scr_sa_class_size(scr_sa_class((bytes + 15) & ~(size_t)15)); - return bools ? block / SCR_ARR_BOOL_SLOT_BYTES : block / SCR_ARR_SLOT_BYTES; + return block / unit; } #else - (void)bools; + (void)unit; #endif return cap; } @@ -614,22 +614,25 @@ static inline void scr_arr_grow_dense_to(ScrArr *a, size_t need, ScrArrGrow mode scr_arr_grow_dense_slow(a, need, mode); return; } + bool states = scr_arr_has_states(a); + size_t unit = sizeof(uint64_t) + (states ? 1 : 0); size_t cap; if (mode == SCR_ARR_GROW_EXACT) { - cap = scr_arr_block_cap(need, false); + cap = scr_arr_block_cap(need, unit); } else { cap = a->cap < SCR_ARR_MIN_GROWTH ? SCR_ARR_MIN_GROWTH : a->cap; while (cap < need) cap += cap; if (cap > SCR_ARR_DENSE_LIMIT) cap = SCR_ARR_DENSE_LIMIT; } size_t old = a->cap; - uint64_t *data = a->data ? scr_mem_realloc(a->data, cap * SCR_ARR_SLOT_BYTES) - : scr_mem_alloc(cap * SCR_ARR_SLOT_BYTES); + uint64_t *data = a->data ? scr_mem_realloc(a->data, cap * unit) : scr_mem_alloc(cap * unit); if (!data) scr_arr_oom(); - /* The states end the allocation: move them to its new end. */ - uint8_t *end = (uint8_t *)(data + cap) + cap; - if (old) memmove(end - old, (uint8_t *)(data + old), old); - memset(end - cap, SCR_ARR_HOLE, cap - old); + if (states) { + /* The states end the allocation: move them to its new end. */ + uint8_t *end = (uint8_t *)(data + cap) + cap; + if (old) memmove(end - old, (uint8_t *)(data + old), old); + memset(end - cap, SCR_ARR_HOLE, cap - old); + } a->data = data; a->cap = cap; } @@ -660,12 +663,12 @@ static void scr_arr_grow_dense_slow(ScrArr *a, size_t need, ScrArrGrow mode) { mode = SCR_ARR_GROW_APPEND; } } - bool bools = a->elem == SCR_ELEM_BOOL; + bool states = scr_arr_has_states(a); size_t width = scr_arr_slot_width(a); - size_t unit = bools ? SCR_ARR_BOOL_SLOT_BYTES : SCR_ARR_SLOT_BYTES; + size_t unit = scr_arr_slot_unit(a); size_t cap; if (mode == SCR_ARR_GROW_EXACT) { - cap = scr_arr_block_cap(need, bools); + cap = scr_arr_block_cap(need, unit); } else { /* Inline slots overflow into the ordinary sequence (4, 8, 16, ...). */ size_t from = scr_arr_inline(a) ? 0 : a->cap; @@ -680,9 +683,9 @@ static void scr_arr_grow_dense_slow(ScrArr *a, size_t need, ScrArrGrow mode) { * appends benefit from geometric headroom. */ if (mode == SCR_ARR_GROW_SPAN && need > SCR_ARR_DENSE_LIMIT) cap = need; } - if (cap > SIZE_MAX / SCR_ARR_SLOT_BYTES) scr_arr_oom(); /* bounds both slot sizes */ + if (cap > SIZE_MAX / (sizeof(uint64_t) + 1)) scr_arr_oom(); /* bounds every slot unit */ if (scr_arr_inline(a)) { - scr_arr_move_inline(a, cap, unit); + scr_arr_move_inline(a, cap); scr_arr_absorb_sparse(a); return; } @@ -692,7 +695,7 @@ static void scr_arr_grow_dense_slow(ScrArr *a, size_t need, ScrArrGrow mode) { if (!data) scr_arr_oom(); a->data = data; a->cap = (uint32_t)cap; - memset(scr_arr_state_run(a, 0, cap), SCR_ARR_HOLE, cap); + if (states) memset(scr_arr_state_run(a, 0, cap), SCR_ARR_HOLE, cap); return; } /* Values and states share one allocation. Move the old states to its new @@ -702,9 +705,11 @@ static void scr_arr_grow_dense_slow(ScrArr *a, size_t need, ScrArrGrow mode) { char *start = scr_mem_realloc((char *)a->data - prefix, prefix + cap * unit); if (!start) scr_arr_oom(); uint64_t *data = (uint64_t *)(start + prefix); - uint8_t *end = (uint8_t *)data + cap * (width + 1); - if (a->cap) memmove(end - a->cap, (uint8_t *)data + a->cap * width, a->cap); - memset(end - cap, SCR_ARR_HOLE, cap - a->cap); + if (states) { + uint8_t *end = (uint8_t *)data + cap * (width + 1); + if (a->cap) memmove(end - a->cap, (uint8_t *)data + a->cap * width, a->cap); + memset(end - cap, SCR_ARR_HOLE, cap - a->cap); + } a->data = data; a->cap = cap; scr_arr_absorb_sparse(a); @@ -714,6 +719,44 @@ static void scr_arr_grow_dense(ScrArr *a, size_t need) { scr_arr_grow_dense_to(a, need, SCR_ARR_GROW_APPEND); } +/* States follow the value slots, and the capacity stays: inline slots + * move to an allocation of their own, and other storage grows by the state + * bytes (immortal storage, which may be static, is copied instead). */ +void scr_arr_add_states(ScrArr *a) { + if (scr_arr_has_states(a)) return; + size_t width = scr_arr_slot_width(a), len = a->len; + if (a->cap == 0) { + a->spare |= SCR_ARR_STATES; + return; + } + if (scr_arr_inline(a)) { + uint64_t *data = scr_mem_alloc(a->cap * (width + 1)); + if (!data) scr_arr_oom(); + memcpy(data, a->data, len * width); + a->data = data; + a->spare &= (uint16_t)~SCR_ARR_INLINE_SLOTS; + } else if (a->rc == SIZE_MAX) { + size_t prefix = scr_arr_prefix(a); + char *start = scr_mem_alloc(prefix + a->cap * (width + 1)); + if (!start) scr_arr_oom(); + if (prefix) memcpy(start, scr_arr_alloc_start(a), prefix); + memcpy(start + prefix, a->data, len * width); + a->data = (uint64_t *)(start + prefix); + a->spare &= (uint16_t)~SCR_ARR_HEAD_ROOM; + } else { + /* Head room goes back to the capacity first. */ + scr_arr_drop_head(a); + size_t prefix = scr_arr_prefix(a); + char *start = scr_mem_realloc(scr_arr_alloc_start(a), prefix + a->cap * (width + 1)); + if (!start) scr_arr_oom(); + a->data = (uint64_t *)(start + prefix); + } + a->spare |= SCR_ARR_STATES; + uint8_t *end = scr_arr_states_end(a); + memset(end - len, SCR_ARR_VALUE, len); + memset(end - a->cap, SCR_ARR_HOLE, a->cap - len); +} + /* Room for `count` values in front of the first element of a dense array: * moves `data` back into head room, raising `cap`; the caller * stores the new first `count` slots and states. A shifted array reuses the @@ -734,21 +777,24 @@ static bool scr_arr_open_head(ScrArr *a, size_t count) { /* The new allocation: head room for the new values plus half the new * length, then the current dense window (its tail capacity included). */ size_t room = count + (len + count) / 2; - if (room > SCR_ARR_MAX_LENGTH || a->cap > SIZE_MAX / SCR_ARR_SLOT_BYTES - room) return false; + size_t unit = scr_arr_slot_unit(a); + if (room > SCR_ARR_MAX_LENGTH || a->cap > SIZE_MAX / unit - room) return false; size_t total = room + a->cap; size_t prefix = scr_arr_prefix(a); - char *start = scr_mem_alloc(prefix + total * SCR_ARR_SLOT_BYTES); + char *start = scr_mem_alloc(prefix + total * unit); if (!start) scr_arr_oom(); /* The ext pointer's word moves with the allocation. */ if (prefix) memcpy(start, scr_arr_alloc_start(a), prefix); uint64_t *block = (uint64_t *)(start + prefix); - /* The window's states end `room` bytes before the allocation does; the - * room's states past it are holes. */ - uint8_t *end = (uint8_t *)(block + total) + a->cap; memcpy(block + room, a->data, len * sizeof(uint64_t)); - memset(end, SCR_ARR_HOLE, room); - memcpy(end - len, scr_arr_state_run(a, 0, len), len); - memset(end - a->cap, SCR_ARR_HOLE, a->cap - len); + if (scr_arr_has_states(a)) { + /* The window's states end `room` bytes before the allocation does; + * the room's states past it are holes. */ + uint8_t *end = (uint8_t *)(block + total) + a->cap; + memset(end, SCR_ARR_HOLE, room); + memcpy(end - len, scr_arr_state_run(a, 0, len), len); + memset(end - a->cap, SCR_ARR_HOLE, a->cap - len); + } scr_mem_free(scr_arr_alloc_start(a)); a->data = block + room; head = room; @@ -764,10 +810,9 @@ static bool scr_arr_open_head(ScrArr *a, size_t count) { * within that gap past capacity. */ static bool scr_arr_dense_tail_populated(const ScrArr *a) { size_t window = a->cap < SCR_ARR_DENSE_GAP ? a->cap : SCR_ARR_DENSE_GAP; - const uint8_t *states = scr_arr_states_end(a); - if (window == 0 || *scr_arr_state_in(states, a->cap - 1) == SCR_ARR_HOLE) return false; + if (window == 0 || scr_arr_dense_state(a, a->cap - 1) == SCR_ARR_HOLE) return false; size_t populated = 0; - for (size_t i = a->cap - window; i < a->cap; i++) populated += *scr_arr_state_in(states, i) != SCR_ARR_HOLE; + for (size_t i = a->cap - window; i < a->cap; i++) populated += scr_arr_dense_state(a, i) != SCR_ARR_HOLE; return populated * 16 >= window * 3; } @@ -841,7 +886,7 @@ static __attribute__((noinline, pure)) ScrArrStateSlot scr_arr_sparse_state_at(c * sparse entries above it. */ static inline uint8_t scr_arr_state_at(const ScrArr *a, size_t index, uint64_t *out) { if (index < a->cap) { - uint8_t state = *scr_arr_state_in(scr_arr_states_end(a), index); + uint8_t state = scr_arr_dense_state(a, index); if (state == SCR_ARR_HOLE) return SCR_ARR_HOLE; if (out) *out = scr_arr_dense_get(a, index); return state; @@ -855,15 +900,19 @@ static inline bool scr_arr_slot_at(const ScrArr *a, size_t index, uint64_t *out) return scr_arr_state_at(a, index, out) != SCR_ARR_HOLE; } -/* Store an owned slot into an empty index. The caller has already established - * the destination length and does not need a retain. */ +/* Store an owned slot into an empty index. The caller does not need a + * retain. An array with states has its destination length established + * already; an array without states takes only a value at its length, + * whose caller then raises the length, and gets states for any other + * store. */ static void scr_arr_store_state_owned(ScrArr *a, size_t index, uint64_t slot, uint8_t state) { if (state != SCR_ARR_VALUE) scr_arr_cut(a, index); + if (!scr_arr_has_states(a) && (state != SCR_ARR_VALUE || index != a->len)) scr_arr_add_states(a); scr_arr_grow_for_index(a, index); if (index < a->cap) { scr_arr_dense_put(a, index, slot); - *scr_arr_state_ptr(a, index) = state; + if (scr_arr_has_states(a)) *scr_arr_state_ptr(a, index) = state; return; } size_t pos = scr_arr_sparse_lower_bound(SCR_AX(a)->sparse, SCR_AX(a)->sparse_len, index); @@ -879,12 +928,15 @@ static void scr_arr_store_owned(ScrArr *a, size_t index, uint64_t slot) { scr_arr_store_state_owned(a, index, slot, SCR_ARR_VALUE); } +/* Take the slot at `index`, leaving a hole: an array without states gets + * them. */ static bool scr_arr_take_state(ScrArr *a, size_t index, uint64_t *out, uint8_t *state_out) { if (index < a->cap) { - uint8_t state = *scr_arr_state_ptr(a, index); + uint8_t state = scr_arr_dense_state(a, index); if (state == SCR_ARR_HOLE) return false; scr_arr_cut(a, index); + scr_arr_need_states(a); *out = scr_arr_dense_get(a, index); *scr_arr_state_ptr(a, index) = SCR_ARR_HOLE; scr_arr_dense_put(a, index, 0); @@ -909,11 +961,19 @@ static void scr_arr_replace_state_owned(ScrArr *a, size_t index, uint64_t slot, uint8_t old_state; uint64_t old = 0; if (state != SCR_ARR_VALUE) scr_arr_cut(a, index); + /* Without states, a value below the length replaces a value. */ + if (!scr_arr_has_states(a) && (state != SCR_ARR_VALUE || index >= a->len)) { + if (state == SCR_ARR_VALUE && index == a->len) { + scr_arr_store_state_owned(a, index, slot, state); + return; + } + scr_arr_add_states(a); + } if (index < a->cap) { - old_state = *scr_arr_state_ptr(a, index); + old_state = scr_arr_dense_state(a, index); if (old_state == SCR_ARR_VALUE) old = scr_arr_dense_get(a, index); scr_arr_dense_put(a, index, slot); - *scr_arr_state_ptr(a, index) = state; + if (scr_arr_has_states(a)) *scr_arr_state_ptr(a, index) = state; } else { size_t pos = scr_arr_sparse_lower_bound(SCR_AX(a)->sparse, SCR_AX(a)->sparse_len, index); if (pos == SCR_AX(a)->sparse_len || SCR_AX(a)->sparse[pos].index != index) { @@ -941,6 +1001,10 @@ static void scr_arr_note_value(ScrArr *a, size_t index) { /* scr_arr_refill bounded to [0, limit): for range operations whose cost is * proportional to the range, not to the whole array. */ static void scr_arr_refill_to(ScrArr *a, size_t limit) { + if (!scr_arr_has_states(a)) { + scr_arr_refill(a); + return; + } size_t end = a->len < a->cap ? a->len : a->cap; if (limit < end) end = limit; size_t f = a->filled; @@ -1022,7 +1086,11 @@ static bool scr_arr_prop_delete(ScrArr *a, double key) { return true; } +/* Detach the storage of `a` for a rebuild through the element stores. Both + * have states: the detached storage for its readers, and the array for + * stores at any index. */ static ScrArrStorage scr_arr_take_storage(ScrArr *a) { + scr_arr_need_states(a); ScrArrExt *x = scr_arr_ext_of(a); ScrArrStorage old = {a->data, scr_arr_states_end(a), a->cap, SCR_AX(a)->sparse, SCR_AX(a)->sparse_len, SCR_AX(a)->sparse_cap, a->elem, @@ -1059,17 +1127,20 @@ static bool scr_arr_is_dense(const ScrArr *a) { return a->len <= a->cap && (!(a->spare & SCR_ARR_HAS_EXT) || scr_arr_ext_of(a)->sparse_len == 0); } +/* Without states, both move the values only: the caller keeps the + * elements contiguous from 0 and sets the length to their count. */ static void scr_arr_move_dense(ScrArr *a, size_t to, size_t from, size_t count) { if (count == 0) return; scr_arr_cut(a, to); memmove(scr_arr_dense_at(a, to), scr_arr_dense_at(a, from), count * scr_arr_slot_width(a)); - memmove(scr_arr_state_run(a, to, count), scr_arr_state_run(a, from, count), count); + if (scr_arr_has_states(a)) + memmove(scr_arr_state_run(a, to, count), scr_arr_state_run(a, from, count), count); } static void scr_arr_clear_dense(ScrArr *a, size_t from, size_t count) { if (count == 0) return; scr_arr_cut(a, from); - memset(scr_arr_state_run(a, from, count), SCR_ARR_HOLE, count); + if (scr_arr_has_states(a)) memset(scr_arr_state_run(a, from, count), SCR_ARR_HOLE, count); memset(scr_arr_dense_at(a, from), 0, count * scr_arr_slot_width(a)); } @@ -1138,10 +1209,20 @@ static inline void scr_arr_copy_slots(uint64_t *dst, const uint64_t *src, size_t } } +/* Whether dense slots [from, from + count) of `a` all hold values. */ +static bool scr_arr_dense_run_values(const ScrArr *a, size_t from, size_t count) { + if (!scr_arr_has_states(a)) return from + count <= a->len; + if (from + count <= a->filled) return true; + return scr_arr_value_run(scr_arr_states_end(a), from, from + count) == from + count; +} + /* The destination owns no values in this range. Only retain adapters run * while copying references: no release can collect an intermediate copy. * Self-appends use disjoint ranges after reserving capacity. Both arrays - * hold the same element kind, so their slots have the same width. */ + * hold the same element kind, so their slots have the same width. A + * destination without states keeps none for a range of values: its caller + * makes the range contiguous with its elements and sets the length; any + * other range gives it states. */ static void scr_arr_copy_dense(ScrArr *dst, size_t to, const ScrArr *src, size_t from, size_t count, bool reverse, bool materialize_holes) { @@ -1149,28 +1230,49 @@ static void scr_arr_copy_dense(ScrArr *dst, size_t to, const ScrArr *src, scr_arr_cut(dst, to); /* 8-byte slots: a constant slot size, and constant-width state ends. */ bool wide = dst->elem != SCR_ELEM_BOOL; + /* Callers copy from an array without states only from its elements + * (which a self-copy reads after moving them, past the length). */ + bool values = !scr_arr_has_states(src) || scr_arr_dense_run_values(src, from, count); + if (!values) scr_arr_need_states(dst); + bool refs = scr_elem_is_ref(src->elem); + if (!scr_arr_has_states(dst)) { + if (!reverse && wide) { + scr_arr_copy_slots(dst->data + to, src->data + from, count); + } else if (!reverse) { + memcpy(scr_arr_dense_at(dst, to), scr_arr_dense_at(src, from), count * scr_arr_slot_width(dst)); + } else { + for (size_t i = 0; i < count; i++) + scr_arr_dense_put(dst, to + i, scr_arr_dense_get(src, from + count - i - 1)); + } + if (!refs) return; + uint8_t elem = src->elem; + const ScrArrOps *ops = scr_arr_ops(src); + for (size_t i = to; i < to + count; i++) dst->data[i] = scr_elem_retain_with(elem, ops, dst->data[i]); + return; + } uint8_t *dst_states = wide ? scr_arr_states_end_w(dst, 8) : scr_arr_states_end_w(dst, 1); - const uint8_t *src_states = wide ? scr_arr_states_end_w(src, 8) : scr_arr_states_end_w(src, 1); + const uint8_t *src_states = values ? NULL : wide ? scr_arr_states_end_w(src, 8) : scr_arr_states_end_w(src, 1); if (!reverse && wide) { scr_arr_copy_slots(dst->data + to, src->data + from, count); - /* A range below the source's hole-free prefix holds only values. */ - if (from + count <= src->filled) + if (values) scr_arr_set_states(dst_states - to - count, SCR_ARR_VALUE, count); else scr_arr_copy_states(dst_states - to - count, src_states - from - count, count); } else if (!reverse) { memcpy(scr_arr_dense_at(dst, to), scr_arr_dense_at(src, from), count * scr_arr_slot_width(dst)); - scr_arr_copy_states(dst_states - to - count, src_states - from - count, count); + if (values) + scr_arr_set_states(dst_states - to - count, SCR_ARR_VALUE, count); + else + scr_arr_copy_states(dst_states - to - count, src_states - from - count, count); } else { for (size_t i = 0; i < count; i++) { size_t source = from + count - i - 1; scr_arr_dense_put(dst, to + i, scr_arr_dense_get(src, source)); - *scr_arr_state_in(dst_states, to + i) = *scr_arr_state_in(src_states, source); + *scr_arr_state_in(dst_states, to + i) = values ? SCR_ARR_VALUE : *scr_arr_state_in(src_states, source); } } - bool refs = scr_elem_is_ref(src->elem); - if (!refs && !materialize_holes) return; + if (!refs && (!materialize_holes || values)) return; uint8_t elem = src->elem; const ScrArrOps *ops = scr_arr_ops(src); for (size_t i = to; i < to + count; i++) { @@ -1197,9 +1299,9 @@ static void scr_arr_put_slots(ScrArr *a, size_t to, size_t count, const uint64_t static inline __attribute__((always_inline)) ScrArr *scr_arr_make(ScrElemKind elem, size_t initial_cap, bool small) { - size_t unit = elem == SCR_ELEM_BOOL ? SCR_ARR_BOOL_SLOT_BYTES : SCR_ARR_SLOT_BYTES; - size_t inline_cap = small ? scr_arr_small_cap(SCR_ARR_INLINE_OFFSET, elem == SCR_ELEM_BOOL) - : scr_arr_known_cap(SCR_ARR_INLINE_OFFSET, initial_cap, elem == SCR_ELEM_BOOL); + size_t unit = scr_elem_slot_width(elem); + size_t inline_cap = small ? scr_arr_small_cap(SCR_ARR_INLINE_OFFSET, unit) + : scr_arr_known_cap(SCR_ARR_INLINE_OFFSET, initial_cap, unit); ScrArr *a = scr_mem_alloc(inline_cap ? SCR_ARR_INLINE_OFFSET + inline_cap * unit : sizeof(ScrArr)); if (!a) scr_arr_oom(); a->rc = 1; @@ -1225,13 +1327,18 @@ ScrArr *scr_arr_new(ScrElemKind elem, size_t initial_cap) { ScrArr *scr_arr_new_small(ScrElemKind elem) { return scr_arr_make(elem, 0, true); } /* A non-cycle array whose final length the runtime knows when it builds - * it (regex match results): its `cap` slots and states share the header's - * block, so construction and teardown cost one allocation. Growth past + * it (regex match results): its `cap` slots share the header's block, so + * construction and teardown cost one allocation. With `states`, the array + * starts with states, whose bytes share the block as well. Growth past * `cap` moves the slots out like any inline array. */ -ScrArr *scr_arr_new_inline(ScrElemKind elem, size_t cap) { - if (cap == 0) return scr_arr_make(elem, 0, false); +ScrArr *scr_arr_new_inline(ScrElemKind elem, size_t cap, bool states) { + if (cap == 0) { + ScrArr *a = scr_arr_make(elem, 0, false); + if (states) a->spare |= SCR_ARR_STATES; + return a; + } if (cap > SCR_ARR_MAX_LENGTH) scr_arr_oom(); - size_t unit = elem == SCR_ELEM_BOOL ? SCR_ARR_BOOL_SLOT_BYTES : SCR_ARR_SLOT_BYTES; + size_t unit = scr_elem_slot_width(elem) + (states ? 1 : 0); ScrArr *a = scr_mem_alloc(SCR_ARR_INLINE_OFFSET + cap * unit); if (!a) scr_arr_oom(); a->rc = 1; @@ -1239,7 +1346,7 @@ ScrArr *scr_arr_new_inline(ScrElemKind elem, size_t cap) { a->cap = 0; a->elem = (uint8_t)elem; a->cyc = 0; - a->spare = 0; + a->spare = states ? SCR_ARR_STATES : 0; a->filled = 0; a->data = NULL; scr_arr_use_inline(a, cap); @@ -1250,10 +1357,9 @@ ScrArr *scr_arr_new_inline(ScrElemKind elem, size_t cap) { } void scr_arr_outline(ScrArr *a) { - if (scr_arr_inline(a)) - scr_arr_move_inline(a, a->cap, - a->elem == SCR_ELEM_BOOL ? SCR_ARR_BOOL_SLOT_BYTES : SCR_ARR_SLOT_BYTES); - /* The swapped fields do not include `spare`: no head room either. */ + if (scr_arr_inline(a)) scr_arr_move_inline(a, a->cap); + /* Of `spare`, a swap of the storage fields may move only the states + * flag: no head room either. */ if (a->spare & SCR_ARR_HEAD_ROOM) { scr_arr_drop_head(a); scr_arr_absorb_sparse(a); @@ -1287,9 +1393,14 @@ void scr_arr_trace_v(void *a0, ScrTraceVisit visit, void *ctx) { if (SCR_AX(a)->metadata) visit(SCR_AX(a)->metadata, ctx); /* Growth and truncation leave spare capacity empty. Sparse slots and * named properties are separate edges, independent of dense length. */ - const uint8_t *states = scr_arr_states_end(a); - for (size_t i = 0, end = a->len < a->cap ? a->len : a->cap; i < end; i++) { - if (*scr_arr_state_in(states, i) == SCR_ARR_VALUE) visit(scr_slot_to_ptr(a->data[i]), ctx); + size_t end = a->len < a->cap ? a->len : a->cap; + if (!scr_arr_has_states(a)) { + for (size_t i = 0; i < end; i++) visit(scr_slot_to_ptr(a->data[i]), ctx); + } else { + const uint8_t *states = scr_arr_states_end(a); + for (size_t i = 0; i < end; i++) { + if (*scr_arr_state_in(states, i) == SCR_ARR_VALUE) visit(scr_slot_to_ptr(a->data[i]), ctx); + } } for (size_t i = 0; i < SCR_AX(a)->sparse_len; i++) { if (SCR_AX(a)->sparse[i].state == SCR_ARR_VALUE) { @@ -1338,9 +1449,10 @@ static inline __attribute__((always_inline)) ScrArr *scr_arr_make_ops(const ScrA ScrArr *a; bool cyc = ops->trace != NULL; size_t prefix = SCR_ARR_INLINE_OFFSET + (cyc ? sizeof(ScrCycHdr) : 0); + size_t unit = scr_elem_slot_width(SCR_ELEM_REF); size_t inline_cap = - small ? scr_arr_small_cap(prefix, false) : scr_arr_known_cap(prefix, initial_cap, false); - size_t size = inline_cap ? SCR_ARR_INLINE_OFFSET + inline_cap * SCR_ARR_SLOT_BYTES : sizeof(ScrArr); + small ? scr_arr_small_cap(prefix, unit) : scr_arr_known_cap(prefix, initial_cap, unit); + size_t size = inline_cap ? SCR_ARR_INLINE_OFFSET + inline_cap * unit : sizeof(ScrArr); if (cyc) { static ScrCycType cyc_type = { &scr_arr_trace_v, &scr_arr_gc_free, SCR_CYC_KIND_ARRAY }; a = scr_cyc_alloc_inline(size, &cyc_type); @@ -1391,18 +1503,28 @@ static void scr_arr_destroy(void *object) { * first (and the storage cannot move under the loop); immortal strings * (interned characters, literals) need no call. */ const uint64_t *data = a->data; - const uint8_t *states = scr_arr_states_end_w(a, 8); - /* Slots below the hole-free prefix bound hold values. */ - size_t filled = a->filled < end ? a->filled : end; + /* Slots below the hole-free prefix bound hold values, as do all + * slots below the length of an array without states. */ + size_t filled = !scr_arr_has_states(a) ? end : a->filled < end ? a->filled : end; for (size_t i = 0; i < filled; i++) { ScrStr *s = (ScrStr *)scr_slot_to_ptr(data[i]); if (s && s->rc != SIZE_MAX) scr_str_release(s); } + const uint8_t *states = scr_arr_states_end_w(a, 8); for (size_t i = filled; i < end; i++) { if (*scr_arr_state_in(states, i) != SCR_ARR_VALUE) continue; ScrStr *s = (ScrStr *)scr_slot_to_ptr(data[i]); if (s && s->rc != SIZE_MAX) scr_str_release(s); } + } else if (!scr_arr_has_states(a)) { + /* Each slot is cleared before its release, as with states. */ + uint8_t elem = a->elem; + const ScrArrOps *ops = scr_arr_ops(a); + for (size_t i = 0; i < end; i++) { + uint64_t old = a->data[i]; + a->data[i] = 0; + scr_elem_release_with(elem, ops, old); + } } else { uint8_t *states = scr_arr_states_end(a); uint8_t elem = a->elem; @@ -1485,6 +1607,7 @@ double scr_arr_next_present(const ScrArr *a, double start) { if (!(start < SCR_ARR_LEN_F64(a->len))) return SCR_ARR_LEN_F64(a->len); size_t i = start > 0 ? (size_t)ceil(start) : 0; size_t end = a->cap < a->len ? a->cap : a->len; + if (!scr_arr_has_states(a)) return (double)i; const uint8_t *states = scr_arr_states_end(a); for (; i < end; i++) { if (*scr_arr_state_in(states, i) != SCR_ARR_HOLE) return (double)i; @@ -1523,6 +1646,22 @@ void scr_arr_set_len(ScrArr *a, double length) { * array must cost only the elements removed, not its historical peak. */ size_t dense_end = a->len < a->cap ? a->len : a->cap; size_t dense_stop = next < dense_end ? next : dense_end; + if (!scr_arr_has_states(a)) { + /* Values only: each slot is cleared before its release. The + * length drops first, so a release that reads the array sees the + * values that remain. */ + a->len = (uint32_t)next; + if (scr_elem_is_ref(a->elem)) { + for (size_t i = dense_stop; i < dense_end; i++) { + uint64_t old = a->data[i]; + a->data[i] = 0; + scr_elem_release(a, old); + } + } else if (dense_stop < dense_end) { + memset(scr_arr_dense_at(a, dense_stop), 0, (dense_end - dense_stop) * scr_arr_slot_width(a)); + } + return; + } if (scr_elem_is_ref(a->elem)) { for (size_t i = dense_stop; i < dense_end; i++) { if (*scr_arr_state_ptr(a, i) != SCR_ARR_HOLE) { @@ -1551,8 +1690,20 @@ void scr_arr_set_len(ScrArr *a, double length) { } if (scr_arr_ext_of(a)) scr_arr_ext_of(a)->sparse_len = scr_arr_sparse_lower_bound(scr_arr_ext_of(a)->sparse, scr_arr_ext_of(a)->sparse_len, next); } - } else if (a->len == 0 && next <= SCR_ARR_PRESIZE_LIMIT && (a->cap == 0 || scr_arr_inline(a))) { - scr_arr_reserve_exact(a, next); + } else if (next > a->len) { + /* The indices past the elements are holes. An empty array gives up + * its inline slots first, so its slots and states take one allocation. */ + if (a->len == 0 && next <= SCR_ARR_PRESIZE_LIMIT && (a->cap == 0 || scr_arr_inline(a))) { + if (scr_arr_inline(a)) { + a->spare &= (uint16_t)~SCR_ARR_INLINE_SLOTS; + a->cap = 0; + a->data = NULL; + } + scr_arr_need_states(a); + scr_arr_reserve_exact(a, next); + } else { + scr_arr_need_states(a); + } } a->len = next; } @@ -1583,11 +1734,24 @@ void scr_arr_copy_range_ex(ScrArr *dst, size_t dst_start, const ScrArr *src, scr_arr_refill_to(dst, dst_start + count); } +/* A destination without states takes a dense copy at its length, whose + * caller then raises the length over the copy; any other copy gives it + * states first. */ static void scr_arr_copy_range_states(ScrArr *dst, size_t dst_start, const ScrArr *src, size_t src_start, size_t count, bool reverse, bool materialize_holes) { - if (src != dst && src_start <= src->cap && count <= src->cap - src_start && - dst_start <= dst->cap && count <= dst->cap - dst_start) { + size_t src_end = scr_arr_has_states(src) ? src->cap : src->len; + bool fits = src != dst && src_start <= src_end && count <= src_end - src_start && + dst_start <= dst->cap && count <= dst->cap - dst_start; + if (!scr_arr_has_states(dst)) { + if (fits && dst_start == dst->len) { + scr_arr_copy_dense(dst, dst_start, src, src_start, count, reverse, materialize_holes); + return; + } + scr_arr_add_states(dst); + fits = fits && dst_start <= dst->cap && count <= dst->cap - dst_start; + } + if (fits) { const uint8_t *dst_states = scr_arr_states_end(dst); size_t empty = 0; while (empty < count && *scr_arr_state_in(dst_states, dst_start + empty) == SCR_ARR_HOLE) empty++; @@ -1673,11 +1837,12 @@ static ScrArr *scr_arr_fill_slot(ScrArr *a, uint64_t slot, uint8_t state, if (from >= until) return scr_arr_retain(a); bool refs = scr_elem_is_ref(a->elem); if (scr_arr_dense_ok(a, until, until - from)) scr_arr_grow_dense(a, until); + if (state != SCR_ARR_VALUE) scr_arr_need_states(a); if (!refs && until <= a->cap) { if (state != SCR_ARR_VALUE) scr_arr_cut(a, from); if (a->elem == SCR_ELEM_BOOL) memset(scr_arr_dense_at(a, from), (int)slot, until - from); else for (size_t i = from; i < until; i++) a->data[i] = slot; - memset(scr_arr_state_run(a, from, until - from), state, until - from); + if (scr_arr_has_states(a)) memset(scr_arr_state_run(a, from, until - from), state, until - from); } else { for (size_t i = from; i < until; i++) { uint64_t value = refs && state == SCR_ARR_VALUE ? scr_elem_retain_slot(a, slot) : slot; @@ -1831,7 +1996,7 @@ void *scr_arr_peek_ref(const ScrArr *a, double i) { size_t idx = (size_t)i; if ((double)idx == i) { if (idx < a->filled) return scr_slot_to_ptr(a->data[idx]); - if (idx >= a->len || *scr_arr_state_ptr(a, idx) != SCR_ARR_VALUE) return NULL; + if (idx >= a->len || scr_arr_dense_state(a, idx) != SCR_ARR_VALUE) return NULL; return scr_slot_to_ptr(a->data[idx]); } } @@ -1861,8 +2026,8 @@ bool scr_arr_index_eq(const ScrArr *a, double i, const ScrArr *b, double j) { if (i >= 0 && i < (double)a->cap && i == trunc(i) && j >= 0 && j < (double)b->cap && j == trunc(j)) { size_t ai = (size_t)i, bi = (size_t)j; - bool ap = ai < a->len && *scr_arr_state_ptr(a, ai) == SCR_ARR_VALUE; - bool bp = bi < b->len && *scr_arr_state_ptr(b, bi) == SCR_ARR_VALUE; + bool ap = ai < a->len && scr_arr_dense_state(a, ai) == SCR_ARR_VALUE; + bool bp = bi < b->len && scr_arr_dense_state(b, bi) == SCR_ARR_VALUE; if (!ap || !bp) return ap == bp; av = scr_arr_dense_get(a, ai); bv = scr_arr_dense_get(b, bi); @@ -1902,6 +2067,17 @@ static void scr_arr_set_slot(ScrArr *a, double i, uint64_t slot) { scr_arr_prop_set(a, i, slot); return; } + if (!scr_arr_has_states(a) && idx >= a->len) { + /* An append keeps the array without states; a write past the length + * leaves holes. */ + if (idx == a->len) { + scr_arr_store_owned(a, idx, slot); + a->len = (uint32_t)(idx + 1); + scr_arr_note_value(a, idx); + return; + } + scr_arr_add_states(a); + } if (idx >= a->len) a->len = idx + 1; /* Unlink-then-release: a release can trigger a cycle collection, which * must never see a heap edge whose count was already given up. */ @@ -1927,6 +2103,7 @@ void scr_arr_set_undefined(ScrArr *a, double i) { scr_arr_prop_set_undefined(a, i); return; } + scr_arr_need_states(a); if (idx >= a->len) a->len = idx + 1; scr_arr_replace_state_owned(a, idx, 0, SCR_ARR_UNDEFINED); } @@ -1976,7 +2153,7 @@ double scr_arr_push_many(ScrArr *a, size_t count, const uint64_t *slots) { scr_arr_grow_dense(a, next); if (count) { scr_arr_put_slots(a, from, count, slots); - scr_arr_set_states(scr_arr_state_run(a, from, count), SCR_ARR_VALUE, count); + if (scr_arr_has_states(a)) scr_arr_set_states(scr_arr_state_run(a, from, count), SCR_ARR_VALUE, count); } a->len = (uint32_t)next; /* The appended states are values: a bound at the old length moves to @@ -1985,6 +2162,7 @@ double scr_arr_push_many(ScrArr *a, size_t count, const uint64_t *slots) { else scr_arr_refill(a); return (double)next; } + scr_arr_need_states(a); for (size_t i = 0; i < count; i++) scr_arr_store_owned(a, from + i, slots[i]); a->len = (uint32_t)next; scr_arr_refill(a); @@ -2021,6 +2199,7 @@ double scr_arr_push_spread(ScrArr *a, const ScrArr *src) { scr_arr_refill(a); return SCR_ARR_LEN_F64(a->len); } + scr_arr_need_states(a); a->len = old_len + add; for (size_t i = 0; i < add; i++) { uint64_t slot; @@ -2065,9 +2244,10 @@ double scr_arr_concat_copy(ScrArr *a, const ScrArr *src) { scr_arr_refill(a); return SCR_ARR_LEN_F64(a->len); } + scr_arr_need_states(a); a->len = old_len + add; for (size_t i = 0; i < src->cap; i++) { - if (*scr_arr_state_ptr(src, i) != SCR_ARR_HOLE) { + if (scr_arr_dense_state(src, i) != SCR_ARR_HOLE) { scr_arr_copy_index(a, old_len + i, src, i); } } @@ -2121,7 +2301,7 @@ static double scr_arr_unshift_slot(ScrArr *a, uint64_t slot) { /* The prefix stays put in front of the new first value. */ size_t f = a->filled; a->data[0] = slot; - *scr_arr_state_ptr(a, 0) = SCR_ARR_VALUE; + if (scr_arr_has_states(a)) *scr_arr_state_ptr(a, 0) = SCR_ARR_VALUE; a->len++; a->filled = (uint32_t)(f + 1); return SCR_ARR_LEN_F64(a->len); @@ -2132,7 +2312,7 @@ static double scr_arr_unshift_slot(ScrArr *a, uint64_t slot) { size_t f = a->filled; scr_arr_move_dense(a, 1, 0, old_len); scr_arr_dense_put(a, 0, slot); - *scr_arr_state_ptr(a, 0) = SCR_ARR_VALUE; + if (scr_arr_has_states(a)) *scr_arr_state_ptr(a, 0) = SCR_ARR_VALUE; a->len++; a->filled = (uint32_t)(f + 1); return SCR_ARR_LEN_F64(a->len); @@ -2173,7 +2353,7 @@ double scr_arr_unshift_many(ScrArr *a, size_t count, const uint64_t *slots) { if (scr_arr_is_dense(a) && scr_arr_open_head(a, count)) { size_t f = a->filled; scr_arr_put_slots(a, 0, count, slots); - memset(scr_arr_state_run(a, 0, count), SCR_ARR_VALUE, count); + if (scr_arr_has_states(a)) memset(scr_arr_state_run(a, 0, count), SCR_ARR_VALUE, count); a->len = old_len + count; a->filled = (uint32_t)(f + count); return SCR_ARR_LEN_F64(a->len); @@ -2183,7 +2363,7 @@ double scr_arr_unshift_many(ScrArr *a, size_t count, const uint64_t *slots) { size_t f = a->filled; scr_arr_move_dense(a, count, 0, old_len); scr_arr_put_slots(a, 0, count, slots); - memset(scr_arr_state_run(a, 0, count), SCR_ARR_VALUE, count); + if (scr_arr_has_states(a)) memset(scr_arr_state_run(a, 0, count), SCR_ARR_VALUE, count); a->len = old_len + count; a->filled = (uint32_t)(f + count); return SCR_ARR_LEN_F64(a->len); @@ -2210,6 +2390,10 @@ double scr_arr_unshift_spread(ScrArr *a, const ScrArr *src) { size_t add = src->len; if (add == 0) return (double)old_len; if (add > SCR_ARR_MAX_LENGTH - old_len) scr_arr_oom(); + /* Holes and present undefined in the source become present undefined: + * the receiver needs states, before its elements move. */ + if (!scr_arr_has_states(a) && (!scr_arr_is_dense(src) || !scr_arr_dense_run_values(src, 0, add))) + scr_arr_add_states(a); if (scr_arr_is_dense(a) && scr_arr_is_dense(src) && scr_arr_open_head(a, add)) { /* A self-spread reads the original values behind the opened room. */ scr_arr_copy_dense(a, 0, src, src == a ? add : 0, add, false, true); @@ -2257,6 +2441,15 @@ double scr_arr_unshift_spread(ScrArr *a, const ScrArr *src) { ScrArr *scr_arr_reverse(ScrArr *a) { size_t len = a->len; + if (!scr_arr_has_states(a)) { + for (size_t i = 0; i < len / 2; i++) { + size_t j = len - i - 1; + uint64_t slot = scr_arr_dense_get(a, i); + scr_arr_dense_put(a, i, scr_arr_dense_get(a, j)); + scr_arr_dense_put(a, j, slot); + } + return scr_arr_retain(a); + } if (scr_arr_is_dense(a)) { /* A hole-free array stays hole-free; any other prefix is recomputed. */ if (a->filled != len) scr_arr_cut(a, 0); @@ -2292,6 +2485,13 @@ uint8_t scr_arr_pop_state(ScrArr *a, uint64_t *slot_out) { if (slot_out) *slot_out = 0; if (a->len == 0) return SCR_ARR_HOLE; size_t idx = a->len - 1; + if (!scr_arr_has_states(a)) { + if (slot_out) *slot_out = scr_arr_dense_get(a, idx); + scr_arr_dense_put(a, idx, 0); + scr_arr_cut(a, idx); + a->len = (uint32_t)idx; + return SCR_ARR_VALUE; + } uint64_t slot = 0; uint8_t state = SCR_ARR_HOLE; if (!scr_arr_take_state(a, idx, &slot, &state)) state = SCR_ARR_HOLE; @@ -2327,7 +2527,7 @@ uint8_t scr_arr_shift_state(ScrArr *a, uint64_t *slot_out) { if (a->len == 0) return SCR_ARR_HOLE; size_t old_len = a->len; if (scr_arr_is_dense(a)) { - uint8_t state = *scr_arr_state_ptr(a, 0); + uint8_t state = scr_arr_dense_state(a, 0); if (slot_out && state == SCR_ARR_VALUE) *slot_out = scr_arr_dense_get(a, 0); /* The first slot becomes head room; the tail stays where it is. */ if (scr_arr_drop_front(a, 1)) return state; @@ -2398,13 +2598,15 @@ ScrArr *scr_arr_splice(ScrArr *a, double start, double deleteCount) { a->elem == SCR_ELEM_REF ? scr_arr_new_ops(scr_arr_ops(a), hint ? hint : 1) : scr_arr_new(a->elem, hint ? hint : 1); - out->len = n; size_t old_len = a->len; if (scr_arr_is_dense(a)) { if (n) { + /* The removed elements keep their states unless all are values. */ + if (!scr_arr_dense_run_values(a, from, n)) scr_arr_add_states(out); memcpy(out->data, scr_arr_dense_at(a, from), n * scr_arr_slot_width(a)); - memcpy(scr_arr_state_run(out, 0, n), scr_arr_state_run(a, from, n), n); + if (scr_arr_has_states(out)) memcpy(scr_arr_state_run(out, 0, n), scr_arr_state_run(a, from, n), n); } + out->len = n; scr_arr_refill(out); /* A removed prefix becomes head room. */ if (from == 0 && scr_arr_drop_front(a, n)) return out; @@ -2418,6 +2620,8 @@ ScrArr *scr_arr_splice(ScrArr *a, double start, double deleteCount) { else scr_arr_refill(a); return out; } + scr_arr_add_states(out); + out->len = n; ScrArrStorage old = scr_arr_take_storage(a); a->len = 0; for (size_t i = 0; i < old.cap; i++) { @@ -2475,12 +2679,17 @@ ScrArr *scr_arr_splice_insert(ScrArr *a, double start, double deleteCount, * is needed only by direct self-spread runtime callers. */ ScrArr *snapshot = items == a ? scr_arr_slice(a, 0, INFINITY) : NULL; if (snapshot) items = snapshot; + /* Holes and present undefined among the items become present + * undefined: the receiver needs states, before its elements move. */ + if (!scr_arr_dense_run_values(items, 0, add)) scr_arr_need_states(a); ScrArr *removed = a->elem == SCR_ELEM_REF ? scr_arr_new_ops(scr_arr_ops(a), n) : scr_arr_new(a->elem, n); if (n) { + if (!scr_arr_dense_run_values(a, from, n)) scr_arr_add_states(removed); memcpy(removed->data, scr_arr_dense_at(a, from), n * scr_arr_slot_width(a)); - memcpy(scr_arr_state_run(removed, 0, n), scr_arr_state_run(a, from, n), n); + if (scr_arr_has_states(removed)) + memcpy(scr_arr_state_run(removed, 0, n), scr_arr_state_run(a, from, n), n); } removed->len = n; scr_arr_refill(removed); @@ -2538,9 +2747,8 @@ void scr_arr_splice_drop(ScrArr *a, double start, double deleteCount, size_t cou removed = scr_mem_alloc(n * sizeof(uint64_t)); if (!removed) scr_arr_oom(); } - const uint8_t *states = scr_arr_states_end(a); for (size_t i = from; i < from + n; i++) { - if (*scr_arr_state_in(states, i) == SCR_ARR_VALUE) removed[kept++] = a->data[i]; + if (scr_arr_dense_state(a, i) == SCR_ARR_VALUE) removed[kept++] = a->data[i]; } } size_t old_len = a->len; @@ -2555,7 +2763,7 @@ void scr_arr_splice_drop(ScrArr *a, double start, double deleteCount, size_t cou scr_arr_move_dense(a, from + count, from + n, old_len - from - n); if (count) { scr_arr_put_slots(a, from, count, slots); - memset(scr_arr_state_run(a, from, count), SCR_ARR_VALUE, count); + if (scr_arr_has_states(a)) memset(scr_arr_state_run(a, from, count), SCR_ARR_VALUE, count); } if (next_len < old_len) scr_arr_clear_dense(a, next_len, old_len - next_len); a->len = next_len; @@ -2580,13 +2788,15 @@ ScrArr *scr_arr_flat_copy(const ScrArr *a, ScrArr *out, bool flatten) { for (size_t j = 0; j < inner->len; j++) { if (scr_arr_state_at(inner, j, NULL) == SCR_ARR_HOLE) continue; if (out->len == SCR_ARR_MAX_LENGTH) scr_arr_oom(); - size_t at = out->len++; + size_t at = out->len; scr_arr_copy_index(out, at, inner, j); + out->len = (uint32_t)(at + 1); } } else { if (out->len == SCR_ARR_MAX_LENGTH) scr_arr_oom(); - size_t at = out->len++; + size_t at = out->len; scr_arr_copy_index(out, at, a, i); + out->len = (uint32_t)(at + 1); } } scr_arr_refill(out); @@ -2611,10 +2821,12 @@ static bool scr_arr_ref_eq(const ScrArr *a, uint64_t slot, void *v) { return p == v; } +#define SCR_ARR_SCAN_VALUE(states, i) (!(states) || *scr_arr_state_in((states), (i)) == SCR_ARR_VALUE) + double scr_arr_index_of_f64(ScrArr *a, double v) { - const uint8_t *states = scr_arr_states_end(a); + const uint8_t *states = scr_arr_scan_states(a); for (size_t i = 0; i < a->cap && i < a->len; i++) { - if (*scr_arr_state_in(states, i) == SCR_ARR_VALUE && scr_slot_to_f64(a->data[i]) == v) return (double)i; + if (SCR_ARR_SCAN_VALUE(states, i) && scr_slot_to_f64(a->data[i]) == v) return (double)i; } for (size_t i = 0; i < SCR_AX(a)->sparse_len; i++) { if (SCR_AX(a)->sparse[i].state == SCR_ARR_VALUE && scr_slot_to_f64(SCR_AX(a)->sparse[i].slot) == v) return (double)SCR_AX(a)->sparse[i].index; @@ -2623,9 +2835,9 @@ double scr_arr_index_of_f64(ScrArr *a, double v) { } double scr_arr_index_of_bool(ScrArr *a, bool v) { - const uint8_t *states = scr_arr_states_end(a); + const uint8_t *states = scr_arr_scan_states(a); for (size_t i = 0; i < a->cap && i < a->len; i++) { - if (*scr_arr_state_in(states, i) == SCR_ARR_VALUE && ((scr_arr_dense_get(a, i) != 0) == v)) return (double)i; + if (SCR_ARR_SCAN_VALUE(states, i) && ((scr_arr_dense_get(a, i) != 0) == v)) return (double)i; } for (size_t i = 0; i < SCR_AX(a)->sparse_len; i++) { if (SCR_AX(a)->sparse[i].state == SCR_ARR_VALUE && (SCR_AX(a)->sparse[i].slot != 0) == v) return (double)SCR_AX(a)->sparse[i].index; @@ -2634,9 +2846,9 @@ double scr_arr_index_of_bool(ScrArr *a, bool v) { } double scr_arr_index_of_ref(ScrArr *a, void *v) { - const uint8_t *states = scr_arr_states_end(a); + const uint8_t *states = scr_arr_scan_states(a); for (size_t i = 0; i < a->cap && i < a->len; i++) { - if (*scr_arr_state_in(states, i) == SCR_ARR_VALUE && scr_arr_ref_eq(a, a->data[i], v)) return (double)i; + if (SCR_ARR_SCAN_VALUE(states, i) && scr_arr_ref_eq(a, a->data[i], v)) return (double)i; } for (size_t i = 0; i < SCR_AX(a)->sparse_len; i++) { if (SCR_AX(a)->sparse[i].state == SCR_ARR_VALUE && scr_arr_ref_eq(a, SCR_AX(a)->sparse[i].slot, v)) return (double)SCR_AX(a)->sparse[i].index; @@ -2645,9 +2857,9 @@ double scr_arr_index_of_ref(ScrArr *a, void *v) { } bool scr_arr_includes_f64(ScrArr *a, double v) { - const uint8_t *states = scr_arr_states_end(a); + const uint8_t *states = scr_arr_scan_states(a); for (size_t i = 0; i < a->cap && i < a->len; i++) { - if (*scr_arr_state_in(states, i) != SCR_ARR_VALUE) continue; + if (!SCR_ARR_SCAN_VALUE(states, i)) continue; double x = scr_slot_to_f64(a->data[i]); if (x == v || (x != x && v != v)) return true; /* SameValueZero: NaN hits */ } @@ -2667,9 +2879,9 @@ bool scr_arr_includes_ref(ScrArr *a, void *v) { if (a->elem == SCR_ELEM_REF && scr_arr_ops(a)->retain == scr_dyn_retain_v) { const ScrDyn *needle = (const ScrDyn *)v; if (needle->kind == SCR_DYN_NUM && needle->v.num != needle->v.num) { - const uint8_t *states = scr_arr_states_end(a); + const uint8_t *states = scr_arr_scan_states(a); for (size_t i = 0; i < a->cap && i < a->len; i++) { - if (*scr_arr_state_in(states, i) != SCR_ARR_VALUE) continue; + if (!SCR_ARR_SCAN_VALUE(states, i)) continue; const ScrDyn *d = (const ScrDyn *)scr_slot_to_ptr(a->data[i]); if (d->kind == SCR_DYN_NUM && d->v.num != d->v.num) return true; } @@ -2730,22 +2942,26 @@ ScrArr *scr_arr_slice(ScrArr *a, double start, double end) { a->elem == SCR_ELEM_REF ? scr_arr_new_ops(scr_arr_ops(a), hint ? hint : 1) : scr_arr_new(a->elem, hint ? hint : 1); - out->len = n; if (to <= a->cap && to <= a->filled && a->elem != SCR_ELEM_BOOL && !scr_elem_is_ref(a->elem)) { /* Numbers below the source's hole-free prefix: values only. */ scr_arr_copy_slots(out->data, a->data + from, n); - scr_arr_set_states(scr_arr_states_end_w(out, 8) - n, SCR_ARR_VALUE, n); + out->len = (uint32_t)n; out->filled = (uint32_t)n; return out; } if (to <= a->cap) { scr_arr_copy_dense(out, 0, a, from, n, false, false); + out->len = (uint32_t)n; /* States copied from below the source's bound are present values. */ if (n > 0 && from < a->filled) out->filled = (uint32_t)((to < a->filled ? to : a->filled) - from); scr_arr_refill(out); return out; } + /* A range reaching the sparse entries: the source has states, and so + * does the copy. */ + scr_arr_add_states(out); + out->len = (uint32_t)n; const uint8_t *states = scr_arr_states_end(a); for (size_t i = 0; i < a->cap && i < to; i++) { if (i < from || *scr_arr_state_in(states, i) == SCR_ARR_HOLE) continue; @@ -2875,8 +3091,9 @@ ScrArr *scr_arr_sort_primitive(ScrArr *a, bool copy) { if (a->len == 0) return copy ? scr_arr_new(a->elem, 0) : scr_arr_retain(a); size_t dense = a->len < a->cap ? a->len : a->cap; size_t count = 0, undefined_count = 0; - const uint8_t *states = scr_arr_states_end(a); - for (size_t i = 0; i < dense; i++) { + const uint8_t *states = scr_arr_scan_states(a); + if (!states) count = dense; + for (size_t i = 0; states && i < dense; i++) { count += *scr_arr_state_in(states, i) == SCR_ARR_VALUE; undefined_count += *scr_arr_state_in(states, i) == SCR_ARR_UNDEFINED; } @@ -2889,7 +3106,7 @@ ScrArr *scr_arr_sort_primitive(ScrArr *a, bool copy) { if (count && !values) scr_arr_oom(); size_t used = 0; for (size_t i = 0; i < dense; i++) - if (*scr_arr_state_in(states, i) == SCR_ARR_VALUE) values[used++] = scr_arr_dense_get(a, i); + if (SCR_ARR_SCAN_VALUE(states, i)) values[used++] = scr_arr_dense_get(a, i); for (size_t i = 0; i < SCR_AX(a)->sparse_len; i++) if (SCR_AX(a)->sparse[i].state == SCR_ARR_VALUE) values[used++] = SCR_AX(a)->sparse[i].slot; ScrNumberSortKey *numbers = NULL; @@ -2911,8 +3128,10 @@ ScrArr *scr_arr_sort_primitive(ScrArr *a, bool copy) { } size_t present = copy ? a->len : count + undefined_count; ScrArr *out = copy ? scr_arr_new(a->elem, present) : a; - if (copy) out->len = a->len; - if (!copy && scr_arr_is_dense(a)) { + if (!copy && !states) { + /* Values only, in place. */ + if (count) scr_arr_put_slots(a, 0, count, values); + } else if (!copy && scr_arr_is_dense(a)) { if (count) scr_arr_put_slots(a, 0, count, values); scr_arr_cut(a, count); memset(scr_arr_state_run(a, 0, count), SCR_ARR_VALUE, count); @@ -2920,14 +3139,20 @@ ScrArr *scr_arr_sort_primitive(ScrArr *a, bool copy) { scr_arr_clear_dense(a, count + undefined_count, a->len - count - undefined_count); if (undefined_count) memset(scr_arr_dense_at(a, count), 0, undefined_count * scr_arr_slot_width(a)); + } else if (copy) { + /* The values, then present undefined for every other index. */ + for (size_t i = 0; i < count; i++) values[i] = scr_elem_retain_slot(a, values[i]); + scr_arr_push_many(out, count, values); + for (size_t i = count; i < present; i++) { + scr_arr_store_state_owned(out, i, 0, SCR_ARR_UNDEFINED); + out->len = (uint32_t)(i + 1); + } } else { - ScrArrStorage old = {0}; - if (!copy) old = scr_arr_take_storage(a); - for (size_t i = 0; i < count; i++) - scr_arr_store_owned(out, i, copy ? scr_elem_retain_slot(a, values[i]) : values[i]); + ScrArrStorage old = scr_arr_take_storage(a); + for (size_t i = 0; i < count; i++) scr_arr_store_owned(out, i, values[i]); for (size_t i = count; i < present; i++) scr_arr_store_state_owned(out, i, 0, SCR_ARR_UNDEFINED); - if (!copy) scr_arr_free_storage(&old); + scr_arr_free_storage(&old); } free(values); scr_arr_refill(out); @@ -4606,7 +4831,7 @@ void scr_arr_sort_values(ScrArr *a, double count, ScrClosure *f, uint32_t arity, * holes, no storage-level undefined and no sparse tail. */ static bool scr_arr_dense_values(const ScrArr *a) { if (!scr_arr_is_dense(a)) return false; - if (a->filled == a->len) return true; + if (a->filled == a->len || !scr_arr_has_states(a)) return true; const uint8_t *states = scr_arr_states_end(a); bool all = true; for (size_t i = 0; i < a->len; i++) all &= *scr_arr_state_in(states, i) == SCR_ARR_VALUE; @@ -4683,16 +4908,16 @@ bool scr_arr_sort_dense(ScrArr *a, ScrClosure *f, uint32_t arity, bool owned, in } else if (a->len >= n && a->cap >= n) { if (!retained) { memcpy(a->data, src, n * sizeof(*src)); - memset(scr_arr_state_run(a, 0, n), SCR_ARR_VALUE, n); + if (scr_arr_has_states(a)) memset(scr_arr_state_run(a, 0, n), SCR_ARR_VALUE, n); } else { /* Publish each new edge before releasing the value it replaces: a * release can collect cycles. The buffer still holds the slots not * yet written. */ for (size_t i = 0; i < n; i++) { - bool held = *scr_arr_state_ptr(a, i) == SCR_ARR_VALUE; + bool held = scr_arr_dense_state(a, i) == SCR_ARR_VALUE; uint64_t old = held ? a->data[i] : 0; a->data[i] = src[i]; - *scr_arr_state_ptr(a, i) = SCR_ARR_VALUE; + if (scr_arr_has_states(a)) *scr_arr_state_ptr(a, i) = SCR_ARR_VALUE; if (held) scr_elem_release(a, old); } } diff --git a/packages/runtime/src/scr_array.h b/packages/runtime/src/scr_array.h index 6c41b3dbd..697912934 100644 --- a/packages/runtime/src/scr_array.h +++ b/packages/runtime/src/scr_array.h @@ -82,7 +82,7 @@ typedef struct ScrArr { * header. Kept beside `elem` so the RC fast paths test it with one load. */ uint8_t cyc; /* The low SCR_ARR_OPS_SHIFT bits: storage flags (SCR_ARR_INLINE_SLOTS, - * SCR_ARR_HEAD_ROOM, SCR_ARR_UNSHIFTED, SCR_ARR_HAS_EXT). The rest: the + * SCR_ARR_HEAD_ROOM, SCR_ARR_UNSHIFTED, SCR_ARR_HAS_EXT, SCR_ARR_STATES). The rest: the * registry id of the element type's ops (scr_arr_ops), 0 for every kind * but SCR_ELEM_REF. */ uint16_t spare; @@ -94,8 +94,9 @@ typedef struct ScrArr { * 0 is always valid. */ uint32_t filled; /* Dense values: `cap` 8-byte slots, or `cap` bytes for SCR_ELEM_BOOL - * (scr_arr_slot_width), followed in the same allocation by one state byte - * per slot in reverse slot order (scr_arr_states_end). Sparse entries and properties keep 8-byte + * (scr_arr_slot_width), followed in the same allocation, when the array + * has states (SCR_ARR_STATES), by one state byte per slot in reverse slot + * order (scr_arr_states_end). Sparse entries and properties keep 8-byte * slots for every kind; a bool slot is 0 or 1 wherever it lives. Rarely * used storage (sparse indices, noncanonical numeric properties, * match-result metadata) lives out of line, and an array that has it @@ -124,14 +125,17 @@ typedef struct ScrArrExt { /* The LLVM backend mirrors ScrArr as %ScrArr: its inline retain/release read * `cyc` (field 4, i8) to decide mark-live / candidate buffering exactly like * scr_arr_retain and scr_arr_release; its inline element access uses len and - * cap (fields 1 and 2, i32), filled (field 6, i32) and data (field 7), and - * finds the state of slot i at data + cap * (slot width + 1) - 1 - i. */ + * cap (fields 1 and 2, i32), spare (field 5, i16, for SCR_ARR_STATES), + * filled (field 6, i32) and data (field 7), and finds the state of slot i + * of an array with states at data + cap * (slot width + 1) - 1 - i. */ _Static_assert(offsetof(ScrArr, len) == sizeof(size_t) && sizeof(((ScrArr *)0)->len) == 4, "emitted length reads load ScrArr.len as %ScrArr field 1, an i32"); _Static_assert(offsetof(ScrArr, cap) == sizeof(size_t) + 4 && sizeof(((ScrArr *)0)->cap) == 4, "emitted capacity reads load ScrArr.cap as %ScrArr field 2, an i32"); _Static_assert(offsetof(ScrArr, cyc) == sizeof(size_t) + 9, "inline RC fast paths read ScrArr.cyc as %ScrArr field 4"); +_Static_assert(offsetof(ScrArr, spare) == sizeof(size_t) + 10, + "inline element access reads ScrArr.spare as %ScrArr field 5"); _Static_assert(offsetof(ScrArr, filled) == sizeof(size_t) + 12, "inline element access reads ScrArr.filled as %ScrArr field 6"); _Static_assert(offsetof(ScrArr, data) == sizeof(size_t) + 16, @@ -145,11 +149,21 @@ _Static_assert(sizeof(ScrArr) == sizeof(size_t) + 16 + sizeof(void *), * count. Unshifted: an unshift had to move every element. Has ext: the word * before the dense allocation (head room included) points at the array's * ScrArrExt; such an array always has an allocation of its own, without - * inline slots. */ + * inline slots. States: the dense slots carry state bytes (see below). */ #define SCR_ARR_INLINE_SLOTS 1u #define SCR_ARR_HEAD_ROOM 2u #define SCR_ARR_UNSHIFTED 4u #define SCR_ARR_HAS_EXT 8u +#define SCR_ARR_STATES 16u + +/* An array starts without state bytes. Without them, every index below + * `len` is a present value in dense storage: len <= cap, there are no + * sparse entries, and `filled` equals `len` whenever emitted code can read + * the array (its inline reads load a state only at or past `filled`). The + * first hole or present `undefined` below the length, + * a length set past the elements, or a sparse entry gives the array its + * states (scr_arr_add_states), which it keeps. */ +static inline bool scr_arr_has_states(const ScrArr *a) { return (a->spare & SCR_ARR_STATES) != 0; } /* The start of the dense slots' allocation: `data` less any head room. */ static inline uint64_t *scr_arr_block(const ScrArr *a) { @@ -173,7 +187,7 @@ static inline struct ScrArrExt *scr_arr_ext_of(const ScrArr *a) { * Id 0 is no ops; ids are assigned on each descriptor's first array, in * any thread. A program with more element types than the field holds keeps * the ops of the rest in the array's ext (SCR_ARR_OPS_OVERFLOW). */ -#define SCR_ARR_OPS_SHIFT 4 +#define SCR_ARR_OPS_SHIFT 5 #define SCR_ARR_OPS_OVERFLOW (UINT16_MAX >> SCR_ARR_OPS_SHIFT) extern const ScrArrOps *scr_arr_ops_table[SCR_ARR_OPS_OVERFLOW]; /* Assigns the id on a descriptor's first array; out of line, so the id @@ -197,17 +211,20 @@ static inline const ScrArrOps *scr_arr_ops(const ScrArr *a) { return scr_arr_ops_table[id]; } -/* Bytes per dense value slot. */ -static inline size_t scr_arr_slot_width(const ScrArr *a) { - return a->elem == SCR_ELEM_BOOL ? 1 : sizeof(uint64_t); +/* Bytes per dense value slot of element kind `elem`. */ +static inline size_t scr_elem_slot_width(uint8_t elem) { + return elem == SCR_ELEM_BOOL ? 1 : sizeof(uint64_t); } -/* The dense states, one byte per slot, share the values' allocation (inline - * slots included). They follow the `cap` value slots in reverse slot order: - * the state of slot i is the byte i + 1 before scr_arr_states_end(a). Removing - * slots from the front (head room: `data` advances, `cap` drops) leaves every - * remaining state at its address. The address is computed on integers, so a - * NULL `data` stays NULL. */ +static inline size_t scr_arr_slot_width(const ScrArr *a) { return scr_elem_slot_width(a->elem); } + +/* The dense states of an array with states, one byte per slot, share the + * values' allocation (inline slots included). They follow the `cap` value + * slots in reverse slot order: the state of slot i is the byte i + 1 before + * scr_arr_states_end(a). Removing slots from the front (head room: `data` + * advances, `cap` drops) leaves every remaining state at its address. The + * address is computed on integers, so a NULL `data` stays NULL. Arrays + * without states have no bytes there. */ static inline uint8_t *scr_arr_states_end(const ScrArr *a) { return (uint8_t *)((uintptr_t)a->data + (uintptr_t)a->cap * (scr_arr_slot_width(a) + 1)); } @@ -233,6 +250,26 @@ static inline uint8_t *scr_arr_state_run(const ScrArr *a, size_t from, size_t co return scr_arr_states_end(a) - from - count; } +/* The state of dense slot `index` (< cap), with or without state bytes. */ +static inline uint8_t scr_arr_dense_state(const ScrArr *a, size_t index) { + if (!scr_arr_has_states(a)) return index < a->len ? SCR_ARR_VALUE : SCR_ARR_HOLE; + return *scr_arr_state_ptr(a, index); +} + +/* The dense states of `a` for a scan (scr_arr_state_in), or NULL when it + * has none: every index below its length is then a value. */ +static inline const uint8_t *scr_arr_scan_states(const ScrArr *a) { + return scr_arr_has_states(a) ? scr_arr_states_end(a) : NULL; +} + +/* Give `a` state bytes: present values below the length, holes above it. + * May move the dense slots. */ +void scr_arr_add_states(ScrArr *a); + +static inline void scr_arr_need_states(ScrArr *a) { + if (!scr_arr_has_states(a)) scr_arr_add_states(a); +} + /* The dense value at `index` (< cap) as an 8-byte slot, and its store. */ static inline uint64_t scr_arr_dense_get(const ScrArr *a, size_t index) { if (a->elem == SCR_ELEM_BOOL) return ((const uint8_t *)a->data)[index]; @@ -270,7 +307,8 @@ static inline size_t scr_arr_value_run(const uint8_t *states_end, size_t f, size static inline void scr_arr_refill(ScrArr *a) { size_t end = a->len < a->cap ? a->len : a->cap; size_t f = a->filled; - if (f < end) f = scr_arr_value_run(scr_arr_states_end(a), f, end); + if (!scr_arr_has_states(a)) f = end; + else if (f < end) f = scr_arr_value_run(scr_arr_states_end(a), f, end); a->filled = (uint32_t)f; } @@ -302,8 +340,9 @@ ScrArr *scr_arr_new_ops(const ScrArrOps *ops, size_t initial_cap); ScrArr *scr_arr_new_small(ScrElemKind elem); ScrArr *scr_arr_new_ops_small(const ScrArrOps *ops); /* scr_arr_new(elem, cap) for a known final length: the `cap` slots share - * the array's own block (one allocation). Returns +1. */ -ScrArr *scr_arr_new_inline(ScrElemKind elem, size_t cap); + * the array's own block (one allocation). With `states`, the array starts + * with states (for holes or present undefined). Returns +1. */ +ScrArr *scr_arr_new_inline(ScrElemKind elem, size_t cap, bool states); static inline ScrArr *scr_arr_retain(ScrArr *a) { if (a->rc != SIZE_MAX) { diff --git a/packages/runtime/src/scr_bytes.c b/packages/runtime/src/scr_bytes.c index 7a6a76cf9..7fec7bd07 100644 --- a/packages/runtime/src/scr_bytes.c +++ b/packages/runtime/src/scr_bytes.c @@ -3122,7 +3122,7 @@ static void scr_bytes_fill_from_arr(uint8_t *out, ScrBytesElem elem, const ScrAr /* Sparse tails and explicit undefined retain ordinary iteration * semantics. Numeric array slots own no conversion callbacks. */ if (index < arr->cap) { - if (*scr_arr_state_ptr(arr, index) == SCR_ARR_VALUE) + if (scr_arr_dense_state(arr, index) == SCR_ARR_VALUE) memcpy(&values[i], &arr->data[index], sizeof(double)); else values[i] = NAN; } else values[i] = scr_arr_get_number(arr, (double)index); diff --git a/packages/runtime/src/scr_copying.c b/packages/runtime/src/scr_copying.c index df4da1738..125685dc6 100644 --- a/packages/runtime/src/scr_copying.c +++ b/packages/runtime/src/scr_copying.c @@ -25,10 +25,19 @@ static ScrArr *copying_arr_new_like(const ScrArr *a, size_t cap) { ScrArr *out = a->elem == SCR_ELEM_REF ? scr_arr_new_ops(scr_arr_ops(a), cap ? cap : 1) : scr_arr_new(a->elem, cap ? cap : 1); - out->len = cap; return out; } +/* Copy [from, from + count) of `src` to the end of `out` and raise its + * length over the copy. */ +static void copying_arr_append_range(ScrArr *out, const ScrArr *src, size_t from, size_t count, + bool reverse) { + size_t at = out->len; + scr_arr_copy_range_ex(out, at, src, from, count, reverse, true); + out->len = (uint32_t)(at + count); + scr_arr_refill(out); +} + static uint64_t copying_arr_retain_slot(const ScrArr *a, uint64_t slot) { if (!copying_elem_is_ref(a->elem)) return slot; void *p = copying_slot_to_ptr(slot); @@ -41,7 +50,7 @@ static uint64_t copying_arr_retain_slot(const ScrArr *a, uint64_t slot) { ScrArr *scr_arr_to_reversed(const ScrArr *a) { ScrArr *out = copying_arr_new_like(a, a->len); - scr_arr_copy_range_ex(out, 0, a, 0, a->len, true, true); + copying_arr_append_range(out, a, 0, a->len, true); return out; } @@ -60,10 +69,9 @@ ScrArr *scr_arr_to_spliced(const ScrArr *a, double start, } size_t out_len = a->len - ndelete + items->len; ScrArr *out = copying_arr_new_like(a, out_len); - scr_arr_copy_range_ex(out, 0, a, 0, from, false, true); - scr_arr_copy_range_ex(out, from, items, 0, items->len, false, true); - scr_arr_copy_range_ex(out, from + items->len, a, from + ndelete, - a->len - from - ndelete, false, true); + copying_arr_append_range(out, a, 0, from, false); + copying_arr_append_range(out, items, 0, items->len, false); + copying_arr_append_range(out, a, from + ndelete, a->len - from - ndelete, false); return out; } @@ -88,7 +96,7 @@ ScrArr *scr_arr_with_f64(ScrArr *a, double index, double value) { size_t i; if (!copying_arr_with_index(a, index, &i)) return NULL; ScrArr *out = copying_arr_new_like(a, a->len); - scr_arr_copy_range_ex(out, 0, a, 0, a->len, false, true); + copying_arr_append_range(out, a, 0, a->len, false); scr_arr_set_f64(out, (double)i, value); return out; } @@ -97,7 +105,7 @@ ScrArr *scr_arr_with_bool(ScrArr *a, double index, bool value) { size_t i; if (!copying_arr_with_index(a, index, &i)) return NULL; ScrArr *out = copying_arr_new_like(a, a->len); - scr_arr_copy_range_ex(out, 0, a, 0, a->len, false, true); + copying_arr_append_range(out, a, 0, a->len, false); scr_arr_set_bool(out, (double)i, value); return out; } @@ -106,7 +114,7 @@ ScrArr *scr_arr_with_ref(ScrArr *a, double index, void *value) { size_t i; if (!copying_arr_with_index(a, index, &i)) return NULL; ScrArr *out = copying_arr_new_like(a, a->len); - scr_arr_copy_range_ex(out, 0, a, 0, a->len, false, true); + copying_arr_append_range(out, a, 0, a->len, false); uint64_t retained = copying_arr_retain_slot(a, copying_slot_from_ptr(value)); scr_arr_set_ref(out, (double)i, copying_slot_to_ptr(retained)); @@ -117,7 +125,7 @@ ScrArr *scr_arr_with_undefined(ScrArr *a, double index) { size_t i; if (!copying_arr_with_index(a, index, &i)) return NULL; ScrArr *out = copying_arr_new_like(a, a->len); - scr_arr_copy_range_ex(out, 0, a, 0, a->len, false, true); + copying_arr_append_range(out, a, 0, a->len, false); scr_arr_set_undefined(out, (double)i); return out; } diff --git a/packages/runtime/src/scr_cycle.c b/packages/runtime/src/scr_cycle.c index 5e5029e14..5956d5dd2 100644 --- a/packages/runtime/src/scr_cycle.c +++ b/packages/runtime/src/scr_cycle.c @@ -1244,9 +1244,9 @@ void scr_cyc_on_release(void *obj) { #define SCR_CYC_ARRAY_WALK(name, visit) \ static __attribute__((noinline)) void name(const ScrArr *arr, void *ctx) { \ size_t end = arr->len < arr->cap ? arr->len : arr->cap; \ - size_t filled = arr->filled < end ? arr->filled : end; \ + const uint8_t *states = scr_arr_scan_states(arr); \ + size_t filled = !states || arr->filled > end ? end : arr->filled; \ for (size_t i = 0; i < filled; i++) visit((void *)(uintptr_t)arr->data[i], ctx); \ - const uint8_t *states = scr_arr_states_end(arr); \ for (size_t i = filled; i < end; i++) \ if (*scr_arr_state_in(states, i) == SCR_ARR_VALUE) \ visit((void *)(uintptr_t)arr->data[i], ctx); \ diff --git a/packages/runtime/src/scr_itable.c b/packages/runtime/src/scr_itable.c index a333aab95..25a074f49 100644 --- a/packages/runtime/src/scr_itable.c +++ b/packages/runtime/src/scr_itable.c @@ -472,7 +472,7 @@ static ScrArrState scr_itable_element(ScrITableElements *w, size_t i, uint64_t * } if (i < a->cap) { *slot = a->data[i]; - return (ScrArrState)*scr_arr_state_ptr(a, i); + return (ScrArrState)scr_arr_dense_state(a, i); } if (scr_arr_ext_of(a)) { while (w->sparse < scr_arr_ext_of(a)->sparse_len && scr_arr_ext_of(a)->sparse[w->sparse].index < i) w->sparse++; diff --git a/packages/runtime/src/scr_itable.test.c b/packages/runtime/src/scr_itable.test.c index f6d98654a..c85370add 100644 --- a/packages/runtime/src/scr_itable.test.c +++ b/packages/runtime/src/scr_itable.test.c @@ -55,6 +55,7 @@ static ScrArr *make_f64s(const double *values, size_t n) { a->filled = (uint32_t)n; a->elem = SCR_ELEM_F64; /* Values, then one state byte per slot (scr_arr_states_end). */ + a->spare = SCR_ARR_STATES; a->data = calloc(n ? n : 1, sizeof(uint64_t) + 1); for (size_t i = 0; i < n; i++) memcpy(&a->data[i], &values[i], sizeof(double)); memset(scr_arr_state_run(a, 0, n), SCR_ARR_VALUE, n); @@ -170,6 +171,7 @@ int main(void) { assert(scr_itable_size(lists) == 4); ScrArr *big = make_f64s(NULL, 0); big->len = big->cap = big->filled = 100; + big->spare = 0; /* values without states */ free(big->data); big->data = calloc(100, sizeof(uint64_t)); for (unsigned i = 0; i < 100; i++) memcpy(&big->data[i], &(double){i}, sizeof(double)); diff --git a/packages/runtime/src/scr_lib.c b/packages/runtime/src/scr_lib.c index 4be596ab5..3f7dcac45 100644 --- a/packages/runtime/src/scr_lib.c +++ b/packages/runtime/src/scr_lib.c @@ -12961,50 +12961,50 @@ static bool scr_set_to_arr_sparse(const ScrMap *s, ScrArr *out) { ScrArr *scr_set_to_arr_f64(const ScrMap *s) { ScrArr *out = scr_arr_new(SCR_ELEM_F64, s->nlive); if (scr_set_to_arr_sparse(s, out)) return out; - uint8_t *states = scr_arr_states_end(out); + /* A fresh array without states: the values in order. */ for (size_t i = 0; i < s->nentries; i++) { if (!scr_map_entry_live(s, i)) continue; - out->data[out->len] = scr_map_entry_at(s, i)->key; - *scr_arr_state_in(states, out->len++) = SCR_ARR_VALUE; + out->data[out->len++] = scr_map_entry_at(s, i)->key; } + scr_arr_refill(out); return out; } ScrArr *scr_set_to_arr_bool(const ScrMap *s) { ScrArr *out = scr_arr_new(SCR_ELEM_BOOL, s->nlive); if (scr_set_to_arr_sparse(s, out)) return out; - uint8_t *states = scr_arr_states_end(out); + /* A fresh array without states: the values in order. */ for (size_t i = 0; i < s->nentries; i++) { if (!scr_map_entry_live(s, i)) continue; - scr_arr_dense_put(out, out->len, scr_map_entry_at(s, i)->key != 0); - *scr_arr_state_in(states, out->len++) = SCR_ARR_VALUE; + scr_arr_dense_put(out, out->len++, scr_map_entry_at(s, i)->key != 0); } + scr_arr_refill(out); return out; } ScrArr *scr_set_to_arr_str(const ScrMap *s) { ScrArr *out = scr_arr_new(SCR_ELEM_STR, s->nlive); if (scr_set_to_arr_sparse(s, out)) return out; - uint8_t *states = scr_arr_states_end(out); + /* A fresh array without states: the values in order. */ for (size_t i = 0; i < s->nentries; i++) { if (!scr_map_entry_live(s, i)) continue; scr_str_retain((ScrStr *)(uintptr_t)scr_map_entry_at(s, i)->key); - out->data[out->len] = scr_map_entry_at(s, i)->key; - *scr_arr_state_in(states, out->len++) = SCR_ARR_VALUE; + out->data[out->len++] = scr_map_entry_at(s, i)->key; } + scr_arr_refill(out); return out; } ScrArr *scr_set_to_arr_ref(const ScrMap *s, const ScrArrOps *ops) { ScrArr *out = scr_arr_new_ops(ops, s->nlive); if (scr_set_to_arr_sparse(s, out)) return out; - uint8_t *states = scr_arr_states_end(out); + /* A fresh array without states: the values in order. */ for (size_t i = 0; i < s->nentries; i++) { if (!scr_map_entry_live(s, i)) continue; ops->retain((void *)(uintptr_t)scr_map_entry_at(s, i)->key); - out->data[out->len] = scr_map_entry_at(s, i)->key; - *scr_arr_state_in(states, out->len++) = SCR_ARR_VALUE; + out->data[out->len++] = scr_map_entry_at(s, i)->key; } + scr_arr_refill(out); return out; } diff --git a/packages/runtime/src/scr_map.c b/packages/runtime/src/scr_map.c index 99caa31a8..1d0cf64b6 100644 --- a/packages/runtime/src/scr_map.c +++ b/packages/runtime/src/scr_map.c @@ -1600,7 +1600,7 @@ void scr_set_add_all(ScrMap *set, ScrArr *values) { /* Packed values stay owned by the seed array throughout construction. * Avoid taking a temporary owner only to release it after insertion. * Sparse/missing positions retain the existing accessor semantics. */ - if (i < values->cap && *scr_arr_state_ptr(values, i) == SCR_ARR_VALUE) { + if (i < values->cap && scr_arr_dense_state(values, i) == SCR_ARR_VALUE) { uint64_t key = scr_arr_dense_get(values, i); if (set->key_kind == SCR_MAP_KEY_F64) key = scr_map_f64_bits(scr_map_slot_to_f64(key)); scr_map_set(set, scr_map_hash_key(set, key), key, 0); diff --git a/packages/runtime/src/scr_publish.c b/packages/runtime/src/scr_publish.c index 675a13ca9..3cb6c5b55 100644 --- a/packages/runtime/src/scr_publish.c +++ b/packages/runtime/src/scr_publish.c @@ -209,7 +209,7 @@ void scr_pub_arr(void *v, ScrPub *ctx, ScrPubFn elem) { ScrPubFn fn = a->elem == SCR_ELEM_STR ? scr_pub_str_fn : a->elem == SCR_ELEM_REF ? elem : NULL; if (!fn) return; size_t dense = a->len < a->cap ? a->len : a->cap; - const uint8_t *states = scr_arr_states_end(a); + const uint8_t *states = scr_arr_scan_states(a); for (size_t i = 0; i < dense; i++) if (!states || *scr_arr_state_in(states, i)) scr_pub_slot(ctx, a->data[i], fn); if (scr_arr_ext_of(a)) diff --git a/packages/runtime/src/scr_regex.c b/packages/runtime/src/scr_regex.c index b62cba896..158697a14 100644 --- a/packages/runtime/src/scr_regex.c +++ b/packages/runtime/src/scr_regex.c @@ -692,7 +692,7 @@ static ScrDyn *scr_regex_groups_of(const ScrArr *a, uint8_t *bc) { for (int k = 1; k < capture_count; k++) { size_t length = strlen(name); bool matched = (size_t)k < a->len && (size_t)k < a->cap && - *scr_arr_state_ptr(a, (size_t)k) == SCR_ARR_VALUE; + scr_arr_dense_state(a, (size_t)k) == SCR_ARR_VALUE; /* ES2025 duplicates sit in distinct alternatives: at most one * participates, and it wins over an earlier undefined. */ if (length && (matched || !scr_dyn_obj_get(groups, name, length))) @@ -845,7 +845,7 @@ static ScrArr *scr_regex_exec_core(ScrStr *s, ScrRegex *re, uint64_t mask, bool if (rc == 1) { int count = lre_get_capture_count(bc); bool spans = lazy != 0 && !input.wide && (count >= 64 || ((mask | lazy) >> count) == 0); - out = scr_arr_new_inline(SCR_ELEM_STR, (size_t)count + spans); + out = scr_arr_new_inline(SCR_ELEM_STR, (size_t)count + spans, true); /* A fresh dense array: every slot is written directly, then the * hole-free prefix is the run of materialized captures from 0. Its * storage pointers are fixed (the inline block never moves here), so @@ -905,14 +905,15 @@ static bool scr_regex_group_bytes(const ScrArr *m, double k, const char **p, siz /* Array lengths stay below 2^53, so signed conversions are exact. */ if (!(k >= 0 && k < (double)(int64_t)m->len)) return false; size_t i = (size_t)(int64_t)k; - if (*scr_arr_state_ptr(m, i) == SCR_ARR_VALUE) { + uint8_t state = scr_arr_dense_state(m, i); + if (state == SCR_ARR_VALUE) { const ScrStr *str = (const ScrStr *)(uintptr_t)m->data[i]; *p = str->data; *n = str->len; return true; } uint64_t span = m->data[i]; - if (*scr_arr_state_ptr(m, i) != SCR_ARR_UNDEFINED || span == SCR_RE_NO_SPAN) return false; + if (state != SCR_ARR_UNDEFINED || span == SCR_RE_NO_SPAN) return false; const ScrStr *subject = (const ScrStr *)(uintptr_t)m->data[m->len - 1]; *p = subject->data + (span >> 32); *n = (size_t)((span & UINT32_MAX) - (span >> 32)); @@ -1043,10 +1044,13 @@ static ScrArr *scr_regex_match_all_core(ScrStr *s, ScrRegex *re, ScrArr *indices int start = scr_capture_index(&input, capture[0]); int end = scr_capture_index(&input, capture[1]); /* One allocation per row, like an exec result: every slot is written - * directly, then the hole-free prefix is the run of captures from 0. */ - ScrArr *row = scr_arr_new_inline(SCR_ELEM_STR, (size_t)capture_count); + * directly, then the hole-free prefix is the run of captures from 0. + * A row needs states only for a capture that did not participate. */ + bool all = true; + for (int k = 0; k < capture_count; k++) all &= capture[2 * k] != NULL && capture[2 * k + 1] != NULL; + ScrArr *row = scr_arr_new_inline(SCR_ELEM_STR, (size_t)capture_count, !all); uint64_t *data = row->data; - uint8_t *states = scr_arr_states_end_w(row, 8); + uint8_t *states = all ? NULL : scr_arr_states_end_w(row, 8); for (int k = 0; k < capture_count; k++) { const uint8_t *cs = capture[2 * k], *ce = capture[2 * k + 1]; if (cs == NULL || ce == NULL) { @@ -1056,11 +1060,11 @@ static ScrArr *scr_regex_match_all_core(ScrStr *s, ScrRegex *re, ScrArr *indices ScrStr *text = scr_regex_span(&input, scr_capture_index(&input, cs), scr_capture_index(&input, ce)); data[k] = (uint64_t)(uintptr_t)text; - *scr_arr_state_in(states, (size_t)k) = SCR_ARR_VALUE; + if (states) *scr_arr_state_in(states, (size_t)k) = SCR_ARR_VALUE; } } size_t filled = 0; - while (filled < (size_t)capture_count && *scr_arr_state_in(states, filled) == SCR_ARR_VALUE) + while (filled < (size_t)capture_count && (!states || *scr_arr_state_in(states, filled) == SCR_ARR_VALUE)) filled++; row->filled = (uint32_t)filled; row->len = (size_t)capture_count; @@ -2795,7 +2799,7 @@ static ScrDyn *scr_native_regex_invoke(void *ptr, ScrDyn *self, const char *meth if (!row) return scr_exc_pending() ? NULL : scr_dyn_new_null(); ScrDyn *match = scr_dyn_new_arr(); for (size_t k = 0; k < row->len; k++) - scr_dyn_arr_push(match, *scr_arr_state_ptr(row, k) == SCR_ARR_VALUE + scr_dyn_arr_push(match, scr_arr_dense_state(row, k) == SCR_ARR_VALUE ? scr_dyn_new_str((ScrStr *)(uintptr_t)row->data[k]) : scr_dyn_retain(scr_dyn_undefined())); scr_arr_copy_metadata(row, match); diff --git a/packages/runtime/src/scr_region.c b/packages/runtime/src/scr_region.c index dbd74f3ac..25bbcf7ae 100644 --- a/packages/runtime/src/scr_region.c +++ b/packages/runtime/src/scr_region.c @@ -805,7 +805,7 @@ static void scr_rgn_walk_arr(ScrRgnWalk *w, ScrArr *a) { ScrRgnSlotKind k = a->elem == SCR_ELEM_ARR ? (ScrRgnSlotKind){SCR_RGN_W_ARR, NULL} : scr_rgn_slot_kind(scr_arr_ops(a) ? scr_arr_ops(a)->release : NULL); size_t dense = a->len < a->cap ? a->len : a->cap; - const uint8_t *states = scr_arr_states_end(a); + const uint8_t *states = scr_arr_scan_states(a); for (size_t i = 0; i < dense; i++) if (!states || *scr_arr_state_in(states, i) == SCR_ARR_VALUE) scr_rgn_slot_as(w, scr_rgn_slot_ptr(a->data[i]), k); if (!scr_arr_ext_of(a)) return; diff --git a/packages/runtime/src/scr_string.c b/packages/runtime/src/scr_string.c index 58272cf67..f94554642 100644 --- a/packages/runtime/src/scr_string.c +++ b/packages/runtime/src/scr_string.c @@ -2571,15 +2571,11 @@ ScrArr *scr_str_split_limit(ScrStr *s, ScrStr *sep, double limit_num) { while (count < 64 && (bytes = scr_str_split_span(s, sep, &cursor, false, &len))) slots[count++] = (uint64_t)(uintptr_t)(s->ascii ? scr_str_from_span_ascii(bytes, len) : scr_str_from_span(bytes, len)); - ScrArr *out = scr_arr_new_inline(SCR_ELEM_STR, count); + ScrArr *out = scr_arr_new_inline(SCR_ELEM_STR, count, false); if (count) { - /* A fresh inline array of exactly `count` slots: store the pieces and - * their states directly. */ - uint8_t *states = scr_arr_states_end(out); - for (size_t k = 0; k < count; k++) { - out->data[k] = slots[k]; - *scr_arr_state_in(states, k) = SCR_ARR_VALUE; - } + /* A fresh inline array of exactly `count` slots: store the pieces + * directly. */ + for (size_t k = 0; k < count; k++) out->data[k] = slots[k]; out->len = (uint32_t)count; out->filled = (uint32_t)count; } diff --git a/packages/runtime/test/test_array.c b/packages/runtime/test/test_array.c index d943acfb2..6e4ead89a 100644 --- a/packages/runtime/test/test_array.c +++ b/packages/runtime/test/test_array.c @@ -1365,11 +1365,13 @@ static void test_exact_construction(void) { } /* The hole-free prefix bound never covers a non-value state, and the - * operations that keep an array hole-free keep the bound at its length. */ + * operations that keep an array hole-free keep the bound at its length. An + * array without states keeps its elements in dense storage. */ static bool filled_ok(const ScrArr *a) { if (a->filled > a->len || a->filled > a->cap) return false; + if (!scr_arr_has_states(a) && (a->len > a->cap || a->filled != a->len)) return false; for (size_t i = 0; i < a->filled; i++) - if (*scr_arr_state_ptr(a, i) != SCR_ARR_VALUE) return false; + if (scr_arr_dense_state(a, i) != SCR_ARR_VALUE) return false; return true; } @@ -1444,7 +1446,7 @@ static void test_filled_prefix(void) { scr_arr_release(scr_arr_sort_primitive(b, false)); check(filled_ok(b), "sorting moves holes past the bound"); for (size_t i = 0; i < b->len; i++) { - if (i < b->cap && *scr_arr_state_ptr(b, i) != SCR_ARR_VALUE) break; + if (i < b->cap && scr_arr_dense_state(b, i) != SCR_ARR_VALUE) break; check(i < b->filled, "sorted values form the bound"); } while (scr_arr_len(b) > 0) { @@ -1464,11 +1466,13 @@ static void test_filled_prefix(void) { } /* Boolean arrays store one byte per dense element, followed by the states in - * the same allocation. Growth moves the states to the end of the larger area, - * densifying absorbs sparse entries into bytes, and every moving and copying - * operation keeps values, holes and present undefined in place. */ + * the same allocation once they have states. Growth moves the states to the + * end of the larger area, densifying absorbs sparse entries into bytes, and + * every moving and copying operation keeps values, holes and present + * undefined in place. */ static bool bool_layout_ok(const ScrArr *a) { - return a->cap == 0 || scr_arr_states_end(a) == (const uint8_t *)a->data + 2 * (size_t)a->cap; + return a->cap == 0 || !scr_arr_has_states(a) || + scr_arr_states_end(a) == (const uint8_t *)a->data + 2 * (size_t)a->cap; } static bool bool_states(const ScrArr *a, const char *want) { @@ -1584,7 +1588,7 @@ static void test_inline_slots(void) { * block; growth past them moves them out like any inline array. */ ScrArr *known = scr_arr_new(SCR_ELEM_F64, 4); check(inline_slots(known) && known->cap >= 4 && known->len == 0 && !scr_arr_has(known, 0) && - scr_arr_states_end(known) == (uint8_t *)(known->data + known->cap) + known->cap, + !scr_arr_has_states(known), "a known-count construction places its slots in the array block"); size_t known_cap = known->cap; for (size_t i = 0; i <= known_cap; i++) scr_arr_push_f64(known, (double)i); @@ -1605,26 +1609,25 @@ static void test_inline_slots(void) { "known-count boolean constructions are inline too"); scr_arr_release(known_flags); ScrArr *a = scr_arr_new_small(SCR_ELEM_F64); - check(inline_slots(a) && a->cap >= 4 && a->len == 0 && - scr_arr_states_end(a) == (uint8_t *)(a->data + a->cap) + a->cap, - "small construction places slots and states in the array block"); + check(inline_slots(a) && a->cap >= 4 && a->len == 0 && !scr_arr_has_states(a), + "small construction places its slots in the array block, without states"); size_t inline_cap = a->cap; for (size_t i = 0; i < inline_cap; i++) scr_arr_push_f64(a, (double)i); check(inline_slots(a) && a->filled == inline_cap, "appends within the inline capacity stay inline"); scr_arr_push_f64(a, 100); check(!inline_slots(a) && a->cap > inline_cap && scr_arr_get_f64(a, 0) == 0 && scr_arr_get_f64(a, (double)inline_cap - 1) == (double)inline_cap - 1 && - scr_arr_get_f64(a, (double)inline_cap) == 100 && - scr_arr_states_end(a) == (uint8_t *)(a->data + a->cap) + a->cap, + scr_arr_get_f64(a, (double)inline_cap) == 100 && !scr_arr_has_states(a), "growth moves inline slots to their own allocation"); scr_arr_release(a); ScrArr *flags = scr_arr_new_small(SCR_ELEM_BOOL); - check(inline_slots(flags) && scr_arr_states_end(flags) == (uint8_t *)flags->data + 2 * (size_t)flags->cap, + check(inline_slots(flags) && flags->cap >= 4 && !scr_arr_has_states(flags), "boolean inline slots are one byte each"); scr_arr_push_bool(flags, true); scr_arr_set_bool(flags, 3, true); - check(scr_arr_get_bool(flags, 0) && !scr_arr_has(flags, 1) && scr_arr_get_bool(flags, 3), + check(scr_arr_get_bool(flags, 0) && !scr_arr_has(flags, 1) && scr_arr_get_bool(flags, 3) && + scr_arr_has_states(flags) && bool_layout_ok(flags), "boolean inline slots keep holes"); scr_arr_release(flags); @@ -2107,8 +2110,8 @@ static void test_head_room(void) { check_f64(scr_arr_shift_f64(q), 0, "shift answers the first element"); check(q->data == block + 1 && q->cap == cap - 1 && q->len == 63, "shift advances the storage instead of moving the tail"); - check(scr_arr_states_end(q) == (uint8_t *)(block + cap) + cap - 1 && head_room(q) && q->data[-1] == 1, - "the remaining states stay at the end of the allocation"); + check(!scr_arr_has_states(q) && head_room(q) && q->data[-1] == 1, + "shift records the vacated slot in front of the data"); check(q->filled == 63, "shift keeps the hole-free prefix"); for (int i = 1; i < 40; i++) scr_arr_shift_f64(q); check(q->data == block + 40 && q->len == 24 && scr_arr_get_f64(q, 0) == 40 && @@ -2223,6 +2226,159 @@ static void test_head_room(void) { #endif } +/* SCR_ARR_STATES (scr_array.h): an array starts without state bytes and + * keeps none while every index below its length is a present value. A + * hole, a present undefined or a length past the elements gives it states, + * with the elements in place. */ +static ScrStr *str_of(const char *text) { return scr_str_new(text, strlen(text)); } + +static bool strs_are(ScrArr *a, const char *want) { + /* want: one character per element, '_' a hole, 'u' undefined. */ + if (scr_arr_len(a) != (double)strlen(want)) return false; + for (size_t i = 0; want[i]; i++) { + double state = scr_arr_state(a, (double)i); + if (want[i] == '_') { + if (state != SCR_ARR_HOLE) return false; + } else if (want[i] == 'u') { + if (state != SCR_ARR_UNDEFINED) return false; + } else { + if (state != SCR_ARR_VALUE) return false; + ScrStr *got = scr_arr_get_ref(a, (double)i); + bool same = got->len == 1 && got->data[0] == want[i]; + scr_str_release(got); + if (!same) return false; + } + } + return filled_ok(a); +} + +static ScrArr *strs_of(const char *text, bool small) { + ScrArr *a = small ? scr_arr_new_small(SCR_ELEM_STR) : scr_arr_new(SCR_ELEM_STR, 0); + for (size_t i = 0; text[i]; i++) scr_arr_push_ref(a, scr_str_new(text + i, 1)); + return a; +} + +static void test_states_on_demand(void) { +#ifdef SCR_RC_AUDIT + long strings0 = scr_str_live_count(); + long arrays0 = scr_arr_live_count(); +#endif + ScrArr *a = strs_of("abcdef", false); + scr_arr_set_ref(a, 1, str_of("B")); + scr_arr_set_ref(a, 6, str_of("g")); + scr_str_release(scr_arr_pop_ref(a)); + scr_arr_set_len(a, 5); + check(!scr_arr_has_states(a) && a->filled == 5 && strs_are(a, "aBcde"), + "pushes, writes below the length, appends, pops and truncation keep no states"); + scr_arr_set_undefined(a, 2); + check(scr_arr_has_states(a) && strs_are(a, "aBude"), "a present undefined gives the array states"); + scr_arr_release(a); + + ScrArr *gap = strs_of("ab", true); + scr_arr_set_ref(gap, 3, str_of("d")); + check(scr_arr_has_states(gap) && strs_are(gap, "ab_d"), "a write past the length leaves holes"); + scr_arr_release(gap); + + ScrArr *longer = strs_of("abc", true); + scr_arr_set_len(longer, 5); + check(scr_arr_has_states(longer) && strs_are(longer, "abc__"), "a longer length adds holes"); + scr_arr_release(longer); + + /* `new Array(n)` and a length set on an empty small array: the inline + * slots give way to one allocation holding the slots and their states. */ + ScrArr *sized = scr_arr_new_small(SCR_ELEM_STR); + scr_arr_set_len(sized, 6); + scr_arr_set_ref(sized, 4, str_of("e")); + check(scr_arr_has_states(sized) && !(sized->spare & SCR_ARR_INLINE_SLOTS) && sized->cap == 6 && + strs_are(sized, "____e_") && filled_ok(sized), + "a length set on an empty inline array reserves that many slots with states"); + scr_arr_release(sized); + + ScrArr *deleted = strs_of("abcd", false); + scr_arr_delete(deleted, 1); + check(scr_arr_has_states(deleted) && strs_are(deleted, "a_cd"), "delete leaves a hole"); + scr_arr_release(deleted); + + /* Conversion keeps out-of-line storage and head room's elements. */ + ScrArr *props = strs_of("abc", true); + scr_arr_set_ref(props, -1, str_of("p")); + scr_arr_set_undefined(props, 3); + ScrStr *prop = scr_arr_get_ref(props, -1); + check(scr_arr_has_states(props) && strs_are(props, "abcu") && prop->data[0] == 'p', + "states keep a noncanonical property"); + scr_str_release(prop); + scr_arr_release(props); + ScrArr *q = strs_of("abcdefghijklmnopqrst", false); + for (int i = 0; i < 3; i++) scr_str_release(scr_arr_shift_ref(q)); + check(!scr_arr_has_states(q) && head_room(q), "shift keeps no states"); + scr_arr_set_undefined(q, 1); + check(scr_arr_has_states(q) && !head_room(q) && strs_are(q, "dufghijklmnopqrst"), + "states move the elements of head room to the front"); + scr_arr_release(q); + + /* Copies keep no states for values, and states for holes. */ + ScrArr *src = strs_of("abcde", false); + ScrArr *part = scr_arr_slice(src, 1, 4); + ScrArr *joined = scr_arr_slice(src, 0, INFINITY); + scr_arr_concat_copy(joined, src); + ScrArr *spliced = scr_arr_splice(joined, 2, 3); + ScrArr *reversed = scr_arr_to_reversed(src); + scr_arr_release(scr_arr_reverse(src)); + scr_arr_release(scr_arr_sort_primitive(src, false)); + check(!scr_arr_has_states(part) && strs_are(part, "bcd"), "a slice of values keeps no states"); + check(!scr_arr_has_states(joined) && strs_are(joined, "ababcde"), + "concat and splice of values keep no states"); + check(!scr_arr_has_states(spliced) && strs_are(spliced, "cde"), "removed values keep no states"); + check(!scr_arr_has_states(reversed) && strs_are(reversed, "edcba"), "toReversed of values keeps no states"); + check(!scr_arr_has_states(src) && strs_are(src, "abcde"), "reverse and sort of values keep no states"); + scr_arr_release(src); + scr_arr_release(part); + scr_arr_release(joined); + scr_arr_release(spliced); + scr_arr_release(reversed); + ScrArr *holey = strs_of("ab", false); + scr_arr_set_ref(holey, 3, str_of("d")); + ScrArr *spread = strs_of("x", true); + scr_arr_push_spread(spread, holey); + ScrArr *concat = strs_of("x", true); + scr_arr_concat_copy(concat, holey); + ScrArr *values = scr_arr_slice(holey, 0, 2); + ScrArr *front = strs_of("yz", false); + scr_arr_unshift_spread(front, holey); + check(scr_arr_has_states(spread) && strs_are(spread, "xabud") && scr_arr_has_states(concat) && + strs_are(concat, "xab_d") && !scr_arr_has_states(values) && strs_are(values, "ab") && + scr_arr_has_states(front) && strs_are(front, "abudyz"), + "copies of holes keep holes or present undefined"); + scr_arr_release(holey); + scr_arr_release(spread); + scr_arr_release(concat); + scr_arr_release(values); + scr_arr_release(front); + + /* Numbers and booleans take the same paths without references. */ + ScrArr *nums = scr_arr_new(SCR_ELEM_F64, 3); + for (int i = 0; i < 40; i++) scr_arr_push_f64(nums, i); + scr_arr_release(scr_arr_fill_f64(nums, 7, 30, INFINITY)); + check(!scr_arr_has_states(nums) && scr_arr_get_number(nums, 39) == 7 && scr_arr_get_number(nums, 40) != 0, + "number fills keep no states"); + scr_arr_release(scr_arr_fill_undefined(nums, 0, 1)); + check(scr_arr_has_states(nums) && scr_arr_state(nums, 0) == SCR_ARR_UNDEFINED && + scr_arr_get_number(nums, 1) == 1 && scr_arr_get_number(nums, 39) == 7 && filled_ok(nums), + "an undefined fill gives a number array states"); + scr_arr_release(nums); + ScrArr *bits = scr_arr_new_small(SCR_ELEM_BOOL); + for (int i = 0; i < 6; i++) scr_arr_push_bool(bits, i % 2 == 0); + scr_arr_set_undefined(bits, 6); + check(scr_arr_has_states(bits) && bool_layout_ok(bits) && scr_arr_get_bool(bits, 4) && + !scr_arr_get_bool(bits, 5) && scr_arr_state(bits, 6) == SCR_ARR_UNDEFINED, + "boolean arrays get one state byte per element"); + scr_arr_release(bits); +#ifdef SCR_RC_AUDIT + check(scr_str_live_count() == strings0, "states on demand: no strings leaked"); + check(scr_arr_live_count() == arrays0, "states on demand: no arrays leaked"); +#endif +} + int main(int argc, char **argv) { if (argc > 1) { ScrArr *a = scr_arr_new(SCR_ELEM_F64, 0); @@ -2279,6 +2435,7 @@ int main(int argc, char **argv) { test_bool_bytes(); test_inline_slots(); test_head_room(); + test_states_on_demand(); fprintf(stderr, "%ld/%ld cases passed\n", total - failed, total); return failed == 0 ? 0 : 1; From f9bed65a65f732c539f50e32e2ac7db10828e71c Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 08:45:10 -0500 Subject: [PATCH 031/197] Name the union destructuring fence without printing the union The fence for a field the arms of a union type differently printed the whole union, whose arm order follows the order record shapes are interned in and changes with unrelated compiler changes. It now names the union without its arms, as the other union fences do. --- packages/compiler/src/frontend/lowering/lower-stmts.ts | 8 ++------ tests/harness/__snapshots__/destructuring-fences.ts.txt | 2 +- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/packages/compiler/src/frontend/lowering/lower-stmts.ts b/packages/compiler/src/frontend/lowering/lower-stmts.ts index 831fec83d..67706a285 100644 --- a/packages/compiler/src/frontend/lowering/lower-stmts.ts +++ b/packages/compiler/src/frontend/lowering/lower-stmts.ts @@ -3653,11 +3653,7 @@ function lowerUnionObjectPattern( for (const el of pattern.elements.nodes) { if (el.name === undefined) continue; if (el.dotDotDotToken) { - lowerer.unsupported( - "SC1031", - el, - `rest bindings over '${lowerer.fmt(srcType)}' sources (${NARROW_FIRST})`, - ); + lowerer.unsupported("SC1031", el, `rest bindings over union sources (${NARROW_FIRST})`); } const prop = el.propertyName ?? el.name; const propName = @@ -3692,7 +3688,7 @@ function lowerUnionObjectPattern( lowerer.unsupported( "SC1031", el, - `destructuring '${propName}' from a '${lowerer.fmt(srcType)}' source (every arm must be a record or class with a same-typed field '${propName}' — ${NARROW_FIRST})`, + `destructuring '${propName}' from a union whose arms type it differently (every arm must be a record or class with a same-typed field '${propName}' — ${NARROW_FIRST})`, ); } let value: IrExpr = { diff --git a/tests/harness/__snapshots__/destructuring-fences.ts.txt b/tests/harness/__snapshots__/destructuring-fences.ts.txt index 2f6afe734..41026a7c8 100644 --- a/tests/harness/__snapshots__/destructuring-fences.ts.txt +++ b/tests/harness/__snapshots__/destructuring-fences.ts.txt @@ -19,7 +19,7 @@ destructuring-fences.ts:31:9 - error SC1090: reading a property that has only a | ^~~~~~ 32 | console.log(onlyIn); -destructuring-fences.ts:45:11 - error SC1031: destructuring 'b' from a '{ b: string; count: number; kind: string } | { b: number; kind: string; label: string }' source (every arm must be a record or class with a same-typed field 'b' — narrow first: check a discriminant field, or compare with '!== undefined'/'!== null' for unit arms) is not supported yet +destructuring-fences.ts:45:11 - error SC1031: destructuring 'b' from a union whose arms type it differently (every arm must be a record or class with a same-typed field 'b' — narrow first: check a discriminant field, or compare with '!== undefined'/'!== null' for unit arms) is not supported yet 44 | function fromUnion(u: E1 | E2): string { 45 | const { b } = u; // union source whose arms type the field differently: narrow first From 17b70e6d6a02ca09bdb6f9214e73305731a03410 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 08:57:59 -0500 Subject: [PATCH 032/197] Find Buffer search candidates with memchr --- .../7249-buffer-index-of-candidates.ts.json | 6 +++++ packages/runtime/src/scr_bytes.c | 14 +++++++--- .../corpus/7249-buffer-index-of-candidates.ts | 27 +++++++++++++++++++ .../7249-buffer-index-of-candidates.ts.json | 1 + 4 files changed, 45 insertions(+), 3 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7249-buffer-index-of-candidates.ts.json create mode 100644 tests/corpus/7249-buffer-index-of-candidates.ts create mode 100644 tests/harness/corpus-costs/7249-buffer-index-of-candidates.ts.json diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7249-buffer-index-of-candidates.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7249-buffer-index-of-candidates.ts.json new file mode 100644 index 000000000..f5299cb9a --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7249-buffer-index-of-candidates.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7249-buffer-index-of-candidates.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_bytes.c b/packages/runtime/src/scr_bytes.c index 7a6a76cf9..183174555 100644 --- a/packages/runtime/src/scr_bytes.c +++ b/packages/runtime/src/scr_bytes.c @@ -3271,16 +3271,24 @@ double scr_bytes_index_of(const ScrBytes *b, const ScrBytes *needle, double off, } if (nlen == 0) return o > (double)len ? (double)len : o; /* '' matches at the clamp */ if (nlen > len) return -1; + /* Candidates are the positions holding the needle's first byte; only + * those compare the rest. */ + const uint8_t first = needle->data[0]; if (fwd) { size_t start = o > (double)len ? len : (size_t)o; - for (size_t i = start; i + nlen <= len; i++) { - if (memcmp(b->data + i, needle->data, nlen) == 0) return (double)i; + const uint8_t *at = b->data + start, *end = b->data + (len - nlen) + 1; + while (at < end) { + at = memchr(at, first, (size_t)(end - at)); + if (!at) return -1; + if (nlen == 1 || memcmp(at + 1, needle->data + 1, nlen - 1) == 0) return (double)(at - b->data); + at++; } return -1; } size_t start = o > (double)(len - nlen) ? len - nlen : (size_t)o; for (size_t i = start;; i--) { - if (memcmp(b->data + i, needle->data, nlen) == 0) return (double)i; + if (b->data[i] == first && (nlen == 1 || memcmp(b->data + i + 1, needle->data + 1, nlen - 1) == 0)) + return (double)i; if (i == 0) break; } return -1; diff --git a/tests/corpus/7249-buffer-index-of-candidates.ts b/tests/corpus/7249-buffer-index-of-candidates.ts new file mode 100644 index 000000000..ae6c2ae23 --- /dev/null +++ b/tests/corpus/7249-buffer-index-of-candidates.ts @@ -0,0 +1,27 @@ +// Buffer indexOf, lastIndexOf and includes with needles whose first byte +// recurs: a candidate whose later bytes differ is skipped, matches at the +// first and last positions are found, byteOffset clamps and counts from the +// end, and the search reads only the buffer's own window of its memory. +const text = Buffer.from("aab aaab aaaab ab\nab"); +console.log(text.indexOf("ab"), text.indexOf("aab"), text.indexOf("aaab"), text.indexOf("aaaab"), text.indexOf("aaaaab")); +console.log(text.lastIndexOf("ab"), text.lastIndexOf("aab"), text.lastIndexOf("aaab"), text.lastIndexOf("b\na")); +console.log(text.indexOf("ab", 2), text.indexOf("ab", -3), text.indexOf("ab", 100), text.indexOf("ab", -100)); +console.log(text.lastIndexOf("ab", 5), text.lastIndexOf("ab", -4), text.lastIndexOf("ab", 0), text.lastIndexOf("aa", 1)); +console.log(text.indexOf(10), text.indexOf(97, 3), text.lastIndexOf(98), text.lastIndexOf(97, -2), text.includes("b a")); +console.log(text.indexOf(""), text.indexOf("", 4), text.indexOf("", 100), text.lastIndexOf(""), text.indexOf("x")); +console.log(text.indexOf(Buffer.from("a\nab")), text.indexOf("ab\nab"), text.indexOf("ab\nabc"), text.lastIndexOf("aab", 4)); + +// A window into a larger buffer never sees the bytes outside it. +const whole = Buffer.from("xxneedlexx needle needlexx"); +const window = whole.subarray(3, 22); +console.log(window.toString(), window.indexOf("needle"), window.lastIndexOf("needle"), window.indexOf("xx"), window.includes("xne")); +console.log(window.indexOf("eedle", 1), window.lastIndexOf("eed"), window.indexOf("needlex"), window.lastIndexOf("n", 0)); + +// A long search with many first-byte candidates. +const long = Buffer.alloc(20000, "ab"); +long.write("abc", 19990); +console.log(long.indexOf("abc"), long.lastIndexOf("aba"), long.indexOf("abc", 19991), long.lastIndexOf("abc"), long.indexOf("bb")); +const lines: number[] = []; +const fasta = Buffer.from(">ONE\nACGT\nTTGA\n>TWO\nGGCC\n"); +for (let at = fasta.indexOf(10); at !== -1; at = fasta.indexOf(10, at + 1)) lines.push(at); +console.log(lines.join(","), fasta.indexOf(62, 1), fasta.lastIndexOf(62)); diff --git a/tests/harness/corpus-costs/7249-buffer-index-of-candidates.ts.json b/tests/harness/corpus-costs/7249-buffer-index-of-candidates.ts.json new file mode 100644 index 000000000..5c265ea01 --- /dev/null +++ b/tests/harness/corpus-costs/7249-buffer-index-of-candidates.ts.json @@ -0,0 +1 @@ +{ "plain": 2870, "san": 3656 } From 1e0dabd74864c4c2f4e6c962cf550bad540047a6 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 09:05:46 -0500 Subject: [PATCH 033/197] Declare every http2 listener event for once and addListener The lowering accepts on, once and addListener alike for each server, session and stream kind, but the fallback Node types (used when a project has no @types/node) declared fewer events for once than for on on the http2 servers, sessions and streams, and no addListener on any of them. `http2Server.once("connection", ...)` and `server.addListener("stream", ...)` failed to type-check (SC0001) where Node runs them. once and addListener on Http2Server, Http2SecureServer, Http2Session, Http2Stream, ServerHttp2Stream and ClientHttp2Stream now declare the same overloads as on. Corpus 7281 runs an HTTP CONNECT proxy on node:http, which #1791's declaration of the http.Server 'connect' event made compile: it tunnels to a local target and pipes both ways, with a non-empty head, once and addListener registrations, and an 'upgrade' handover. Corpus 7282 registers http2 server, session and stream listeners with once and addListener. Closes #1439 --- .../ambient/scriptc-node-fallback.d.ts | 142 +++++++++++++++++- .../7281-http-server-connect-tunnel.ts.json | 6 + .../7282-http2-once-add-listener.ts.json | 6 + .../corpus/7281-http-server-connect-tunnel.ts | 81 ++++++++++ tests/corpus/7282-http2-once-add-listener.ts | 54 +++++++ .../7281-http-server-connect-tunnel.ts.json | 1 + .../7282-http2-once-add-listener.ts.json | 1 + 7 files changed, 286 insertions(+), 5 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7281-http-server-connect-tunnel.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7282-http2-once-add-listener.ts.json create mode 100644 tests/corpus/7281-http-server-connect-tunnel.ts create mode 100644 tests/corpus/7282-http2-once-add-listener.ts create mode 100644 tests/harness/corpus-costs/7281-http-server-connect-tunnel.ts.json create mode 100644 tests/harness/corpus-costs/7282-http2-once-add-listener.ts.json diff --git a/packages/compiler/ambient/scriptc-node-fallback.d.ts b/packages/compiler/ambient/scriptc-node-fallback.d.ts index 2a3f11be9..02b88a677 100644 --- a/packages/compiler/ambient/scriptc-node-fallback.d.ts +++ b/packages/compiler/ambient/scriptc-node-fallback.d.ts @@ -5929,6 +5929,7 @@ declare module "http2" { export interface Http2Stream { on(event: "error" | "close", listener: (err: Error) => void): void; once(event: "error" | "close", listener: (err: Error) => void): void; + addListener(event: "error" | "close", listener: (err: Error) => void): void; destroy(): void; } export interface Http2ServerRequest { @@ -6029,11 +6030,6 @@ declare module "http2" { listener: (stream: ServerHttp2Stream, headers: IncomingHttpHeaders, flags: number) => void, ): void; on(event: "session", listener: (session: ServerHttp2Session) => void): void; - once( - event: "stream", - listener: (stream: ServerHttp2Stream, headers: IncomingHttpHeaders, flags: number) => void, - ): void; - once(event: "session", listener: (session: ServerHttp2Session) => void): void; /* HTTP CONNECT (and, in Node, the h2 extended CONNECT — which the * allowHTTP1 lowering never sees): the second argument is the raw * socket on HTTP/1.1 connections; narrow with instanceof. */ @@ -6051,6 +6047,42 @@ declare module "http2" { ): void; once(event: "error", listener: (err: Error) => void): void; once(event: "close", listener: () => void): void; + once(event: "sessionError", listener: (err: Error, session: ServerHttp2Session) => void): void; + once( + event: "stream", + listener: (stream: ServerHttp2Stream, headers: IncomingHttpHeaders, flags: number) => void, + ): void; + once(event: "session", listener: (session: ServerHttp2Session) => void): void; + once( + event: "connect", + listener: ( + req: Http2ServerRequest | import("http").IncomingMessage, + resOrSocket: Http2ServerResponse | Socket, + ) => void, + ): void; + addListener(event: "connection", listener: (socket: Socket) => void): void; + addListener( + event: "request", + listener: (req: Http2ServerRequest, res: Http2ServerResponse) => void, + ): void; + addListener(event: "error", listener: (err: Error) => void): void; + addListener(event: "close", listener: () => void): void; + addListener( + event: "sessionError", + listener: (err: Error, session: ServerHttp2Session) => void, + ): void; + addListener( + event: "stream", + listener: (stream: ServerHttp2Stream, headers: IncomingHttpHeaders, flags: number) => void, + ): void; + addListener(event: "session", listener: (session: ServerHttp2Session) => void): void; + addListener( + event: "connect", + listener: ( + req: Http2ServerRequest | import("http").IncomingMessage, + resOrSocket: Http2ServerResponse | Socket, + ) => void, + ): void; } export interface SecureServerOptions { allowHTTP1?: boolean; @@ -6392,6 +6424,28 @@ declare module "http2" { listener: (errorCode: number, lastStreamID: number, opaqueData?: Uint8Array) => void, ): void; once(event: "localSettings" | "remoteSettings", listener: (settings: any) => void): void; + once(event: "frameError", listener: (type: number, code: number, id: number) => void): void; + once(event: "ping", listener: (payload: Uint8Array) => void): void; + addListener(event: "close" | "timeout", listener: () => void): void; + addListener(event: "error", listener: (err: Error) => void): void; + addListener( + event: "connect", + listener: (session: Http2Session, socket: import("net").Socket) => void, + ): void; + addListener( + event: "stream", + listener: (stream: ServerHttp2Stream, headers: IncomingHttpHeaders, flags: number) => void, + ): void; + addListener( + event: "goaway", + listener: (errorCode: number, lastStreamID: number, opaqueData?: Uint8Array) => void, + ): void; + addListener(event: "localSettings" | "remoteSettings", listener: (settings: any) => void): void; + addListener( + event: "frameError", + listener: (type: number, code: number, id: number) => void, + ): void; + addListener(event: "ping", listener: (payload: Uint8Array) => void): void; } export interface ClientHttp2Session extends Http2Session { @@ -6469,7 +6523,30 @@ declare module "http2" { listener: () => void, ): void; once(event: "error", listener: (err: Error) => void): void; + once(event: "frameError", listener: (type: number, code: number, id: number) => void): void; once(event: "trailers", listener: (trailers: IncomingHttpHeaders, flags: number) => void): void; + addListener(event: "data", listener: (chunk: any) => void): void; + addListener( + event: + | "end" + | "close" + | "aborted" + | "ready" + | "timeout" + | "drain" + | "finish" + | "wantTrailers", + listener: () => void, + ): void; + addListener(event: "error", listener: (err: Error) => void): void; + addListener( + event: "frameError", + listener: (type: number, code: number, id: number) => void, + ): void; + addListener( + event: "trailers", + listener: (trailers: IncomingHttpHeaders, flags: number) => void, + ): void; respond( headers?: OutgoingHttpHeaders, options?: { endStream?: boolean; waitForTrailers?: boolean }, @@ -6519,6 +6596,7 @@ declare module "http2" { on(event: "trailers", listener: (trailers: IncomingHttpHeaders, flags: number) => void): void; once(event: "response", listener: (headers: IncomingHttpHeaders, flags: number) => void): void; once(event: "push", listener: (headers: IncomingHttpHeaders, flags: number) => void): void; + once(event: "headers", listener: (headers: IncomingHttpHeaders, flags: number) => void): void; once(event: "data", listener: (chunk: any) => void): void; once( event: @@ -6533,6 +6611,42 @@ declare module "http2" { listener: () => void, ): void; once(event: "error", listener: (err: Error) => void): void; + once(event: "frameError", listener: (type: number, code: number, id: number) => void): void; + once(event: "trailers", listener: (trailers: IncomingHttpHeaders, flags: number) => void): void; + addListener( + event: "response", + listener: (headers: IncomingHttpHeaders, flags: number) => void, + ): void; + addListener( + event: "push", + listener: (headers: IncomingHttpHeaders, flags: number) => void, + ): void; + addListener( + event: "headers", + listener: (headers: IncomingHttpHeaders, flags: number) => void, + ): void; + addListener(event: "data", listener: (chunk: any) => void): void; + addListener( + event: + | "end" + | "close" + | "aborted" + | "ready" + | "timeout" + | "drain" + | "finish" + | "wantTrailers", + listener: () => void, + ): void; + addListener(event: "error", listener: (err: Error) => void): void; + addListener( + event: "frameError", + listener: (type: number, code: number, id: number) => void, + ): void; + addListener( + event: "trailers", + listener: (trailers: IncomingHttpHeaders, flags: number) => void, + ): void; } export interface Http2Server { @@ -6567,6 +6681,24 @@ declare module "http2" { ): void; once(event: "error", listener: (err: Error) => void): void; once(event: "close" | "listening" | "timeout", listener: () => void): void; + once(event: "connection", listener: (socket: import("net").Socket) => void): void; + once(event: "sessionError", listener: (err: Error, session: ServerHttp2Session) => void): void; + addListener( + event: "stream", + listener: (stream: ServerHttp2Stream, headers: IncomingHttpHeaders, flags: number) => void, + ): void; + addListener(event: "session", listener: (session: ServerHttp2Session) => void): void; + addListener( + event: "request", + listener: (req: Http2ServerRequest, res: Http2ServerResponse) => void, + ): void; + addListener(event: "error", listener: (err: Error) => void): void; + addListener(event: "close" | "listening" | "timeout", listener: () => void): void; + addListener(event: "connection", listener: (socket: import("net").Socket) => void): void; + addListener( + event: "sessionError", + listener: (err: Error, session: ServerHttp2Session) => void, + ): void; } export interface ServerOptions { diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7281-http-server-connect-tunnel.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7281-http-server-connect-tunnel.ts.json new file mode 100644 index 000000000..2328d508b --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7281-http-server-connect-tunnel.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7281-http-server-connect-tunnel.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7282-http2-once-add-listener.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7282-http2-once-add-listener.ts.json new file mode 100644 index 000000000..96c879197 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7282-http2-once-add-listener.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7282-http2-once-add-listener.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7281-http-server-connect-tunnel.ts b/tests/corpus/7281-http-server-connect-tunnel.ts new file mode 100644 index 000000000..d8b396c07 --- /dev/null +++ b/tests/corpus/7281-http-server-connect-tunnel.ts @@ -0,0 +1,81 @@ +// An HTTP CONNECT proxy on node:http: the server's 'connect' listener gets +// the request, the raw client socket and the bytes read past the request +// head, opens a TCP connection to the requested host and pipes both ways. +// 'once' and 'addListener' register the same event, and 'upgrade' hands +// over the socket for an Upgrade request. +import * as http from "node:http"; +import * as net from "node:net"; + +const target = net.createServer((socket) => { + socket.on("data", (d) => socket.write(`echo:${d.toString()}`)); +}); + +const proxy = http.createServer((_req, res) => res.end("plain")); +const seen: string[] = []; +proxy.on("connect", (req, clientSocket, head) => { + seen.push(`connect ${req.method} ${req.url} head=${head.length}`); + const [host, port] = (req.url ?? "").split(":"); + const upstream = net.connect(Number(port), host, () => { + clientSocket.write("HTTP/1.1 200 Connection Established\r\n\r\n"); + if (head.length > 0) upstream.write(head); + upstream.pipe(clientSocket); + clientSocket.pipe(upstream); + }); +}); +proxy.once("connect", (req) => seen.push(`once ${req.url}`)); +proxy.addListener("upgrade", (req, socket, head) => { + seen.push(`upgrade ${req.headers.upgrade} head=${head.length}`); + socket.end("HTTP/1.1 101 Switching Protocols\r\nUpgrade: echo\r\nConnection: Upgrade\r\n\r\n"); +}); + +function tunnel( + proxyPort: number, + targetPort: number, + payload: string, + withHead: boolean, +): Promise { + return new Promise((resolve) => { + const c = net.connect(proxyPort, "127.0.0.1", () => { + const request = `CONNECT 127.0.0.1:${targetPort} HTTP/1.1\r\nHost: 127.0.0.1:${targetPort}\r\n\r\n`; + c.write(withHead ? request + payload : request); + }); + let text = ""; + let sent = withHead; + c.on("data", (d) => { + text += d.toString(); + if (!sent && text.includes("\r\n\r\n")) { + sent = true; + c.write(payload); + } + if (sent && text.includes(`echo:${payload}`)) { + c.end(); + resolve(text); + } + }); + }); +} + +function upgrade(proxyPort: number): Promise { + return new Promise((resolve) => { + const u = net.connect(proxyPort, "127.0.0.1", () => { + u.write("GET /ws HTTP/1.1\r\nHost: x\r\nConnection: Upgrade\r\nUpgrade: echo\r\n\r\n"); + }); + let text = ""; + u.on("data", (d) => (text += d.toString())); + u.on("end", () => resolve(text.split("\r\n")[0] ?? "")); + }); +} + +target.listen(0, "127.0.0.1", () => { + const targetPort = (target.address() as net.AddressInfo).port; + proxy.listen(0, "127.0.0.1", async () => { + const proxyPort = (proxy.address() as net.AddressInfo).port; + console.log(JSON.stringify(await tunnel(proxyPort, targetPort, "first", false))); + console.log(JSON.stringify(await tunnel(proxyPort, targetPort, "second", false))); + console.log(JSON.stringify(await tunnel(proxyPort, targetPort, "with-head", true))); + console.log(JSON.stringify(await upgrade(proxyPort))); + console.log(seen.join("\n").replaceAll(String(targetPort), "PORT")); + proxy.close(); + target.close(); + }); +}); diff --git a/tests/corpus/7282-http2-once-add-listener.ts b/tests/corpus/7282-http2-once-add-listener.ts new file mode 100644 index 000000000..ab1e24a9f --- /dev/null +++ b/tests/corpus/7282-http2-once-add-listener.ts @@ -0,0 +1,54 @@ +// node:http2 listeners registered with once and addListener on the server, +// the client session and both stream kinds. A once listener runs for the +// first event only; addListener is on. +import * as http2 from "node:http2"; +import * as net from "node:net"; + +const log: string[] = []; +const accepted: string[] = []; +const server = http2.createServer(); +server.once("connection", (socket: net.Socket) => accepted.push(`connection ${typeof socket.remotePort}`)); +server.addListener("session", () => accepted.push("session")); +server.once("stream", () => log.push("server first stream")); +server.addListener("stream", (stream, headers) => { + const header = headers[":path"]; + const path = typeof header === "string" ? header : ""; + log.push(`server stream ${path}`); + let body = ""; + stream.addListener("data", (chunk) => (body += chunk.toString())); + stream.once("end", () => { + stream.respond({ ":status": 200, "x-path": path }); + stream.end(`${path}:${body}`); + }); +}); +server.once("close", () => log.push("server close")); + +function request(client: http2.ClientHttp2Session, path: string, body: string): Promise { + return new Promise((resolve) => { + const req = client.request({ ":path": path, ":method": "POST" }); + req.once("response", (headers) => { + const status = headers[":status"]; + const echoed = headers["x-path"]; + log.push(`client response ${typeof status === "number" ? status : "?"} ${typeof echoed === "string" ? echoed : "?"}`); + }); + let text = ""; + req.addListener("data", (chunk) => (text += chunk.toString())); + req.once("end", () => resolve(text)); + req.end(body); + }); +} + +server.listen(0, () => { + const port = (server.address() as net.AddressInfo).port; + const client = http2.connect(`http://localhost:${port}`); + client.once("connect", () => log.push("client connect")); + client.addListener("close", () => { + log.push("client close"); + server.close(() => console.log([accepted.sort().join(", "), ...log].join("\n"))); + }); + void (async () => { + console.log(await request(client, "/a", "one")); + console.log(await request(client, "/b", "two")); + client.close(); + })(); +}); diff --git a/tests/harness/corpus-costs/7281-http-server-connect-tunnel.ts.json b/tests/harness/corpus-costs/7281-http-server-connect-tunnel.ts.json new file mode 100644 index 000000000..c35274f8c --- /dev/null +++ b/tests/harness/corpus-costs/7281-http-server-connect-tunnel.ts.json @@ -0,0 +1 @@ +{ "plain": 3700, "san": 3100 } diff --git a/tests/harness/corpus-costs/7282-http2-once-add-listener.ts.json b/tests/harness/corpus-costs/7282-http2-once-add-listener.ts.json new file mode 100644 index 000000000..9e5407833 --- /dev/null +++ b/tests/harness/corpus-costs/7282-http2-once-add-listener.ts.json @@ -0,0 +1 @@ +{ "plain": 4000, "san": 5100 } From 20f4d1ff125804302495c2f357f1f10cfa5efa3d Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 09:08:35 -0500 Subject: [PATCH 034/197] Support the legacy createCipheriv modes and crypto.checkPrime/generatePrime --- .../limitations/builtin-apis/crypto.mdx | 5 +- .../src/generated/node-v24-compatibility.json | 24 +- .../generated/node-v24-internal.json | 40 +- internal/compatibility/static-support.json | 8 +- .../ambient/scriptc-node-fallback.d.ts | 58 +- .../compiler/src/backend/llvm/lib-shared.ts | 6 + .../src/backend/target-diagnostics.ts | 8 +- .../src/frontend/lowering/builtins/crypto.ts | 77 ++- .../src/frontend/lowering/surfaces.ts | 9 +- packages/compiler/src/ir/builtin-effects.ts | 6 + .../compiler/src/ir/builtin-signatures.ts | 10 + packages/compiler/src/ir/runtime-features.ts | 18 +- packages/compiler/src/library/ir-policy.ts | 2 + packages/compiler/surface-manifest.json | 26 +- .../corpus/7150-cipher-legacy-modes.ts.json | 6 + .../corpus/7151-cipher-legacy-errors.cjs.json | 6 + .../tests/corpus/7152-crypto-prime.ts.json | 6 + .../corpus/7153-crypto-prime-errors.cjs.json | 6 + packages/runtime/src/scr_cipher.c | 324 +++++++++-- packages/runtime/src/scr_crypto.h | 8 + packages/runtime/src/scr_dh.c | 536 +++++++++++++++++- tests/corpus/7150-cipher-legacy-modes.ts | 74 +++ tests/corpus/7151-cipher-legacy-errors.cjs | 83 +++ tests/corpus/7152-crypto-prime.ts | 96 ++++ tests/corpus/7153-crypto-prime-errors.cjs | 80 +++ tests/fixtures/node-types/fenced.ts | 8 +- .../__snapshots__/node-types-fenced.txt | 14 +- .../7150-cipher-legacy-modes.ts.json | 1 + .../7151-cipher-legacy-errors.cjs.json | 1 + .../corpus-costs/7152-crypto-prime.ts.json | 1 + .../7153-crypto-prime-errors.cjs.json | 1 + tests/harness/errors.test.ts | 5 +- 32 files changed, 1445 insertions(+), 108 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7150-cipher-legacy-modes.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7151-cipher-legacy-errors.cjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7152-crypto-prime.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7153-crypto-prime-errors.cjs.json create mode 100644 tests/corpus/7150-cipher-legacy-modes.ts create mode 100644 tests/corpus/7151-cipher-legacy-errors.cjs create mode 100644 tests/corpus/7152-crypto-prime.ts create mode 100644 tests/corpus/7153-crypto-prime-errors.cjs create mode 100644 tests/harness/corpus-costs/7150-cipher-legacy-modes.ts.json create mode 100644 tests/harness/corpus-costs/7151-cipher-legacy-errors.cjs.json create mode 100644 tests/harness/corpus-costs/7152-crypto-prime.ts.json create mode 100644 tests/harness/corpus-costs/7153-crypto-prime-errors.cjs.json diff --git a/docs/content/limitations/builtin-apis/crypto.mdx b/docs/content/limitations/builtin-apis/crypto.mdx index c84ce3e52..971065408 100644 --- a/docs/content/limitations/builtin-apis/crypto.mdx +++ b/docs/content/limitations/builtin-apis/crypto.mdx @@ -8,7 +8,7 @@ `randomFill`, `randomFillSync` (every view kind and ArrayBuffer), `randomInt`, `randomUUID`, and `randomUUIDv7` follow Node 24.22, including callback timing; a `randomInt` callback whose error parameter is typed `Error | null` receives `null` where Node passes `undefined`. Scrypt supports its default parameters, numeric options, aliases, and memory limit. Both KDFs accept strings, Buffers, numeric typed arrays, DataViews, and fixed-length ArrayBuffers. - `createCipheriv` and `createDecipheriv` support aes-128, aes-192, and aes-256 in CBC, CTR, and GCM modes and chacha20-poly1305, on native targets, through `update()`, `final()`, `setAAD()`, `setAutoPadding()`, `getAuthTag()`, and `setAuthTag()`; other ciphers and wasm32-wasi builds are refused. + `createCipheriv` and `createDecipheriv` support aes-128, aes-192, and aes-256 in CBC, CTR, GCM, ECB, CFB, CFB8, CFB1, and OFB modes, des-ede3 in CBC, ECB, CFB, CFB8, CFB1, and OFB modes, des-ede in CBC, ECB, CFB, and OFB modes, chacha20, and chacha20-poly1305, on native targets, through `update()`, `final()`, `setAAD()`, `setAutoPadding()`, `getAuthTag()`, and `setAuthTag()`; other ciphers and wasm32-wasi builds are refused. Cipheriv and Decipheriv values are also Transforms: they can be written, piped into, used as `pipeline()` stages, read, iterated with `for await`, and listened to, and their stream properties read as in Node; written chunks go through `update()`, and the end pushes `final()` or reports its error through `'error'`. Through `unknown` and `any` slots and untyped JavaScript, in a program that uses streams, their stream properties and their `on`, `once`, `off`, `pipe`, `unpipe`, `read`, `pause`, `resume`, `setEncoding`, `write`, `end`, `destroy`, `cork`, and `uncork` members work; `write` and `end` take no encoding or callback there, and the other stream members throw a coded SC2020 error. Hash, Hmac, Cipheriv, and Decipheriv streams count a written string's bytes in `writableLength`, where Node counts the string's length. @@ -25,6 +25,9 @@ `Object.keys()` and `in` do not see a DiffieHellman's own `verifyError` property, and `JSON.stringify()` and `instanceof` are refused. After `createDiffieHellman(primeLength)` throws `Invalid DH parameters`, the next OpenSSL error Node throws can list the failed generation's errors in `opensslErrorStack`; errors in a compiled binary list only their own. `crypto.diffieHellman`, which takes DH, EC, or X25519 KeyObjects, and the `DiffieHellman` and `DiffieHellmanGroup` constructors are not available. + `checkPrime`, `checkPrimeSync`, `generatePrime`, and `generatePrimeSync` support bigint, ArrayBuffer, typed-array, Buffer, and DataView values and the `checks`, `safe`, `add`, `rem`, and `bigint` options on native targets, with Node's errors; the callback forms compute before they return and call back in a later macrotask. + Candidates and sizes longer than 262144 bits throw a coded SC2020 error, and wasm32-wasi builds are refused. + Node calls their callbacks with `undefined` as the error, which a callback whose error parameter is typed `Error | null`, as @types/node declares it, receives as `null`. RSA, EC, and Ed25519 private and public KeyObjects come from `generateKeyPairSync`, `generateKeyPair`, `createPrivateKey`, and `createPublicKey`, on native targets. The EC curves are prime256v1 (P-256), secp384r1 (P-384), secp521r1 (P-521), and secp256k1. Keys read and write PEM and DER as PKCS#1, SEC1, PKCS#8, and SubjectPublicKeyInfo, and JWK (Ed25519 keys use PKCS#8, SubjectPublicKeyInfo, and JWK); `createPublicKey` also takes a private key or an X.509 certificate. diff --git a/docs/src/generated/node-v24-compatibility.json b/docs/src/generated/node-v24-compatibility.json index 574c49ff4..c341da852 100644 --- a/docs/src/generated/node-v24-compatibility.json +++ b/docs/src/generated/node-v24-compatibility.json @@ -11085,9 +11085,9 @@ "inherited": true }, "static": { - "status": "not-implemented", - "detail": "No static compiler lowering is registered for this API yet.", - "verification": "registry-gap" + "status": "partial", + "detail": "Implemented for the call shapes accepted by the compiler lowering.", + "verification": "test-backed" } }, { @@ -11108,9 +11108,9 @@ "inherited": true }, "static": { - "status": "not-implemented", - "detail": "No static compiler lowering is registered for this API yet.", - "verification": "registry-gap" + "status": "partial", + "detail": "Implemented for the call shapes accepted by the compiler lowering.", + "verification": "test-backed" } }, { @@ -11614,9 +11614,9 @@ "inherited": true }, "static": { - "status": "not-implemented", - "detail": "No static compiler lowering is registered for this API yet.", - "verification": "registry-gap" + "status": "partial", + "detail": "Implemented for the call shapes accepted by the compiler lowering.", + "verification": "test-backed" } }, { @@ -11637,9 +11637,9 @@ "inherited": true }, "static": { - "status": "not-implemented", - "detail": "No static compiler lowering is registered for this API yet.", - "verification": "registry-gap" + "status": "partial", + "detail": "Implemented for the call shapes accepted by the compiler lowering.", + "verification": "test-backed" } }, { diff --git a/internal/compatibility/generated/node-v24-internal.json b/internal/compatibility/generated/node-v24-internal.json index 31ce5da02..9393fcaa8 100644 --- a/internal/compatibility/generated/node-v24-internal.json +++ b/internal/compatibility/generated/node-v24-internal.json @@ -19328,6 +19328,10 @@ "tests/corpus/7084-crypto-key-pairs.cjs", "tests/corpus/7122-webcrypto-node-crypto.ts", "tests/corpus/7123-webcrypto-node-crypto.cjs", + "tests/corpus/7150-cipher-legacy-modes.ts", + "tests/corpus/7151-cipher-legacy-errors.cjs", + "tests/corpus/7152-crypto-prime.ts", + "tests/corpus/7153-crypto-prime-errors.cjs", "tests/corpus/node24-crypto-kdf-errors.ts", "tests/corpus/node24-crypto-kdf-options.ts", "tests/corpus/node24-crypto-kdf.ts", @@ -19658,8 +19662,12 @@ "inherited": true }, "static": { - "status": "not-implemented", - "evidence": "compiler-unmatched:crypto" + "status": "partial", + "evidence": "surface-manifest:node-builtin.crypto.checkPrime", + "tests": [ + "tests/corpus/7152-crypto-prime.ts", + "tests/corpus/7153-crypto-prime-errors.cjs" + ] }, "anchorSource": "exact" }, @@ -19681,8 +19689,12 @@ "inherited": true }, "static": { - "status": "not-implemented", - "evidence": "compiler-unmatched:crypto" + "status": "partial", + "evidence": "surface-manifest:node-builtin.crypto.checkPrimeSync", + "tests": [ + "tests/corpus/7152-crypto-prime.ts", + "tests/corpus/7153-crypto-prime-errors.cjs" + ] }, "anchorSource": "exact" }, @@ -19707,6 +19719,8 @@ "status": "partial", "evidence": "surface-manifest:node-builtin.crypto.createCipheriv", "tests": [ + "tests/corpus/7150-cipher-legacy-modes.ts", + "tests/corpus/7151-cipher-legacy-errors.cjs", "tests/corpus/5250-cipher-aes-gcm.ts", "tests/corpus/5251-cipher-aes-cbc.ts", "tests/corpus/5252-cipher-aes-ctr.ts", @@ -19746,6 +19760,8 @@ "status": "partial", "evidence": "surface-manifest:node-builtin.crypto.createDecipheriv", "tests": [ + "tests/corpus/7150-cipher-legacy-modes.ts", + "tests/corpus/7151-cipher-legacy-errors.cjs", "tests/corpus/5250-cipher-aes-gcm.ts", "tests/corpus/5251-cipher-aes-cbc.ts", "tests/corpus/5252-cipher-aes-ctr.ts", @@ -20364,8 +20380,12 @@ "inherited": true }, "static": { - "status": "not-implemented", - "evidence": "compiler-unmatched:crypto" + "status": "partial", + "evidence": "surface-manifest:node-builtin.crypto.generatePrime", + "tests": [ + "tests/corpus/7152-crypto-prime.ts", + "tests/corpus/7153-crypto-prime-errors.cjs" + ] }, "anchorSource": "exact" }, @@ -20387,8 +20407,12 @@ "inherited": true }, "static": { - "status": "not-implemented", - "evidence": "compiler-unmatched:crypto" + "status": "partial", + "evidence": "surface-manifest:node-builtin.crypto.generatePrimeSync", + "tests": [ + "tests/corpus/7152-crypto-prime.ts", + "tests/corpus/7153-crypto-prime-errors.cjs" + ] }, "anchorSource": "exact" }, diff --git a/internal/compatibility/static-support.json b/internal/compatibility/static-support.json index a59d6365a..06aad26cf 100644 --- a/internal/compatibility/static-support.json +++ b/internal/compatibility/static-support.json @@ -42,8 +42,10 @@ "node-builtin.child_process": ["tests/corpus/1361-spawn-events.ts", "tests/corpus/1565-spawn-pipe-streams.ts", "tests/corpus/2911-execfile-callback.ts", "tests/corpus/2939-child-shell-text.ts", "tests/corpus/2946-child-fork-ipc/main.ts", "tests/corpus/3127-spawn-runtime-options.ts", "tests/corpus/3130-child-generic-streams.ts", "tests/corpus/3131-child-rest-listeners.js", "tests/corpus/3132-child-generic-stream-completion.ts"], "node-builtin.cluster": ["tests/corpus/957-builtins-namespace.ts"], "node-builtin.console": ["tests/corpus/console-imports.cjs", "tests/corpus/2440-console-inspect-args.ts", "tests/corpus/console-native-value.js", "tests/harness/npm-static.test.ts"], - "node-builtin.crypto.createCipheriv": ["tests/corpus/5250-cipher-aes-gcm.ts", "tests/corpus/5251-cipher-aes-cbc.ts", "tests/corpus/5252-cipher-aes-ctr.ts", "tests/corpus/5253-cipher-chacha20-poly1305.ts", "tests/corpus/5254-cipher-errors.js", "tests/corpus/5255-cipher-encodings.cjs", "tests/corpus/5256-cipher-deprecation.cjs", "tests/corpus/5257-cipher-vectors.ts", "tests/corpus/5258-cipher-handles.js", "tests/corpus/5259-cipher-chunked.ts", "tests/corpus/7055-cipher-streams.ts", "tests/corpus/7056-cipher-stream-errors.ts", "tests/corpus/7057-cipher-streams.mjs", "tests/fixtures/node-types/cipher.ts"], - "node-builtin.crypto.createDecipheriv": ["tests/corpus/5250-cipher-aes-gcm.ts", "tests/corpus/5251-cipher-aes-cbc.ts", "tests/corpus/5252-cipher-aes-ctr.ts", "tests/corpus/5253-cipher-chacha20-poly1305.ts", "tests/corpus/5254-cipher-errors.js", "tests/corpus/5255-cipher-encodings.cjs", "tests/corpus/5256-cipher-deprecation.cjs", "tests/corpus/5257-cipher-vectors.ts", "tests/corpus/5258-cipher-handles.js", "tests/corpus/5259-cipher-chunked.ts", "tests/corpus/7055-cipher-streams.ts", "tests/corpus/7056-cipher-stream-errors.ts", "tests/corpus/7057-cipher-streams.mjs", "tests/fixtures/node-types/cipher.ts"], + "node-builtin.crypto.checkPrime": ["tests/corpus/7152-crypto-prime.ts", "tests/corpus/7153-crypto-prime-errors.cjs"], + "node-builtin.crypto.checkPrimeSync": ["tests/corpus/7152-crypto-prime.ts", "tests/corpus/7153-crypto-prime-errors.cjs"], + "node-builtin.crypto.createCipheriv": ["tests/corpus/7150-cipher-legacy-modes.ts", "tests/corpus/7151-cipher-legacy-errors.cjs", "tests/corpus/5250-cipher-aes-gcm.ts", "tests/corpus/5251-cipher-aes-cbc.ts", "tests/corpus/5252-cipher-aes-ctr.ts", "tests/corpus/5253-cipher-chacha20-poly1305.ts", "tests/corpus/5254-cipher-errors.js", "tests/corpus/5255-cipher-encodings.cjs", "tests/corpus/5256-cipher-deprecation.cjs", "tests/corpus/5257-cipher-vectors.ts", "tests/corpus/5258-cipher-handles.js", "tests/corpus/5259-cipher-chunked.ts", "tests/corpus/7055-cipher-streams.ts", "tests/corpus/7056-cipher-stream-errors.ts", "tests/corpus/7057-cipher-streams.mjs", "tests/fixtures/node-types/cipher.ts"], + "node-builtin.crypto.createDecipheriv": ["tests/corpus/7150-cipher-legacy-modes.ts", "tests/corpus/7151-cipher-legacy-errors.cjs", "tests/corpus/5250-cipher-aes-gcm.ts", "tests/corpus/5251-cipher-aes-cbc.ts", "tests/corpus/5252-cipher-aes-ctr.ts", "tests/corpus/5253-cipher-chacha20-poly1305.ts", "tests/corpus/5254-cipher-errors.js", "tests/corpus/5255-cipher-encodings.cjs", "tests/corpus/5256-cipher-deprecation.cjs", "tests/corpus/5257-cipher-vectors.ts", "tests/corpus/5258-cipher-handles.js", "tests/corpus/5259-cipher-chunked.ts", "tests/corpus/7055-cipher-streams.ts", "tests/corpus/7056-cipher-stream-errors.ts", "tests/corpus/7057-cipher-streams.mjs", "tests/fixtures/node-types/cipher.ts"], "node-builtin.crypto.createDiffieHellman": ["tests/corpus/6706-dh-groups.ts", "tests/corpus/6707-dh-explicit-params.ts", "tests/corpus/6708-dh-errors.js", "tests/corpus/6709-dh-generate.ts", "tests/corpus/6710-dh-encodings.cjs", "tests/corpus/6711-dh-handles.ts", "tests/fixtures/node-types/dh.ts"], "node-builtin.crypto.createDiffieHellmanGroup": ["tests/corpus/6706-dh-groups.ts", "tests/corpus/6707-dh-explicit-params.ts", "tests/corpus/6708-dh-errors.js", "tests/corpus/6709-dh-generate.ts", "tests/corpus/6710-dh-encodings.cjs", "tests/corpus/6711-dh-handles.ts", "tests/fixtures/node-types/dh.ts"], "node-builtin.crypto.createECDH": ["tests/corpus/6700-ecdh-shared-secret.ts", "tests/corpus/6701-ecdh-formats.ts", "tests/corpus/6702-ecdh-errors.js", "tests/corpus/6703-ecdh-private-keys.ts", "tests/corpus/6704-ecdh-set-public-key.cjs", "tests/corpus/6705-ecdh-handles.ts", "tests/fixtures/node-types/ecdh.ts"], @@ -120,6 +122,8 @@ "node-builtin.os.version": ["tests/corpus/5140-os-host-facts.ts"], "node-builtin.os.constants": ["tests/corpus/5141-os-constants.ts"], "node-builtin.fs.constants": ["tests/corpus/6207-fs-constants-object.ts", "tests/corpus/6208-fs-constants-require.cjs"], + "node-builtin.crypto.generatePrime": ["tests/corpus/7152-crypto-prime.ts", "tests/corpus/7153-crypto-prime-errors.cjs"], + "node-builtin.crypto.generatePrimeSync": ["tests/corpus/7152-crypto-prime.ts", "tests/corpus/7153-crypto-prime-errors.cjs"], "node-builtin.crypto.getDiffieHellman": ["tests/corpus/6706-dh-groups.ts", "tests/corpus/6707-dh-explicit-params.ts", "tests/corpus/6708-dh-errors.js", "tests/corpus/6709-dh-generate.ts", "tests/corpus/6710-dh-encodings.cjs", "tests/corpus/6711-dh-handles.ts", "tests/fixtures/node-types/dh.ts"], "node-builtin.crypto.getRandomValues": ["tests/corpus/6204-crypto-get-random-values.ts", "tests/corpus/6205-crypto-get-random-values.js"], "node-builtin.url.parse": ["tests/corpus/5220-url-parse-warnings.ts", "tests/corpus/5221-url-parse-shapes.ts", "tests/corpus/5222-url-parse-hosts.ts", "tests/corpus/5223-url-parse-query.ts", "tests/corpus/5227-url-legacy-dependency/main.ts", "tests/corpus/5229-url-legacy-errors.cjs", "packages/runtime/test/url-legacy.test.ts", "tests/fixtures/node-types/url-legacy.ts"], diff --git a/packages/compiler/ambient/scriptc-node-fallback.d.ts b/packages/compiler/ambient/scriptc-node-fallback.d.ts index 2a3f11be9..519608393 100644 --- a/packages/compiler/ambient/scriptc-node-fallback.d.ts +++ b/packages/compiler/ambient/scriptc-node-fallback.d.ts @@ -3171,7 +3171,8 @@ declare module "crypto" { callback: (error: Error | null, buffer: Buffer) => void, ): void; export type BinaryLike = string | ArrayBuffer | ArrayBufferView; - /* Cipheriv/Decipheriv: aes-128/192/256 in cbc, ctr, and gcm modes and + /* Cipheriv/Decipheriv: aes-128/192/256 in cbc, ctr, gcm, ecb, cfb, cfb8, + * cfb1, and ofb modes, des-ede3 and des-ede, chacha20, and * chacha20-poly1305. Both are Transforms: written chunks go through * update(), and the end pushes final(). */ export type CipherGCMTypes = "aes-128-gcm" | "aes-192-gcm" | "aes-256-gcm"; @@ -3351,6 +3352,61 @@ declare module "crypto" { ): DiffieHellman; export function getDiffieHellman(groupName: string): DiffieHellmanGroup; export function createDiffieHellmanGroup(name: string): DiffieHellmanGroup; + export type LargeNumberLike = ArrayBufferView | SharedArrayBuffer | ArrayBuffer | bigint; + export interface GeneratePrimeOptions { + add?: LargeNumberLike | undefined; + rem?: LargeNumberLike | undefined; + safe?: boolean | undefined; + bigint?: boolean | undefined; + } + export interface GeneratePrimeOptionsBigInt extends GeneratePrimeOptions { + bigint: true; + } + export interface GeneratePrimeOptionsArrayBuffer extends GeneratePrimeOptions { + bigint?: false | undefined; + } + export function generatePrime( + size: number, + callback: (err: Error | null, prime: ArrayBuffer) => void, + ): void; + export function generatePrime( + size: number, + options: GeneratePrimeOptionsBigInt, + callback: (err: Error | null, prime: bigint) => void, + ): void; + export function generatePrime( + size: number, + options: GeneratePrimeOptionsArrayBuffer, + callback: (err: Error | null, prime: ArrayBuffer) => void, + ): void; + export function generatePrime( + size: number, + options: GeneratePrimeOptions, + callback: (err: Error | null, prime: ArrayBuffer | bigint) => void, + ): void; + export function generatePrimeSync(size: number): ArrayBuffer; + export function generatePrimeSync(size: number, options: GeneratePrimeOptionsBigInt): bigint; + export function generatePrimeSync( + size: number, + options: GeneratePrimeOptionsArrayBuffer, + ): ArrayBuffer; + export function generatePrimeSync( + size: number, + options: GeneratePrimeOptions, + ): ArrayBuffer | bigint; + export interface CheckPrimeOptions { + checks?: number | undefined; + } + export function checkPrime( + value: LargeNumberLike, + callback: (err: Error | null, result: boolean) => void, + ): void; + export function checkPrime( + value: LargeNumberLike, + options: CheckPrimeOptions, + callback: (err: Error | null, result: boolean) => void, + ): void; + export function checkPrimeSync(candidate: LargeNumberLike, options?: CheckPrimeOptions): boolean; export interface ScryptOptions { N?: number; r?: number; diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index 9a65d9a24..23f0a0291 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -579,6 +579,10 @@ export const LIB_FN_SYMS: Record = { "crypto.ecdhNew": "scr_crypto_ecdh_new", "crypto.ecdhConvertKey": "scr_crypto_ecdh_convert_key", "crypto.dhNew": "scr_crypto_dh_new", + "crypto.checkPrimeSync": "scr_crypto_check_prime_sync", + "crypto.checkPrimeCb": "scr_crypto_check_prime_async", + "crypto.generatePrimeSync": "scr_crypto_generate_prime_sync", + "crypto.generatePrimeCb": "scr_crypto_generate_prime_async", "crypto.dhGroupNew": "scr_crypto_dh_group_new", "crypto.hmacNewStr": "scr_crypto_hmac_new_str", "crypto.hmacNewBytes": "scr_crypto_hmac_new_bytes", @@ -1600,6 +1604,8 @@ export function mockTimerSym( export const USES_TIMERS_LIB_FNS = new Set([ "crypto.generateKeyPairCb", + "crypto.checkPrimeCb", + "crypto.generatePrimeCb", "process.emitWarning", // Constructor-function streams defer their events on the stream ticks. "dyn.builtinStreamClass", diff --git a/packages/compiler/src/backend/target-diagnostics.ts b/packages/compiler/src/backend/target-diagnostics.ts index 3e97d0c10..236ce12cf 100644 --- a/packages/compiler/src/backend/target-diagnostics.ts +++ b/packages/compiler/src/backend/target-diagnostics.ts @@ -3,6 +3,7 @@ import type { LlvmUnsupportedError } from "./llvm/emitter.js"; import type { ScrDiagnostic } from "../diagnostics/diagnostic.js"; import { type IrModule, type SrcLoc } from "../ir/ir.js"; import { + DH_LIB_FNS, PKI_LIB_FNS, moduleUsesFetch, moduleUsesWebCryptoAsymmetric, @@ -232,9 +233,12 @@ export function moduleWasiUnavailableSurface( }; return false; } - if (node.fn === "crypto.dhNew" || node.fn === "crypto.dhGroupNew") { + if (DH_LIB_FNS.has(node.fn)) { found = { - surface: "DiffieHellman (the wasm32-wasi runtime does not include the mbedTLS library)", + surface: + node.fn === "crypto.dhNew" || node.fn === "crypto.dhGroupNew" + ? "DiffieHellman (the wasm32-wasi runtime does not include the mbedTLS library)" + : "checkPrime and generatePrime (the wasm32-wasi runtime does not include the mbedTLS library)", loc, }; return false; diff --git a/packages/compiler/src/frontend/lowering/builtins/crypto.ts b/packages/compiler/src/frontend/lowering/builtins/crypto.ts index 7a1791197..47e66f9b2 100644 --- a/packages/compiler/src/frontend/lowering/builtins/crypto.ts +++ b/packages/compiler/src/frontend/lowering/builtins/crypto.ts @@ -702,6 +702,8 @@ export function lowerCryptoModuleCall( } const pki = lowerPkiCall(lowerer, expr, bi.member, args, loc); if (pki) return pki; + const prime = lowerPrimeCall(lowerer, expr, bi.member, args, loc); + if (prime) return prime; if (bi.member === "createSecretKey") { if (args.length < 1 || args.length > 2 || args.some(ts.isSpreadElement)) lowerer.noLowering( @@ -867,11 +869,29 @@ const IMPLEMENTED_CIPHERS = new Set([ "id-aes192-gcm", "id-aes256-gcm", "chacha20-poly1305", + ...["128", "192", "256"].flatMap((bits) => + ["ecb", "cfb", "cfb8", "cfb1", "ofb"].map((mode) => `aes-${bits}-${mode}`), + ), + "des-ede3-cbc", + "des3", + "des-ede3", + "des-ede3-ecb", + "des-ede3-cfb", + "des-ede3-cfb8", + "des-ede3-cfb1", + "des-ede3-ofb", + "des-ede-cbc", + "des-ede", + "des-ede-ecb", + "des-ede-cfb", + "des-ede-ofb", + "chacha20", ]); const SUPPORTED_CIPHERS_HINT = - "the supported ciphers are aes-128-cbc, aes-192-cbc, aes-256-cbc, aes-128-ctr, aes-192-ctr, " + - "aes-256-ctr, aes-128-gcm, aes-192-gcm, aes-256-gcm, and chacha20-poly1305"; + "the supported ciphers are aes-128, aes-192 and aes-256 in cbc, ctr, gcm, ecb, cfb, cfb8, cfb1 " + + "and ofb modes, des-ede3 in cbc, ecb, cfb, cfb8, cfb1 and ofb modes, des-ede in cbc, ecb, cfb " + + "and ofb modes, chacha20, and chacha20-poly1305"; /** A literal cipher name that Node knows but the runtime does not * implement is refused at compile time. Unknown literal names keep Node's @@ -1346,6 +1366,59 @@ export function isWebCryptoReceiver(lowerer: Lowerer, receiver: ts.Expression): return false; } +/** Node calls checkPrime's and generatePrime's callbacks with undefined as + * the error, where @types/node declares Error | null: whether this + * callback's error parameter can hold undefined. One that cannot receives + * null instead. */ +function takesUndefinedError(lowerer: Lowerer, callback: ts.Expression): boolean { + const type = lowerer.irTypeOf(callback); + if (type.kind !== "func") return true; + const error = type.params[0]; + if (error === undefined || error.kind === "dyn" || error.kind === "undefinedT") return true; + if (error.kind !== "union") return false; + return !!lowerer.unions.get(error.unionId)?.arms.some((arm) => arm.kind === "undefinedT"); +} + +/** checkPrime, checkPrimeSync, generatePrime and generatePrimeSync + * (scr_dh.c): checked-dynamic arguments, which the runtime checks as Node + * does (a function in the options position is the callback), and the + * sync forms' boolean, ArrayBuffer or bigint converted to the call site's + * type. Null for other members. */ +function lowerPrimeCall( + lowerer: Lowerer, + expr: ts.CallExpression, + member: string, + args: readonly ts.Expression[], + loc: SrcLoc, +): IrExpr | null { + const forms: Readonly> = { + checkPrimeSync: ["crypto.checkPrimeSync", 1, 2, "checkPrimeSync(candidate[, options])"], + checkPrime: ["crypto.checkPrimeCb", 1, 3, "checkPrime(candidate[, options], callback)"], + generatePrimeSync: ["crypto.generatePrimeSync", 1, 2, "generatePrimeSync(size[, options])"], + generatePrime: ["crypto.generatePrimeCb", 1, 3, "generatePrime(size[, options], callback)"], + }; + const form = forms[member]; + if (!form) return null; + const [fn, min, max, supported] = form; + if (args.length < min || args.length > max || args.some(ts.isSpreadElement)) + lowerer.noLowering( + `crypto.${member} with ${args.length} arguments`, + expr, + `${supported} is supported`, + ); + const values = Array.from({ length: max }, (_, i) => dynArg(lowerer, args[i], loc)); + if (fn === "crypto.checkPrimeCb" || fn === "crypto.generatePrimeCb") { + const callback = args[args.length - 1]; + const nullError = callback !== undefined && !takesUndefinedError(lowerer, callback); + values.push({ kind: "boolLit", value: nullError, type: BOOL, loc }); + return { kind: "libCall", fn, args: values, type: VOID, loc }; + } + const call: IrExpr = { kind: "libCall", fn, args: values, type: DYN, loc }; + const result = lowerer.mapTypeOf(lowerer.typeOf(expr)); + if (result === null || result.kind === "dyn") return call; + return lowerer.coerceInto(expr, call, result); +} + /** The asymmetric-key and signature members (scr_pki.c): checked-dynamic * arguments, and the result converted to the call site's type * (KeyObject, Sign and Verify are checked-dynamic handles; key pairs, diff --git a/packages/compiler/src/frontend/lowering/surfaces.ts b/packages/compiler/src/frontend/lowering/surfaces.ts index 0167adf12..cb16fd257 100644 --- a/packages/compiler/src/frontend/lowering/surfaces.ts +++ b/packages/compiler/src/frontend/lowering/surfaces.ts @@ -1590,6 +1590,12 @@ export const BUILTIN_MODULE_FNS: Record< createDiffieHellman: { fn: "crypto.dhNew", params: [DYN, DYN, DYN, DYN], result: DYN }, getDiffieHellman: { fn: "crypto.dhGroupNew", params: [DYN], result: DYN }, createDiffieHellmanGroup: { fn: "crypto.dhGroupNew", params: [DYN], result: DYN }, + // Prime checks and generation (lowerPrimeCall): checked-dynamic + // arguments, results converted to the call site's type. + checkPrimeSync: { fn: "crypto.checkPrimeSync", params: [], result: DYN }, + checkPrime: { fn: "crypto.checkPrimeCb", params: [], result: VOID }, + generatePrimeSync: { fn: "crypto.generatePrimeSync", params: [], result: DYN }, + generatePrime: { fn: "crypto.generatePrimeCb", params: [], result: VOID }, }, zlib: { // The default-options convenience family is special-cased in @@ -3099,7 +3105,8 @@ export const BUILTIN_MODULE_FENCE_HINTS: Record< ["getCipherInfo"].map((m) => [ m, "cipher metadata has no lowering — createCipheriv and createDecipheriv support " + - "aes-128/192/256 in cbc, ctr, and gcm modes and chacha20-poly1305", + "aes-128/192/256 in cbc, ctr, gcm, ecb, cfb, cfb8, cfb1, and ofb modes, des-ede3 and " + + "des-ede, chacha20, and chacha20-poly1305", ]), ), setFips: diff --git a/packages/compiler/src/ir/builtin-effects.ts b/packages/compiler/src/ir/builtin-effects.ts index 16db45b6c..e98ae0bd9 100644 --- a/packages/compiler/src/ir/builtin-effects.ts +++ b/packages/compiler/src/ir/builtin-effects.ts @@ -16,6 +16,12 @@ export const MAY_THROW_LIB_FNS: ReadonlySet = new Set([ // Node's prime, generator, size, encoding and group-name errors. "crypto.dhNew", "crypto.dhGroupNew", + // Node's candidate, size, options, add and rem errors, and OpenSSL's + // generation errors. + "crypto.checkPrimeSync", + "crypto.checkPrimeCb", + "crypto.generatePrimeSync", + "crypto.generatePrimeCb", // node:test's { timeout } and { concurrency } range checks. "test.register", "test.registerEmpty", diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index fcdbe8694..edadaa244 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -2820,6 +2820,16 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ /** getDiffieHellman(groupName) and createDiffieHellmanGroup(name): a * native DiffieHellmanGroup handle. */ "crypto.dhGroupNew": { argTypes: [DYN], result: DYN }, + /** checkPrimeSync(candidate, options) and generatePrimeSync(size, + * options) over checked-dynamic arguments (scr_dh.c): a boolean, and an + * ArrayBuffer or bigint, as checked-dynamic results. The Cb forms take + * the callback and whether its error parameter takes null for Node's + * undefined, and call it in a later macrotask; Node's argument errors + * throw (may-throw). */ + "crypto.checkPrimeSync": { argTypes: [DYN, DYN], result: DYN }, + "crypto.checkPrimeCb": { argTypes: [DYN, DYN, DYN, BOOL], result: VOID }, + "crypto.generatePrimeSync": { argTypes: [DYN, DYN], result: DYN }, + "crypto.generatePrimeCb": { argTypes: [DYN, DYN, DYN, BOOL], result: VOID }, "crypto.hmacNewStr": { argTypes: [STRING, STRING], result: CRYPTOHMAC_T }, "crypto.hmacNewBytes": { argTypes: [STRING, BYTES_U8], result: CRYPTOHMAC_T }, "crypto.hashUpdateStr": { argTypes: [CRYPTOHASH_T, STRING], result: CRYPTOHASH_T }, diff --git a/packages/compiler/src/ir/runtime-features.ts b/packages/compiler/src/ir/runtime-features.ts index 5b984f318..ec2edb931 100644 --- a/packages/compiler/src/ir/runtime-features.ts +++ b/packages/compiler/src/ir/runtime-features.ts @@ -626,6 +626,16 @@ const rule = (r: RuntimeFeatureRule): RuntimeFeatureRule => r; /* The HTTP, HTTPS, HTTP/2 and TLS surfaces, which link the net unit. */ const NET_PREFIXES: readonly string[] = ["net.", "http.", "tls.", "https.", "http2."]; +/** The scr_dh.c entry points (the dh unit). */ +export const DH_LIB_FNS: ReadonlySet = new Set([ + "crypto.dhNew", + "crypto.dhGroupNew", + "crypto.checkPrimeSync", + "crypto.checkPrimeCb", + "crypto.generatePrimeSync", + "crypto.generatePrimeCb", +]); + /** The scr_pki.c entry points (the pki unit). */ export const PKI_LIB_FNS: ReadonlySet = new Set([ "crypto.createPrivateKey", @@ -820,9 +830,10 @@ const RUNTIME_FEATURE_RULES = { * process-warning unit, and unknown encodings and formats render through * the inspect unit. */ ecdh: rule({ fns: ["crypto.ecdhNew", "crypto.ecdhConvertKey"] }), - /** scr_dh.c over the mbedTLS archive; unknown encodings render through + /** scr_dh.c over the mbedTLS archive, also for checkPrime and + * generatePrime; unknown encodings and argument values render through * the inspect unit. */ - dh: rule({ fns: ["crypto.dhNew", "crypto.dhGroupNew"] }), + dh: rule({ fns: [...DH_LIB_FNS] }), emitter: rule({ initial: (mod) => mod.workers === true || (mod.classes ?? []).some((c) => c.name === RUNTIME_EMITTER_CLASS), @@ -872,8 +883,7 @@ const RUNTIME_FEATURE_RULES = { "webcrypto.unwrapKey", "crypto.ecdhNew", "crypto.ecdhConvertKey", - "crypto.dhNew", - "crypto.dhGroupNew", + ...DH_LIB_FNS, ...BUILTIN_CLASS_LIB_FNS, ...UTIL_CALLABLE_LIB_FNS, ], diff --git a/packages/compiler/src/library/ir-policy.ts b/packages/compiler/src/library/ir-policy.ts index fb0575121..e8fad6dd1 100644 --- a/packages/compiler/src/library/ir-policy.ts +++ b/packages/compiler/src/library/ir-policy.ts @@ -54,6 +54,8 @@ const LIB_MODE_REFUSED_PREFIXES: readonly [string, string][] = [ ["crypto.ecdh", "crypto.createECDH and ECDH.convertKey"], ["crypto.dhNew", "crypto.createDiffieHellman"], ["crypto.dhGroupNew", "crypto.getDiffieHellman and createDiffieHellmanGroup"], + ["crypto.checkPrime", "crypto.checkPrime and checkPrimeSync"], + ["crypto.generatePrime", "crypto.generatePrime and generatePrimeSync"], ["crypto.createPrivateKey", "asymmetric KeyObjects, generateKeyPair, sign and verify"], ["crypto.createPublicKey", "asymmetric KeyObjects, generateKeyPair, sign and verify"], ["crypto.createSign", "asymmetric KeyObjects, generateKeyPair, sign and verify"], diff --git a/packages/compiler/surface-manifest.json b/packages/compiler/surface-manifest.json index 45b3edbaa..d6eb69f6f 100644 --- a/packages/compiler/surface-manifest.json +++ b/packages/compiler/surface-manifest.json @@ -480,6 +480,18 @@ "status": "static", "note": "recognized module (bare and node:-prefixed specifiers)" }, + { + "id": "node-builtin.crypto.checkPrime", + "kind": "node-builtin", + "name": "crypto.checkPrime", + "status": "static" + }, + { + "id": "node-builtin.crypto.checkPrimeSync", + "kind": "node-builtin", + "name": "crypto.checkPrimeSync", + "status": "static" + }, { "id": "node-builtin.crypto.createCipheriv", "kind": "node-builtin", @@ -584,13 +596,25 @@ "name": "crypto.generateKeySync", "status": "static" }, + { + "id": "node-builtin.crypto.generatePrime", + "kind": "node-builtin", + "name": "crypto.generatePrime", + "status": "static" + }, + { + "id": "node-builtin.crypto.generatePrimeSync", + "kind": "node-builtin", + "name": "crypto.generatePrimeSync", + "status": "static" + }, { "id": "node-builtin.crypto.getCipherInfo", "kind": "node-builtin", "name": "crypto.getCipherInfo", "status": "unsupported", "code": "SC2020", - "note": "cipher metadata has no lowering — createCipheriv and createDecipheriv support aes-128/192/256 in cbc, ctr, and gcm modes and chacha20-poly1305" + "note": "cipher metadata has no lowering — createCipheriv and createDecipheriv support aes-128/192/256 in cbc, ctr, gcm, ecb, cfb, cfb8, cfb1, and ofb modes, des-ede3 and des-ede, chacha20, and chacha20-poly1305" }, { "id": "node-builtin.crypto.getDiffieHellman", diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7150-cipher-legacy-modes.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7150-cipher-legacy-modes.ts.json new file mode 100644 index 000000000..c0e5f9812 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7150-cipher-legacy-modes.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7150-cipher-legacy-modes.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7151-cipher-legacy-errors.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7151-cipher-legacy-errors.cjs.json new file mode 100644 index 000000000..1f37c6b64 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7151-cipher-legacy-errors.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7151-cipher-legacy-errors.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7152-crypto-prime.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7152-crypto-prime.ts.json new file mode 100644 index 000000000..722ebaacb --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7152-crypto-prime.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7152-crypto-prime.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7153-crypto-prime-errors.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7153-crypto-prime-errors.cjs.json new file mode 100644 index 000000000..9d121895a --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7153-crypto-prime-errors.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7153-crypto-prime-errors.cjs" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_cipher.c b/packages/runtime/src/scr_cipher.c index c8a10a731..00d229bad 100644 --- a/packages/runtime/src/scr_cipher.c +++ b/packages/runtime/src/scr_cipher.c @@ -20,16 +20,30 @@ #include "mbedtls/aes.h" #include "mbedtls/chacha20.h" +#include "mbedtls/des.h" #include "mbedtls/gcm.h" #include "mbedtls/poly1305.h" #include "mbedtls/platform_util.h" -typedef enum { SC_CBC, SC_CTR, SC_GCM, SC_CHACHA } ScCipherMode; +/* SC_CHACHA is chacha20-poly1305; SC_CHACHA20 the bare stream cipher. */ +typedef enum { + SC_CBC, + SC_CTR, + SC_GCM, + SC_CHACHA, + SC_ECB, + SC_CFB, + SC_CFB8, + SC_CFB1, + SC_OFB, + SC_CHACHA20, +} ScCipherMode; typedef struct { const char *name; ScCipherMode mode; unsigned key_len; + bool des; /* triple DES (two- or three-key) rather than AES */ } ScCipherSpec; /* Node's names for the implemented ciphers, including OpenSSL's aliases. @@ -41,6 +55,35 @@ static const ScCipherSpec sc_specs[] = { {"aes-128-gcm", SC_GCM, 16}, {"aes-192-gcm", SC_GCM, 24}, {"aes-256-gcm", SC_GCM, 32}, {"id-aes128-gcm", SC_GCM, 16}, {"id-aes192-gcm", SC_GCM, 24}, {"id-aes256-gcm", SC_GCM, 32}, {"chacha20-poly1305", SC_CHACHA, 32}, + {"aes-128-ecb", SC_ECB, 16}, {"aes-192-ecb", SC_ECB, 24}, {"aes-256-ecb", SC_ECB, 32}, + {"aes-128-cfb", SC_CFB, 16}, {"aes-192-cfb", SC_CFB, 24}, {"aes-256-cfb", SC_CFB, 32}, + {"aes-128-cfb8", SC_CFB8, 16}, {"aes-192-cfb8", SC_CFB8, 24}, {"aes-256-cfb8", SC_CFB8, 32}, + {"aes-128-cfb1", SC_CFB1, 16}, {"aes-192-cfb1", SC_CFB1, 24}, {"aes-256-cfb1", SC_CFB1, 32}, + {"aes-128-ofb", SC_OFB, 16}, {"aes-192-ofb", SC_OFB, 24}, {"aes-256-ofb", SC_OFB, 32}, + {"des-ede3-cbc", SC_CBC, 24, true}, {"des3", SC_CBC, 24, true}, + {"des-ede3", SC_ECB, 24, true}, {"des-ede3-ecb", SC_ECB, 24, true}, + {"des-ede3-cfb", SC_CFB, 24, true}, {"des-ede3-cfb8", SC_CFB8, 24, true}, + {"des-ede3-cfb1", SC_CFB1, 24, true}, {"des-ede3-ofb", SC_OFB, 24, true}, + {"des-ede-cbc", SC_CBC, 16, true}, {"des-ede", SC_ECB, 16, true}, + {"des-ede-ecb", SC_ECB, 16, true}, {"des-ede-cfb", SC_CFB, 16, true}, + {"des-ede-ofb", SC_OFB, 16, true}, + {"chacha20", SC_CHACHA20, 32}, +}; + +/* Ciphers OpenSSL 3 keeps in its legacy provider, which Node 24 does not + * load: crypto.getCipherInfo() knows them, and createCipheriv checks their + * IV length, then fails with OpenSSL's "unsupported" error. */ +static const struct { + const char *name; + unsigned iv_len; +} sc_legacy[] = { + {"des", 8}, {"des-cbc", 8}, {"des-ecb", 0}, {"des-cfb", 8}, {"des-cfb1", 8}, {"des-cfb8", 8}, + {"des-ofb", 8}, {"desx", 8}, {"desx-cbc", 8}, {"bf", 8}, {"bf-cbc", 8}, {"bf-ecb", 0}, + {"bf-cfb", 8}, {"bf-ofb", 8}, {"blowfish", 8}, {"cast", 8}, {"cast-cbc", 8}, {"cast5-cbc", 8}, + {"cast5-ecb", 0}, {"cast5-cfb", 8}, {"cast5-ofb", 8}, {"rc2", 8}, {"rc2-cbc", 8}, {"rc2-ecb", 0}, + {"rc2-cfb", 8}, {"rc2-ofb", 8}, {"rc2-40-cbc", 8}, {"rc2-64-cbc", 8}, {"rc4", 0}, {"rc4-40", 0}, + {"rc4-hmac-md5", 0}, {"seed", 16}, {"seed-cbc", 16}, {"seed-ecb", 0}, {"seed-cfb", 16}, + {"seed-ofb", 16}, {"idea", 8}, {"idea-cbc", 8}, {"idea-ecb", 0}, {"idea-cfb", 8}, {"idea-ofb", 8}, }; /* crypto.getCiphers() on Node 24.22 (OpenSSL 3.5). Names in this list that @@ -88,15 +131,21 @@ typedef struct ScCipher { ScCipherMode mode; bool decrypt; bool active; /* false after final() or a failed backend call */ - bool pad; /* CBC automatic padding */ + bool pad; /* CBC and ECB automatic padding */ + bool des; /* triple DES rather than AES */ + unsigned bs; /* the block size: 16 for AES, 8 for triple DES */ bool data_started; /* a non-empty update() ran, so non-empty AAD is refused */ int tag_len; /* SC_NO_TAG until known */ int tag_state; unsigned char tag[16]; - unsigned char block[16]; /* CBC: chaining value; CTR: counter block */ - unsigned char buf[16]; /* CBC: buffered input; CTR: keystream block */ - size_t bufsz; /* CBC: buffered bytes; CTR: keystream offset */ + unsigned char block[16]; /* CBC: chaining value; CTR: counter block; CFB, OFB: + shift register; chacha20: the IV */ + unsigned char buf[16]; /* CBC, ECB: buffered input; CTR: keystream block */ + size_t bufsz; /* CBC, ECB: buffered bytes; CTR, CFB, OFB: keystream + offset */ + uint64_t chacha_left; /* chacha20: bytes before the low counter word wraps */ mbedtls_aes_context aes; + mbedtls_des3_context des3; mbedtls_gcm_context gcm; mbedtls_chacha20_context chacha; mbedtls_poly1305_context poly; @@ -127,6 +176,7 @@ static void sc_trace(void *h, ScrTraceVisit visit, void *ctx) { static void sc_gcfree(void *h) { ScCipher *c = h; mbedtls_aes_free(&c->aes); + mbedtls_des3_free(&c->des3); mbedtls_gcm_free(&c->gcm); mbedtls_chacha20_free(&c->chacha); mbedtls_poly1305_free(&c->poly); @@ -184,18 +234,21 @@ static void sc_invalid_tag(unsigned long length) { /* An OpenSSL provider error as Node reports it: the formatted message plus * own opensslErrorStack, library, reason and code properties, in Node's * key order. */ -static void sc_throw_openssl(const char *packed, const char *reason, const char *code) { +/* OpenSSL's error with its library, reason and error stack; Node's + * code. */ +static void sc_throw_openssl_in(const char *packed, const char *library, const char *reason, + const char *code) { char message[128]; - snprintf(message, sizeof message, "error:%s:Provider routines::%s", packed, reason); + snprintf(message, sizeof message, "error:%s:%s::%s", packed, library, reason); ScrStr *text = scr_str_new(message, strlen(message)); ScrError *e = scr_error_new(SCR_ERR_ERROR, text); ScrDyn *stack = scr_dyn_new_arr(); scr_dyn_arr_push(stack, scr_dyn_new_str(text)); scr_str_release(text); scr_error_set_prop(e, "opensslErrorStack", 17, stack); - ScrStr *library = scr_str_new("Provider routines", 17); - scr_error_set_prop(e, "library", 7, scr_dyn_new_str(library)); - scr_str_release(library); + ScrStr *library_text = scr_str_new(library, strlen(library)); + scr_error_set_prop(e, "library", 7, scr_dyn_new_str(library_text)); + scr_str_release(library_text); ScrStr *reason_text = scr_str_new(reason, strlen(reason)); scr_error_set_prop(e, "reason", 6, scr_dyn_new_str(reason_text)); scr_str_release(reason_text); @@ -206,6 +259,10 @@ static void sc_throw_openssl(const char *packed, const char *reason, const char scr_throw_obj(e, &scr_error_retain_v, &scr_error_release_v, scr_error_trace_arg()); } +static void sc_throw_openssl(const char *packed, const char *reason, const char *code) { + sc_throw_openssl_in(packed, "Provider routines", reason, code); +} + /* ERR_INVALID_ARG_VALUE's property flavor: "The property 'options.x' is * invalid. Received ". */ static void sc_invalid_property(const char *name, const ScrDyn *value) { @@ -440,6 +497,27 @@ static const ScCipherSpec *sc_lookup(const ScrStr *name) { return NULL; } +/* The legacy-provider IV length of `name`, or -1 for other names. */ +static int sc_legacy_iv(const ScrStr *name) { + for (size_t i = 0; i < sizeof sc_legacy / sizeof sc_legacy[0]; i++) + if (sc_ascii_eq_ci(name->data, name->len, sc_legacy[i].name)) return (int)sc_legacy[i].iv_len; + return -1; +} + +/* The IV length a mode takes; SC_GCM takes any nonzero length. */ +static size_t sc_iv_length(ScCipherMode mode, bool des) { + switch (mode) { + case SC_ECB: + return 0; + case SC_CHACHA: + return 12; + case SC_CHACHA20: + return 16; + default: + return des ? 8 : 16; + } +} + static bool sc_known_name(const ScrStr *name) { for (size_t i = 0; i < sizeof sc_known / sizeof sc_known[0]; i++) if (sc_ascii_eq_ci(name->data, name->len, sc_known[i])) return true; @@ -451,9 +529,10 @@ static void sc_unsupported_cipher(const ScrStr *name) { scr_jb_init(&b); scr_jb_puts(&b, "The cipher '"); scr_jb_put_str(&b, name); - scr_jb_puts(&b, "' has no native implementation; the supported ciphers are aes-128-cbc, aes-192-cbc, " - "aes-256-cbc, aes-128-ctr, aes-192-ctr, aes-256-ctr, aes-128-gcm, aes-192-gcm, " - "aes-256-gcm and chacha20-poly1305"); + scr_jb_puts(&b, "' has no native implementation; the supported ciphers are aes-128, aes-192 and " + "aes-256 in cbc, ctr, gcm, ecb, cfb, cfb8, cfb1 and ofb modes, des-ede3 in cbc, ecb, " + "cfb, cfb8, cfb1 and ofb modes, des-ede in cbc, ecb, cfb and ofb modes, chacha20 and " + "chacha20-poly1305"); ScrStr *message = scr_jb_finish(&b); scr_throw_error_msg_code(SCR_ERR_ERROR, message->data, message->len, "SC2020"); scr_str_release(message); @@ -522,14 +601,24 @@ static ScrDyn *sc_cipher_new(bool decrypt, ScrDyn *cipher, ScrDyn *key, ScrDyn * goto done; } const ScCipherSpec *spec = sc_lookup(cipher->v.str); + size_t iv_len = iv_bytes ? iv_bytes->len : 0; if (!spec) { - if (sc_known_name(cipher->v.str)) sc_unsupported_cipher(cipher->v.str); - else sc_throw(SCR_ERR_ERROR, "ERR_CRYPTO_UNKNOWN_CIPHER", "Unknown cipher"); + int legacy_iv = sc_legacy_iv(cipher->v.str); + if (legacy_iv >= 0) { + if (iv_len != (size_t)legacy_iv || (legacy_iv > 0 && !iv_bytes)) + sc_throw(SCR_ERR_TYPE, "ERR_CRYPTO_INVALID_IV", "Invalid initialization vector"); + else + sc_throw_openssl_in("0308010C", "digital envelope routines", "unsupported", + "ERR_OSSL_EVP_UNSUPPORTED"); + } else if (sc_known_name(cipher->v.str)) { + sc_unsupported_cipher(cipher->v.str); + } else { + sc_throw(SCR_ERR_ERROR, "ERR_CRYPTO_UNKNOWN_CIPHER", "Unknown cipher"); + } goto done; } - size_t iv_len = iv_bytes ? iv_bytes->len : 0; - bool aead = spec->mode == SC_GCM || spec->mode == SC_CHACHA; - if (iv_len == 0 || (!aead && iv_len != 16) || (spec->mode == SC_CHACHA && iv_len != 12)) { + size_t expected_iv = sc_iv_length(spec->mode, spec->des); + if (spec->mode == SC_GCM ? iv_len == 0 : iv_len != expected_iv || (expected_iv > 0 && !iv_bytes)) { sc_throw(SCR_ERR_TYPE, "ERR_CRYPTO_INVALID_IV", "Invalid initialization vector"); goto done; } @@ -570,8 +659,11 @@ static ScrDyn *sc_cipher_new(bool decrypt, ScrDyn *cipher, ScrDyn *key, ScrDyn * c->decrypt = decrypt; c->active = true; c->pad = true; + c->des = spec->des; + c->bs = spec->des ? 8 : 16; c->tag_len = tag_len; mbedtls_aes_init(&c->aes); + mbedtls_des3_init(&c->des3); mbedtls_gcm_init(&c->gcm); mbedtls_chacha20_init(&c->chacha); mbedtls_poly1305_init(&c->poly); @@ -579,9 +671,39 @@ static ScrDyn *sc_cipher_new(bool decrypt, ScrDyn *cipher, ScrDyn *key, ScrDyn * unsigned bits = spec->key_len * 8; switch (spec->mode) { case SC_CBC: - rc = decrypt ? mbedtls_aes_setkey_dec(&c->aes, key_bytes->data, bits) - : mbedtls_aes_setkey_enc(&c->aes, key_bytes->data, bits); + case SC_ECB: + case SC_CFB: + case SC_CFB8: + case SC_CFB1: + case SC_OFB: { + /* CBC and ECB decrypt with the inverse cipher; the feedback modes run + * the forward cipher both ways. */ + bool inverse = decrypt && (spec->mode == SC_CBC || spec->mode == SC_ECB); + if (spec->des && spec->key_len == 24) + rc = inverse ? mbedtls_des3_set3key_dec(&c->des3, key_bytes->data) + : mbedtls_des3_set3key_enc(&c->des3, key_bytes->data); + else if (spec->des) + rc = inverse ? mbedtls_des3_set2key_dec(&c->des3, key_bytes->data) + : mbedtls_des3_set2key_enc(&c->des3, key_bytes->data); + else + rc = inverse ? mbedtls_aes_setkey_dec(&c->aes, key_bytes->data, bits) + : mbedtls_aes_setkey_enc(&c->aes, key_bytes->data, bits); + if (expected_iv) memcpy(c->block, iv_bytes->data, expected_iv); + break; + } + case SC_CHACHA20: + /* OpenSSL's IV is a 64-bit little-endian block counter and an 8-byte + * nonce; mbedTLS takes a 32-bit counter and a 12-byte nonce, so the + * stream restarts each time the low counter word wraps. */ memcpy(c->block, iv_bytes->data, 16); + rc = mbedtls_chacha20_setkey(&c->chacha, key_bytes->data); + if (rc == 0) { + uint32_t low = (uint32_t)c->block[0] | (uint32_t)c->block[1] << 8 | + (uint32_t)c->block[2] << 16 | (uint32_t)c->block[3] << 24; + c->chacha_left = ((uint64_t)1 << 32) - low; + c->chacha_left *= 64; + rc = mbedtls_chacha20_starts(&c->chacha, c->block + 4, low); + } break; case SC_CTR: rc = mbedtls_aes_setkey_enc(&c->aes, key_bytes->data, bits); @@ -635,39 +757,123 @@ done: /* ── the cipher transforms ────────────────────────────────────────────── */ -/* Raw CBC over whole blocks. */ +/* Raw CBC or ECB over whole blocks. */ static bool sc_cbc_blocks(ScCipher *c, const unsigned char *in, size_t len, unsigned char *out) { + if (c->mode == SC_ECB) { + for (size_t off = 0; off < len; off += c->bs) { + int rc = c->des ? mbedtls_des3_crypt_ecb(&c->des3, in + off, out + off) + : mbedtls_aes_crypt_ecb(&c->aes, c->decrypt ? MBEDTLS_AES_DECRYPT : MBEDTLS_AES_ENCRYPT, + in + off, out + off); + if (rc != 0) return false; + } + return true; + } + if (c->des) + return mbedtls_des3_crypt_cbc(&c->des3, c->decrypt ? MBEDTLS_DES_DECRYPT : MBEDTLS_DES_ENCRYPT, len, c->block, + in, out) == 0; return mbedtls_aes_crypt_cbc(&c->aes, c->decrypt ? MBEDTLS_AES_DECRYPT : MBEDTLS_AES_ENCRYPT, len, c->block, in, out) == 0; } -/* OpenSSL's ossl_cipher_generic_block_update for CBC: complete blocks - * are processed, the remainder is buffered, and a decryption that ends on - * a block boundary holds the last block back for padding removal. Returns - * the output length, or -1 when the backend fails; out has room for - * len + 16 bytes. */ +/* One forward block of the feedback modes' cipher. */ +static bool sc_forward(ScCipher *c, const unsigned char *in, unsigned char *out) { + return (c->des ? mbedtls_des3_crypt_ecb(&c->des3, in, out) + : mbedtls_aes_crypt_ecb(&c->aes, MBEDTLS_AES_ENCRYPT, in, out)) == 0; +} + +/* CFB (full-block feedback), CFB8, CFB1 and OFB, as OpenSSL's + * CRYPTO_cfb128_encrypt, CRYPTO_cfb128_8_encrypt, CRYPTO_cfb128_1_encrypt + * and CRYPTO_ofb128_encrypt: CFB and OFB keep their keystream offset across + * updates; CFB1 feeds back one bit at a time, most significant first. */ +static bool sc_feedback(ScCipher *c, const unsigned char *in, size_t len, unsigned char *out) { + unsigned char stream[16]; + size_t bs = c->bs; + for (size_t i = 0; i < len; i++) { + switch (c->mode) { + case SC_CFB: + case SC_OFB: + if (c->bufsz == 0 && !sc_forward(c, c->block, c->block)) return false; + out[i] = (unsigned char)(in[i] ^ c->block[c->bufsz]); + if (c->mode == SC_CFB) c->block[c->bufsz] = c->decrypt ? in[i] : out[i]; + c->bufsz = (c->bufsz + 1) % bs; + break; + case SC_CFB8: { + if (!sc_forward(c, c->block, stream)) return false; + unsigned char byte = (unsigned char)(in[i] ^ stream[0]); + memmove(c->block, c->block + 1, bs - 1); + c->block[bs - 1] = c->decrypt ? in[i] : byte; + out[i] = byte; + break; + } + default: { + unsigned char result = 0; + for (int bit = 7; bit >= 0; bit--) { + if (!sc_forward(c, c->block, stream)) return false; + unsigned in_bit = (in[i] >> bit) & 1u; + unsigned out_bit = in_bit ^ (stream[0] >> 7); + unsigned feed = c->decrypt ? in_bit : out_bit; + for (size_t k = 0; k + 1 < bs; k++) + c->block[k] = (unsigned char)(c->block[k] << 1 | c->block[k + 1] >> 7); + c->block[bs - 1] = (unsigned char)(c->block[bs - 1] << 1 | feed); + result |= (unsigned char)(out_bit << bit); + } + out[i] = result; + break; + } + } + } + mbedtls_platform_zeroize(stream, sizeof stream); + return true; +} + +/* chacha20 over len bytes: when the 64-bit counter's low word wraps, the + * high word steps (and wraps itself, as OpenSSL's does) and the stream + * restarts from block 0 of the next word. */ +static bool sc_chacha20_update(ScCipher *c, const unsigned char *in, size_t len, unsigned char *out) { + while (len > 0) { + size_t n = len < c->chacha_left ? len : (size_t)c->chacha_left; + if (mbedtls_chacha20_update(&c->chacha, n, in, out) != 0) return false; + in += n; + out += n; + len -= n; + c->chacha_left -= n; + if (c->chacha_left == 0) { + for (int k = 4; k < 8 && ++c->block[k] == 0; k++) { + } + memset(c->block, 0, 4); + c->chacha_left = (uint64_t)1 << 38; + if (mbedtls_chacha20_starts(&c->chacha, c->block + 4, 0) != 0) return false; + } + } + return true; +} + +/* OpenSSL's ossl_cipher_generic_block_update for CBC and ECB: complete + * blocks are processed, the remainder is buffered, and a decryption that + * ends on a block boundary holds the last block back for padding removal. + * Returns the output length, or -1 when the backend fails; out has room + * for len + 16 bytes. */ static long sc_cbc_update(ScCipher *c, const unsigned char *in, size_t inl, unsigned char *out) { + size_t bs = c->bs; size_t outl = 0; size_t nextblocks; if (c->bufsz != 0) { - size_t remain = 16 - c->bufsz; + size_t remain = bs - c->bufsz; if (inl < remain) remain = inl; memcpy(c->buf + c->bufsz, in, remain); in += remain; inl -= remain; c->bufsz += remain; - nextblocks = inl & ~(size_t)15; - } else { - nextblocks = inl & ~(size_t)15; } - if (c->bufsz == 16 && (!c->decrypt || inl > 0 || !c->pad)) { - if (!sc_cbc_blocks(c, c->buf, 16, out)) return -1; + nextblocks = inl - inl % bs; + if (c->bufsz == bs && (!c->decrypt || inl > 0 || !c->pad)) { + if (!sc_cbc_blocks(c, c->buf, bs, out)) return -1; c->bufsz = 0; - outl = 16; - out += 16; + outl = bs; + out += bs; } if (nextblocks > 0) { - if (c->decrypt && c->pad && nextblocks == inl) nextblocks -= 16; + if (c->decrypt && c->pad && nextblocks == inl) nextblocks -= bs; if (nextblocks > 0) { if (!sc_cbc_blocks(c, in, nextblocks, out)) return -1; outl += nextblocks; @@ -682,33 +888,35 @@ static long sc_cbc_update(ScCipher *c, const unsigned char *in, size_t inl, unsi return (long)outl; } -/* ossl_cipher_generic_block_final for CBC. Returns the output length, -1 - * after throwing OpenSSL's error, or -2 when the backend fails. */ +/* ossl_cipher_generic_block_final for CBC and ECB. Returns the output + * length, -1 after throwing OpenSSL's error, or -2 when the backend + * fails. */ static long sc_cbc_final(ScCipher *c, unsigned char out[16]) { + size_t bs = c->bs; if (!c->decrypt) { if (c->pad) { - unsigned char fill = (unsigned char)(16 - c->bufsz); - for (size_t i = c->bufsz; i < 16; i++) c->buf[i] = fill; + unsigned char fill = (unsigned char)(bs - c->bufsz); + for (size_t i = c->bufsz; i < bs; i++) c->buf[i] = fill; } else if (c->bufsz == 0) { return 0; - } else if (c->bufsz != 16) { + } else if (c->bufsz != bs) { sc_throw_openssl("1C80006B", "wrong final block length", "ERR_OSSL_WRONG_FINAL_BLOCK_LENGTH"); return -1; } - if (!sc_cbc_blocks(c, c->buf, 16, out)) return -2; + if (!sc_cbc_blocks(c, c->buf, bs, out)) return -2; c->bufsz = 0; - return 16; + return (long)bs; } - if (c->bufsz != 16) { + if (c->bufsz != bs) { if (c->bufsz == 0 && !c->pad) return 0; sc_throw_openssl("1C80006B", "wrong final block length", "ERR_OSSL_WRONG_FINAL_BLOCK_LENGTH"); return -1; } - if (!sc_cbc_blocks(c, c->buf, 16, c->buf)) return -2; - size_t len = 16; + if (!sc_cbc_blocks(c, c->buf, bs, c->buf)) return -2; + size_t len = bs; if (c->pad) { - unsigned char fill = c->buf[15]; - bool bad = fill == 0 || fill > 16; + unsigned char fill = c->buf[bs - 1]; + bool bad = fill == 0 || fill > bs; for (size_t i = 0; !bad && i < fill; i++) bad = c->buf[--len] != fill; if (bad) { sc_throw_openssl("1C800064", "bad decrypt", "ERR_OSSL_BAD_DECRYPT"); @@ -769,7 +977,8 @@ static ScrBytes *sc_transform(ScCipher *c, const unsigned char *in, size_t len) size_t produced = 0; bool ok = true; switch (c->mode) { - case SC_CBC: { + case SC_CBC: + case SC_ECB: { long n = sc_cbc_update(c, in, len, out->data); ok = n >= 0; produced = ok ? (size_t)n : 0; @@ -786,6 +995,17 @@ static ScrBytes *sc_transform(ScCipher *c, const unsigned char *in, size_t len) ok = len == 0 || sc_chacha_update(c, in, len, out->data); produced = len; break; + case SC_CFB: + case SC_CFB8: + case SC_CFB1: + case SC_OFB: + ok = sc_feedback(c, in, len, out->data); + produced = len; + break; + case SC_CHACHA20: + ok = len == 0 || sc_chacha20_update(c, in, len, out->data); + produced = len; + break; } if (!ok) { mbedtls_platform_zeroize(out->data, room); @@ -926,7 +1146,8 @@ static ScrBytes *sc_final_bytes(ScCipher *c) { out->len = 0; bool ok = true, backend = true; switch (c->mode) { - case SC_CBC: { + case SC_CBC: + case SC_ECB: { long produced = sc_cbc_final(c, out->data); if (produced == -1) { scr_bytes_release(out); @@ -937,6 +1158,11 @@ static ScrBytes *sc_final_bytes(ScCipher *c) { break; } case SC_CTR: + case SC_CFB: + case SC_CFB8: + case SC_CFB1: + case SC_OFB: + case SC_CHACHA20: break; case SC_GCM: { unsigned char full[16]; diff --git a/packages/runtime/src/scr_crypto.h b/packages/runtime/src/scr_crypto.h index 546e3d75b..0e6c6e07f 100644 --- a/packages/runtime/src/scr_crypto.h +++ b/packages/runtime/src/scr_crypto.h @@ -108,6 +108,14 @@ ScrDyn *scr_crypto_ecdh_convert_key(ScrDyn *key, ScrDyn *curve, ScrDyn *in_enc, * Node's error pending. */ ScrDyn *scr_crypto_dh_new(ScrDyn *size_or_key, ScrDyn *key_encoding, ScrDyn *generator, ScrDyn *generator_encoding); ScrDyn *scr_crypto_dh_group_new(ScrDyn *name); +/* crypto.checkPrimeSync, checkPrime, generatePrimeSync and generatePrime + * (scr_dh.c): checked-dynamic arguments, borrowed; the sync forms answer + * +1 or NULL with the error pending. The callbacks receive undefined as + * the error, as Node passes it, or null when `null_error`. */ +ScrDyn *scr_crypto_check_prime_sync(ScrDyn *candidate, ScrDyn *options); +void scr_crypto_check_prime_async(ScrDyn *candidate, ScrDyn *options, ScrDyn *callback, bool null_error); +ScrDyn *scr_crypto_generate_prime_sync(ScrDyn *size, ScrDyn *options); +void scr_crypto_generate_prime_async(ScrDyn *size, ScrDyn *options, ScrDyn *callback, bool null_error); ScrBytes *scr_crypto_pbkdf2(ScrBytes *password, ScrBytes *salt, double iterations, double keylen, ScrStr *digest); ScrBytes *scr_crypto_pbkdf2_job(ScrBytes *password, ScrBytes *salt, double iterations, diff --git a/packages/runtime/src/scr_dh.c b/packages/runtime/src/scr_dh.c index cf44338f5..dfa6fd2f3 100644 --- a/packages/runtime/src/scr_dh.c +++ b/packages/runtime/src/scr_dh.c @@ -183,17 +183,17 @@ static void sd_value_too_long(void) { scr_throw_error_msg_code(SCR_ERR_ERROR, message, n < 0 ? 0 : (size_t)n, "SC2020"); } -/* validateInt32(value, name). */ -static bool sd_validate_int32(const char *name, double value) { +/* validateInt32(value, name, min). */ +static bool sd_validate_int32_min(const char *name, double value, int32_t min) { bool integer = isfinite(value) && trunc(value) == value; - if (integer && value >= INT32_MIN && value <= INT32_MAX) return true; + if (integer && value >= min && value <= INT32_MAX) return true; char received[48]; scr_num_received(value, received); char text[192]; int n = integer ? snprintf(text, sizeof text, - "The value of \"%s\" is out of range. It must be >= -2147483648 && <= 2147483647. " + "The value of \"%s\" is out of range. It must be >= %ld && <= 2147483647. " "Received %s", - name, received) + name, (long)min, received) : snprintf(text, sizeof text, "The value of \"%s\" is out of range. It must be an integer. Received %s", name, received); @@ -201,6 +201,11 @@ static bool sd_validate_int32(const char *name, double value) { return false; } +/* validateInt32(value, name). */ +static bool sd_validate_int32(const char *name, double value) { + return sd_validate_int32_min(name, value, INT32_MIN); +} + /* ── arithmetic ───────────────────────────────────────────────────────── */ /* The exponent digit width that keeps every mbedtls_mpi_exp_mod exponent @@ -630,9 +635,9 @@ static int sd_mr_rounds(size_t bits) { return bits > 2048 ? 128 : 64; } -/* BN_check_prime: trial division by the small primes, then Miller-Rabin - * rounds with random bases. */ -static bool sd_is_prime(ScDhSmallPrimes *t, const mbedtls_mpi *x) { +/* BN_is_prime_ex(x, checks): trial division by the small primes, then + * Miller-Rabin rounds with random bases, at least OpenSSL's round count. */ +static bool sd_is_prime_checks(ScDhSmallPrimes *t, const mbedtls_mpi *x, int checks) { if (mbedtls_mpi_cmp_int(x, 2) < 0) return false; if (mbedtls_mpi_get_bit(x, 0) == 0) return mbedtls_mpi_cmp_int(x, 2) == 0; if (!sd_small_residues(t, x)) return false; @@ -643,11 +648,17 @@ static bool sd_is_prime(ScDhSmallPrimes *t, const mbedtls_mpi *x) { ScDhMillerRabin m; bool prime = sd_mr_init(&m, x); int rounds = sd_mr_rounds(mbedtls_mpi_bitlen(x)); + if (checks > rounds) rounds = checks; for (int i = 0; i < rounds && prime; i++) prime = sd_mr_round(&m, x, false) == 1; sd_mr_free(&m); return prime; } +/* BN_check_prime. */ +static bool sd_is_prime(ScDhSmallPrimes *t, const mbedtls_mpi *x) { + return sd_is_prime_checks(t, x, 0); +} + /* DH_check for parameters without q, as ncrypto's CheckDhParams reports * it. */ static int sd_check_params(const mbedtls_mpi *p, const mbedtls_mpi *g) { @@ -793,8 +804,9 @@ static void sd_sieve_window(ScDhSieve *sv, unsigned add) { * when p is a strong probable prime to the base 2 and q = (p - 1) / 2 * passes OpenSSL's Miller-Rabin round count; a prime q then proves p prime * (Pocklington's criterion with 2^(p-1) ≡ 1 and gcd(2^2 - 1, p) = 1). - * Candidates past `bits` bits take a new start. */ -static bool sd_generate_safe_prime(mbedtls_mpi *p, size_t bits, unsigned add, unsigned rem) { + * Candidates past `bits` bits take a new start. With `top_two`, starts + * also set the second-highest bit, as BN_RAND_TOP_TWO does. */ +static bool sd_generate_safe_prime(mbedtls_mpi *p, size_t bits, unsigned add, unsigned rem, bool top_two) { ScDhSieve *sv = sd_sieve_new(bits, add); int rounds = sd_mr_rounds(bits); mbedtls_mpi start, q; @@ -803,7 +815,8 @@ static bool sd_generate_safe_prime(mbedtls_mpi *p, size_t bits, unsigned add, un int ret = 0; while (ret == 0) { mbedtls_mpi_uint r; - if (!sd_random_bits(&start, bits, true) || mbedtls_mpi_mod_int(&r, &start, add) != 0 || + if (!sd_random_bits(&start, bits, true) || (top_two && mbedtls_mpi_set_bit(&start, bits - 2, 1) != 0) || + mbedtls_mpi_mod_int(&r, &start, add) != 0 || mbedtls_mpi_sub_int(&start, &start, (mbedtls_mpi_sint)r) != 0 || mbedtls_mpi_add_int(&start, &start, rem) != 0 || (mbedtls_mpi_bitlen(&start) < bits && mbedtls_mpi_add_int(&start, &start, add) != 0)) { @@ -900,7 +913,7 @@ static ScrDyn *sd_generate(double bits, const ScrBytes *generator_bytes, double if (generator == 2) add = 24; else if (generator == 5) add = 60; ScDh *d = sd_alloc(); - if (!sd_generate_safe_prime(&d->p, (size_t)bits, add, add - 1) || mbedtls_mpi_lset(&d->g, (mbedtls_mpi_sint)generator) != 0) { + if (!sd_generate_safe_prime(&d->p, (size_t)bits, add, add - 1, false) || mbedtls_mpi_lset(&d->g, (mbedtls_mpi_sint)generator) != 0) { sd_release(d); sd_throw(SCR_ERR_TYPE, "ERR_INVALID_ARG_VALUE", "Invalid DH parameters"); return NULL; @@ -1034,6 +1047,505 @@ ScrDyn *scr_crypto_dh_group_new(ScrDyn *name) { return sd_box(d); } +/* ── checkPrime and generatePrime ───────────────────────────────────────── + * crypto.checkPrime(Sync) and generatePrime(Sync) over the primality test + * and prime search above, with random.js's argument checks and ncrypto's + * RandomPrimeConfig and BN_generate_prime_ex2 errors. The callback forms + * compute at once and call back in a later macrotask. */ + +/* The longest candidate the Miller-Rabin rounds take: sd_mont_init's R^2 + * has twice the candidate's limbs, within MBEDTLS_MPI_MAX_LIMBS, and + * mbedtls_mpi_exp_mod stops at MBEDTLS_MPI_MAX_BITS. */ +#if SD_MONT +#define SD_MAX_PRIME_BITS 262144 +#else +#define SD_MAX_PRIME_BITS MBEDTLS_MPI_MAX_BITS +#endif + +static void sd_prime_too_long(const char *what) { + char message[128]; + int n = snprintf(message, sizeof message, "%s longer than %d bits have no native implementation", what, + SD_MAX_PRIME_BITS); + scr_throw_error_msg_code(SCR_ERR_ERROR, message, n < 0 ? 0 : (size_t)n, "SC2020"); +} + +/* An OpenSSL bignum error as a crypto job reports it: an Error with the + * formatted message and no other properties. */ +static void sd_throw_bignum(const char *packed, const char *reason) { + char message[96]; + int n = snprintf(message, sizeof message, "error:%s:bignum routines::%s", packed, reason); + ScrStr *text = scr_str_new(message, n < 0 ? 0 : (size_t)n); + ScrError *e = scr_error_new(SCR_ERR_ERROR, text); + scr_str_release(text); + scr_throw_obj(e, &scr_error_retain_v, &scr_error_release_v, scr_error_trace_arg()); +} + +/* validateObject(value, name): an object that is neither null nor an + * array. */ +static bool sd_validate_object(const char *name, const ScrDyn *value) { + ScrStr *type = scr_dyn_typeof(value); + bool object = type->len == 6 && memcmp(type->data, "object", 6) == 0; + scr_str_release(type); + if (object && value->kind != SCR_DYN_NULL && value->kind != SCR_DYN_ARR) return true; + scr_dyn_arg_type_fail(name, "of type object", value); + return false; +} + +static bool sd_validate_function(const char *name, const ScrDyn *value) { + if (scr_dyn_is_callable(value)) return true; + scr_dyn_arg_type_fail(name, "of type function", value); + return false; +} + +/* options[key] (borrowed), or undefined. */ +static const ScrDyn *sd_option(const ScrDyn *options, const char *key) { + const ScrDyn *value = options->kind == SCR_DYN_OBJ ? scr_dyn_obj_get(options, key, strlen(key)) : NULL; + return value ? value : scr_dyn_undefined(); +} + +static const char sd_large_number_types[] = + "of type bigint or an instance of ArrayBuffer, TypedArray, Buffer, or DataView"; + +/* A LargeNumberLike argument as a bignum: a bigint (unsignedBigIntToBuffer, + * which rejects negative values), or the bytes of an ArrayBuffer or view. + * False with the error pending. */ +static bool sd_large_number(mbedtls_mpi *x, const char *name, bool property, const ScrDyn *value) { + if (value->kind == SCR_DYN_BIGINT) { + ScrStr *text = scr_bigint_to_string(value->v.bigint, 16); + if (!text) return false; + bool negative = text->len > 0 && text->data[0] == '-'; + if (negative) { + scr_str_release(text); + ScrStr *decimal = scr_bigint_to_string(value->v.bigint, 10); + if (!decimal) return false; + /* ERR_OUT_OF_RANGE's received value groups the digits of a bigint + * past 2^32 in threes. */ + ScrBigInt *limit = scr_bigint_from_i64(-((int64_t)1 << 32)); + bool group = scr_bigint_cmp_f64(value->v.bigint, limit) < 0; + scr_bigint_release(limit); + char digits[160]; + size_t len = decimal->len < 120 ? decimal->len : 120, out = 0; + for (size_t i = 0; i < len; i++) { + size_t rest = len - i; + if (group && i > 1 && rest % 3 == 0) digits[out++] = '_'; + digits[out++] = decimal->data[i]; + } + digits[out] = '\0'; + scr_str_release(decimal); + char message[256]; + int n = snprintf(message, sizeof message, "The value of \"%s\" is out of range. It must be >= 0. Received %sn", + name, digits); + scr_throw_error_msg_code(SCR_ERR_RANGE, message, n < 0 ? 0 : (size_t)n, "ERR_OUT_OF_RANGE"); + return false; + } + char *hex = malloc(text->len + 1); + if (!hex) scr_trap("scriptc: out of memory\n"); + memcpy(hex, text->data, text->len); + hex[text->len] = '\0'; + scr_str_release(text); + bool ok = strlen(hex) <= SD_MAX_VALUE_BYTES * 2; + if (ok && mbedtls_mpi_read_string(x, 16, hex) != 0) scr_trap("scriptc: out of memory\n"); + free(hex); + if (!ok) sd_value_too_long(); + return ok; + } + if (!sd_is_view(value)) { + if (property) scr_dyn_prop_type_fail(name, sd_large_number_types, value); + else scr_dyn_arg_type_fail(name, sd_large_number_types, value); + return false; + } + ScrBytes *bytes = scr_bytes_buffer_source(value); + if (!bytes) return false; + bool held = sd_read(x, bytes); + scr_bytes_release(bytes); + if (!held) sd_value_too_long(); + return held; +} + +/* calc_trial_divisions: how many of the first primes (2 included) the + * search divides a candidate of `bits` bits by. */ +static size_t sd_trial_divisions(size_t bits) { + return bits <= 512 ? 64 : bits <= 1024 ? 128 : bits <= 2048 ? 384 : bits <= 4096 ? 1024 : 2048; +} + +static uint64_t sd_word(const mbedtls_mpi *x) { + if (mbedtls_mpi_bitlen(x) > 64) return UINT64_MAX; + unsigned char buf[8]; + if (mbedtls_mpi_write_binary(x, buf, 8) != 0) return UINT64_MAX; + uint64_t w = 0; + for (int i = 0; i < 8; i++) w = w << 8 | buf[i]; + return w; +} + +static bool sd_add_word(mbedtls_mpi *x, uint64_t w) { + unsigned char buf[8]; + for (int i = 7; i >= 0; i--, w >>= 8) buf[i] = (unsigned char)w; + mbedtls_mpi t; + mbedtls_mpi_init(&t); + bool ok = mbedtls_mpi_read_binary(&t, buf, 8) == 0 && mbedtls_mpi_add_mpi(x, x, &t) == 0; + mbedtls_mpi_free(&t); + return ok; +} + +/* probable_prime and bn_probable_prime_dh: a random odd candidate of + * `bits` bits (the top two bits set, or with `add` the top bit, moved to + * rem modulo add), then the first one stepping by 2, by 4 for a safe + * prime, or by add, that no trial-division prime divides, nor for a safe + * prime divides (p - 1) / 2. */ +static bool sd_probable_prime(ScDhSmallPrimes *t, mbedtls_mpi *rnd, size_t bits, bool safe, const mbedtls_mpi *add, + const mbedtls_mpi *rem) { + size_t trials = sd_trial_divisions(bits) - 1; /* odd primes */ + uint64_t step = safe ? 4 : 2; + uint64_t maxdelta = UINT64_MAX - t->primes[trials - 1]; + if (add) { + step = sd_word(add); + if (maxdelta > UINT64_MAX - step) maxdelta = UINT64_MAX - step; + } + mbedtls_mpi r; + mbedtls_mpi_init(&r); + bool ok = true; + for (;;) { + if (!sd_random_bits(rnd, bits, true)) ok = false; + if (ok && !add) { + if ((bits >= 2 && mbedtls_mpi_set_bit(rnd, bits - 2, 1) != 0) || (safe && mbedtls_mpi_set_bit(rnd, 1, 1) != 0)) + ok = false; + } else if (ok) { + ok = mbedtls_mpi_mod_mpi(&r, rnd, add) == 0 && mbedtls_mpi_sub_mpi(rnd, rnd, &r) == 0 && + (rem ? mbedtls_mpi_add_mpi(rnd, rnd, rem) == 0 : mbedtls_mpi_add_int(rnd, rnd, safe ? 3 : 1) == 0); + if (ok && (mbedtls_mpi_bitlen(rnd) < bits || sd_word(rnd) < (safe ? 5u : 3u))) + ok = mbedtls_mpi_add_mpi(rnd, rnd, add) == 0; + } + if (!ok || !sd_small_residues(t, rnd)) { + ok = false; + break; + } + uint64_t delta = 0, word = sd_word(rnd); + bool again = false; + for (size_t i = 0; i < trials && !again;) { + uint64_t prime = t->primes[i]; + /* A one-word candidate needs only the primes up to its square root. */ + if (bits <= 31 && delta <= 0x7fffffff && prime * prime > word + delta) break; + uint64_t m = (t->mods[i] + delta) % prime; + if (safe ? m <= 1 : m == 0) { + delta += step; + if (delta > maxdelta) again = true; + i = 0; + continue; + } + i++; + } + if (again) continue; + if (!sd_add_word(rnd, delta)) { + ok = false; + break; + } + if (add || mbedtls_mpi_bitlen(rnd) == bits) break; + } + mbedtls_mpi_free(&r); + return ok; +} + +/* BN_generate_prime_ex2(p, bits, safe, add, rem): false with OpenSSL's + * error pending. Long safe primes without add take the sieve over + * p ≡ 11 (mod 12), which every safe prime above 7 satisfies. */ +static bool sd_generate_prime(mbedtls_mpi *p, size_t bits, bool safe, const mbedtls_mpi *add, const mbedtls_mpi *rem) { + if (bits < 2 || (!add && safe && bits < 6 && bits != 3)) { + sd_throw_bignum("01800076", "bits too small"); + return false; + } + if (add && mbedtls_mpi_cmp_int(add, 0) == 0) { + sd_throw_bignum("01800067", "div by zero"); + return false; + } + if (!add && safe && bits >= 64) { + if (!sd_generate_safe_prime(p, bits, 12, 11, true)) scr_trap("scriptc: out of memory\n"); + return true; + } + ScDhSmallPrimes *t = sd_small_primes(); + mbedtls_mpi q; + mbedtls_mpi_init(&q); + bool found = false; + while (!found) { + if (!sd_probable_prime(t, p, bits, safe, add, rem)) scr_trap("scriptc: out of memory\n"); + if (!safe) { + found = sd_is_prime(t, p); + } else { + if (mbedtls_mpi_copy(&q, p) != 0 || mbedtls_mpi_shift_r(&q, 1) != 0) scr_trap("scriptc: out of memory\n"); + found = sd_is_prime(t, p) && sd_is_prime(t, &q); + } + } + mbedtls_mpi_free(&q); + free(t); + return true; +} + +/* checkPrime's arguments after the candidate: Node's options and checks + * rules. -1 with the error pending. */ +static int sd_prime_checks(const ScrDyn *options) { + if (!sd_validate_object("options", options)) return -1; + const ScrDyn *checks = sd_option(options, "checks"); + if (checks->kind == SCR_DYN_UNDEF) return 0; + if (checks->kind != SCR_DYN_NUM) { + scr_dyn_prop_type_fail("options.checks", "of type number", checks); + return -1; + } + if (!sd_validate_int32_min("options.checks", checks->v.num, 0)) return -1; + return (int)checks->v.num; +} + +/* The candidate's verdict as a boolean dyn (+1), or NULL with the error + * pending. */ +static ScrDyn *sd_check_prime(const mbedtls_mpi *candidate, int checks) { + if (mbedtls_mpi_bitlen(candidate) > SD_MAX_PRIME_BITS) { + sd_prime_too_long("checkPrime candidates"); + return NULL; + } + ScDhSmallPrimes *t = sd_small_primes(); + bool prime = sd_is_prime_checks(t, candidate, checks); + free(t); + return scr_dyn_new_bool(prime); +} + +/* A borrowed, unboxed view of each argument (+1 each); false after a + * throw. */ +static bool sd_unbox_all(ScrDyn **out, ScrDyn *const *args, size_t count) { + for (size_t i = 0; i < count; i++) out[i] = NULL; + for (size_t i = 0; i < count; i++) { + ScrDyn *value = args[i]; + out[i] = value->kind == SCR_DYN_TYPED_REF ? scr_dyn_typed_ref_materialize(value) : scr_dyn_retain(value); + if (!out[i]) return false; + } + return true; +} + +static void sd_release_all(ScrDyn **values, size_t count) { + for (size_t i = 0; i < count; i++) + if (values[i]) scr_dyn_release(values[i]); +} + +/* checkPrimeSync(candidate[, options]). */ +ScrDyn *scr_crypto_check_prime_sync(ScrDyn *candidate, ScrDyn *options) { + ScrDyn *args[2] = {candidate, options}; + ScrDyn *v[2]; + ScrDyn *result = NULL; + mbedtls_mpi x; + mbedtls_mpi_init(&x); + if (sd_unbox_all(v, args, 2) && sd_large_number(&x, "candidate", false, v[0])) { + ScrDyn *empty = v[1]->kind == SCR_DYN_UNDEF ? scr_dyn_new_obj() : NULL; + int checks = sd_prime_checks(empty ? empty : v[1]); + if (empty) scr_dyn_release(empty); + if (checks >= 0) result = sd_check_prime(&x, checks); + } + mbedtls_mpi_free(&x); + sd_release_all(v, 2); + return result; +} + +/* A deferred callback(...args): the callback and its one or two + * arguments, each in a box. */ +static void sd_deliver(ScrClosure *self) { + ScrDyn *callback = (ScrDyn *)scr_box_get_ref(self->caps[0]); + size_t argc = self->ncaps - 1; + ScrDyn *args[2] = {NULL, NULL}; + for (size_t i = 0; i < argc; i++) args[i] = (ScrDyn *)scr_box_get_ref(self->caps[i + 1]); + ScrDyn *result = scr_dyn_call(callback, args, argc, "callback"); + if (result) scr_dyn_release(result); + for (size_t i = 0; i < argc; i++) scr_dyn_release(args[i]); + scr_dyn_release(callback); +} + +/* Calls callback(first[, second]) in a later macrotask (moves first and + * second). */ +static void sd_defer(const ScrDyn *callback, ScrDyn *first, ScrDyn *second) { + ScrDyn *values[3] = {scr_dyn_retain((ScrDyn *)callback), first, second}; + size_t count = second ? 3 : 2; + ScrClosure *c = scr_closure_new((void *)sd_deliver, count); + for (size_t i = 0; i < count; i++) { + c->caps[i] = scr_box_new_obj(scr_dyn_retain_v, scr_dyn_release_v, NULL); + scr_box_set_ref(c->caps[i], values[i]); + } + scr_loop_defer(c); +} + +/* The callback's error argument on success (+1): Node's undefined, or + * null for a callback typed to take Error | null. */ +static ScrDyn *sd_no_error(bool null_error) { + return null_error ? scr_dyn_new_null() : scr_dyn_retain((ScrDyn *)scr_dyn_undefined()); +} + +/* The pending error, taken as a dyn value (+1). */ +static ScrDyn *sd_take_error(void) { + ScrCaught *caught = scr_exc_take(); + ScrDyn *error = scr_caught_to_dyn(caught); + scr_caught_release(caught); + return error; +} + +/* checkPrime(candidate[, options], callback): argument errors throw; the + * callback receives (undefined, verdict) in a later macrotask (null for + * undefined when `null_error`). */ +void scr_crypto_check_prime_async(ScrDyn *candidate, ScrDyn *options, ScrDyn *callback, bool null_error) { + ScrDyn *args[3] = {candidate, options, callback}; + ScrDyn *v[3]; + mbedtls_mpi x; + mbedtls_mpi_init(&x); + if (sd_unbox_all(v, args, 3) && sd_large_number(&x, "candidate", false, v[0])) { + ScrDyn *bag = v[1], *fn = v[2]; + ScrDyn *empty = NULL; + if (scr_dyn_is_callable(bag)) { + fn = bag; + bag = empty = scr_dyn_new_obj(); + } else if (bag->kind == SCR_DYN_UNDEF) { + bag = empty = scr_dyn_new_obj(); + } + int checks = sd_validate_function("callback", fn) ? sd_prime_checks(bag) : -1; + ScrDyn *verdict = checks >= 0 ? sd_check_prime(&x, checks) : NULL; + if (verdict) sd_defer(fn, sd_no_error(null_error), verdict); + if (empty) scr_dyn_release(empty); + } + mbedtls_mpi_free(&x); + sd_release_all(v, 3); +} + +/* RandomPrimeConfig: the size and options checks, then the add and rem + * bounds. False with the error pending. */ +typedef struct { + size_t bits; + bool safe, bigint, has_add, has_rem; + mbedtls_mpi add, rem; +} ScDhPrimeConfig; + +static void sd_prime_config_free(ScDhPrimeConfig *c) { + mbedtls_mpi_free(&c->add); + mbedtls_mpi_free(&c->rem); +} + +static bool sd_prime_size(const ScrDyn *size) { + if (size->kind != SCR_DYN_NUM) { + scr_dyn_arg_type_fail("size", "of type number", size); + return false; + } + return sd_validate_int32_min("size", size->v.num, 1); +} + +static bool sd_prime_config(ScDhPrimeConfig *c, double size, const ScrDyn *options) { + if (!sd_validate_object("options", options)) return false; + const ScrDyn *safe = sd_option(options, "safe"); + const ScrDyn *bigint = sd_option(options, "bigint"); + const ScrDyn *add = sd_option(options, "add"); + const ScrDyn *rem = sd_option(options, "rem"); + if (safe->kind != SCR_DYN_UNDEF && safe->kind != SCR_DYN_BOOL) { + scr_dyn_prop_type_fail("options.safe", "of type boolean", safe); + return false; + } + if (bigint->kind != SCR_DYN_UNDEF && bigint->kind != SCR_DYN_BOOL) { + scr_dyn_prop_type_fail("options.bigint", "of type boolean", bigint); + return false; + } + c->safe = safe->kind == SCR_DYN_BOOL && safe->v.b; + c->bigint = bigint->kind == SCR_DYN_BOOL && bigint->v.b; + c->has_add = add->kind != SCR_DYN_UNDEF; + c->has_rem = rem->kind != SCR_DYN_UNDEF; + if (c->has_add && !sd_large_number(&c->add, "options.add", true, add)) return false; + if (c->has_rem && !sd_large_number(&c->rem, "options.rem", true, rem)) return false; + c->bits = (size_t)size; + if (c->has_add) { + if (mbedtls_mpi_bitlen(&c->add) > c->bits) { + sd_throw(SCR_ERR_RANGE, "ERR_OUT_OF_RANGE", "invalid options.add"); + return false; + } + if (c->has_rem && mbedtls_mpi_cmp_mpi(&c->add, &c->rem) <= 0) { + sd_throw(SCR_ERR_RANGE, "ERR_OUT_OF_RANGE", "invalid options.rem"); + return false; + } + } + if (c->bits > SD_MAX_PRIME_BITS) { + sd_prime_too_long("generatePrime sizes"); + return false; + } + return true; +} + +/* The generated prime as an ArrayBuffer of its bytes, or a bigint (+1), + * or NULL with OpenSSL's error pending. */ +static ScrDyn *sd_prime_result(ScDhPrimeConfig *c) { + mbedtls_mpi p; + mbedtls_mpi_init(&p); + ScrDyn *result = NULL; + if (sd_generate_prime(&p, c->bits, c->safe, c->has_add ? &c->add : NULL, c->has_add && c->has_rem ? &c->rem : NULL)) { + if (c->bigint) { + size_t len = 0; + mbedtls_mpi_write_string(&p, 16, NULL, 0, &len); + char *hex = malloc(len + 2); + if (!hex) scr_trap("scriptc: out of memory\n"); + hex[0] = '0'; + hex[1] = 'x'; + if (mbedtls_mpi_write_string(&p, 16, hex + 2, len, &len) != 0) scr_trap("scriptc: out of memory\n"); + ScrStr *text = scr_str_new(hex, strlen(hex)); + free(hex); + ScrBigInt *value = scr_bigint_parse(text); + scr_str_release(text); + if (value) { + result = scr_dyn_new_bigint(value); + scr_bigint_release(value); + } + } else { + size_t len = mbedtls_mpi_size(&p); + ScrBytes *bytes = scr_bytes_new(SCR_BYTES_U8, (double)len); + if (mbedtls_mpi_write_binary(&p, bytes->data, len) != 0) scr_trap("scriptc: out of memory\n"); + result = scr_array_buffer_from_bytes(bytes); + scr_bytes_release(bytes); + } + } + mbedtls_mpi_free(&p); + return result; +} + +/* generatePrimeSync(size[, options]). */ +ScrDyn *scr_crypto_generate_prime_sync(ScrDyn *size, ScrDyn *options) { + ScrDyn *args[2] = {size, options}; + ScrDyn *v[2]; + ScrDyn *result = NULL; + ScDhPrimeConfig c = {0}; + mbedtls_mpi_init(&c.add); + mbedtls_mpi_init(&c.rem); + if (sd_unbox_all(v, args, 2) && sd_prime_size(v[0])) { + ScrDyn *empty = v[1]->kind == SCR_DYN_UNDEF ? scr_dyn_new_obj() : NULL; + if (sd_prime_config(&c, v[0]->v.num, empty ? empty : v[1])) result = sd_prime_result(&c); + if (empty) scr_dyn_release(empty); + } + sd_prime_config_free(&c); + sd_release_all(v, 2); + return result; +} + +/* generatePrime(size[, options], callback): argument errors throw; the + * callback receives (undefined, prime), or OpenSSL's error alone, in a + * later macrotask (null for undefined when `null_error`). */ +void scr_crypto_generate_prime_async(ScrDyn *size, ScrDyn *options, ScrDyn *callback, bool null_error) { + ScrDyn *args[3] = {size, options, callback}; + ScrDyn *v[3]; + ScDhPrimeConfig c = {0}; + mbedtls_mpi_init(&c.add); + mbedtls_mpi_init(&c.rem); + if (sd_unbox_all(v, args, 3) && sd_prime_size(v[0])) { + ScrDyn *bag = v[1], *fn = v[2]; + ScrDyn *empty = NULL; + if (scr_dyn_is_callable(bag)) { + fn = bag; + bag = empty = scr_dyn_new_obj(); + } + if (sd_validate_function("callback", fn) && sd_prime_config(&c, v[0]->v.num, bag)) { + ScrDyn *prime = sd_prime_result(&c); + if (prime) sd_defer(fn, sd_no_error(null_error), prime); + else sd_defer(fn, sd_take_error(), NULL); + } + if (empty) scr_dyn_release(empty); + } + sd_prime_config_free(&c); + sd_release_all(v, 3); +} + /* ── methods ──────────────────────────────────────────────────────────── */ static ScrDyn *sd_arg(ScrDyn *const *args, size_t argc, size_t index) { diff --git a/tests/corpus/7150-cipher-legacy-modes.ts b/tests/corpus/7150-cipher-legacy-modes.ts new file mode 100644 index 000000000..80edfd029 --- /dev/null +++ b/tests/corpus/7150-cipher-legacy-modes.ts @@ -0,0 +1,74 @@ +// createCipheriv and createDecipheriv in the legacy modes: AES in ECB, +// CFB, CFB8, CFB1 and OFB, triple DES (three- and two-key) in CBC, ECB, +// CFB, CFB8, CFB1 and OFB, and bare chacha20. Fixed keys, IVs and inputs +// give Node's exact ciphertexts; chunked updates, string encodings, +// disabled padding and chacha20's 64-bit counter carry round-trip. +import { createCipheriv, createDecipheriv } from "node:crypto"; + +function bytes(length: number, seed: number): Buffer { + const out = Buffer.alloc(length); + for (let i = 0; i < length; i++) out[i] = (i * 29 + seed) & 0xff; + return out; +} + +const plaintext = Buffer.from("The five boxing wizards jump quickly; sphinx of black quartz, judge my vow."); + +function encrypt(name: string, key: Buffer, iv: Buffer | null, data: Buffer, chunk: number): Buffer { + const cipher = createCipheriv(name, key, iv); + const parts: Buffer[] = []; + for (let off = 0; off < data.length; off += chunk) parts.push(cipher.update(data.subarray(off, off + chunk))); + parts.push(cipher.final()); + return Buffer.concat(parts); +} + +function decrypt(name: string, key: Buffer, iv: Buffer | null, data: Buffer, chunk: number): Buffer { + const decipher = createDecipheriv(name, key, iv); + const parts: Buffer[] = []; + for (let off = 0; off < data.length; off += chunk) parts.push(decipher.update(data.subarray(off, off + chunk))); + parts.push(decipher.final()); + return Buffer.concat(parts); +} + +const cases: [string, number, number][] = []; +for (const bits of [128, 192, 256]) + for (const mode of ["ecb", "cfb", "cfb8", "cfb1", "ofb"]) cases.push([`aes-${bits}-${mode}`, bits / 8, mode === "ecb" ? 0 : 16]); +for (const name of ["des-ede3-cbc", "des3", "des-ede3", "des-ede3-ecb", "des-ede3-cfb", "des-ede3-cfb8", "des-ede3-cfb1", "des-ede3-ofb"]) + cases.push([name, 24, name === "des-ede3" || name === "des-ede3-ecb" ? 0 : 8]); +for (const name of ["des-ede-cbc", "des-ede", "des-ede-ecb", "des-ede-cfb", "des-ede-ofb"]) + cases.push([name, 16, name === "des-ede" || name === "des-ede-ecb" ? 0 : 8]); +cases.push(["chacha20", 32, 16]); + +for (const [name, keyLength, ivLength] of cases) { + const key = bytes(keyLength, keyLength + 3); + const iv = ivLength ? bytes(ivLength, 11) : null; + const sealed = encrypt(name, key, iv, plaintext, plaintext.length); + const chunked = encrypt(name, key, iv, plaintext, 7); + const opened = decrypt(name, key, iv, sealed, 5); + console.log(name.padEnd(14), sealed.length, sealed.toString("hex").slice(0, 48), chunked.equals(sealed), opened.equals(plaintext)); +} + +// String input and output encodings through a stream mode. +const ofb = createCipheriv("aes-128-ofb", bytes(16, 1), bytes(16, 2)); +const encoded = ofb.update("68656c6c6f20776f726c64", "hex", "base64") + ofb.final("base64"); +const unofb = createDecipheriv("aes-128-ofb", bytes(16, 1), bytes(16, 2)); +console.log(encoded, unofb.update(encoded, "base64", "utf8") + unofb.final("utf8")); + +// ECB and triple-DES CBC without padding over whole blocks. +for (const [name, keyLength, ivLength, block] of [["aes-256-ecb", 32, 0, 16], ["des-ede3-cbc", 24, 8, 8]] as const) { + const cipher = createCipheriv(name, bytes(keyLength, 5), ivLength ? bytes(ivLength, 6) : null); + cipher.setAutoPadding(false); + const sealed = Buffer.concat([cipher.update(bytes(block * 3, 7)), cipher.final()]); + const decipher = createDecipheriv(name, bytes(keyLength, 5), ivLength ? bytes(ivLength, 6) : null); + decipher.setAutoPadding(false); + const opened = Buffer.concat([decipher.update(sealed), decipher.final()]); + console.log(name, "unpadded", sealed.length, sealed.toString("hex").slice(0, 32), opened.equals(bytes(block * 3, 7))); +} + +// chacha20's counter is the IV's first 8 bytes, little-endian: the low +// word carries into the high word, which wraps. +const chachaKey = bytes(32, 9); +for (const counter of ["ffffffff00000000", "ffffffffffffffff", "fffffffe01000000"]) { + const iv = Buffer.from(counter + "0102030405060708", "hex"); + const stream = encrypt("chacha20", chachaKey, iv, Buffer.alloc(200), 50); + console.log("chacha20", counter, stream.toString("hex").slice(120, 152), stream.subarray(64, 96).toString("hex")); +} diff --git a/tests/corpus/7151-cipher-legacy-errors.cjs b/tests/corpus/7151-cipher-legacy-errors.cjs new file mode 100644 index 000000000..fe9c59bcc --- /dev/null +++ b/tests/corpus/7151-cipher-legacy-errors.cjs @@ -0,0 +1,83 @@ +// createCipheriv errors for the legacy modes, in Node's order: the IV +// length each mode takes (none for ECB), the key length, OpenSSL's +// "unsupported" for the ciphers its legacy provider holds, and the block +// modes' final-block and padding failures; the AEAD-only methods throw +// Node's invalid-state errors on the stream modes. +const crypto = require("crypto"); + +function attempt(label, run) { + try { + const value = run(); + console.log(label, "ok", Buffer.isBuffer(value) ? value.toString("hex") : value); + } catch (e) { + console.log(label, e.name, e.code, e.message, e.library, e.reason, JSON.stringify(e.opensslErrorStack)); + } +} + +const k16 = Buffer.alloc(16, 1), k24 = Buffer.alloc(24, 1), k32 = Buffer.alloc(32, 1); +attempt("ecb null iv", () => crypto.createCipheriv("aes-128-ecb", k16, null).final()); +attempt("ecb empty iv", () => crypto.createCipheriv("aes-128-ecb", k16, Buffer.alloc(0)).final()); +attempt("ecb 16-byte iv", () => crypto.createCipheriv("aes-128-ecb", k16, Buffer.alloc(16))); +attempt("cfb 15-byte iv", () => crypto.createCipheriv("aes-128-cfb", k16, Buffer.alloc(15))); +attempt("cfb null iv", () => crypto.createCipheriv("aes-128-cfb", k16, null)); +attempt("ofb key 15", () => crypto.createCipheriv("aes-128-ofb", Buffer.alloc(15), Buffer.alloc(16))); +attempt("des3 16-byte iv", () => crypto.createCipheriv("des-ede3-cbc", k24, Buffer.alloc(16))); +attempt("des3 16-byte key", () => crypto.createCipheriv("des-ede3-cbc", k16, Buffer.alloc(8))); +attempt("des-ede 24-byte key", () => crypto.createCipheriv("des-ede-cbc", k24, Buffer.alloc(8))); +attempt("des-ede3 ecb with iv", () => crypto.createCipheriv("des-ede3", k24, Buffer.alloc(8))); +attempt("chacha20 12-byte iv", () => crypto.createCipheriv("chacha20", k32, Buffer.alloc(12))); +attempt("chacha20 authTagLength ignored", () => crypto.createCipheriv("chacha20", k32, Buffer.alloc(16), { authTagLength: 16 }).update("x")); +attempt("upper-case name", () => crypto.createCipheriv("DES-EDE3-CBC", k24, Buffer.alloc(8)).final()); +attempt("des-cbc", () => crypto.createCipheriv("des-cbc", Buffer.alloc(8), Buffer.alloc(8))); +attempt("des-cbc bad iv", () => crypto.createCipheriv("des-cbc", Buffer.alloc(8), Buffer.alloc(3))); +attempt("des-cbc bad key", () => crypto.createCipheriv("des-cbc", Buffer.alloc(3), Buffer.alloc(8))); +attempt("bf-cbc", () => crypto.createDecipheriv("bf-cbc", k16, Buffer.alloc(8))); +attempt("rc4", () => crypto.createCipheriv("rc4", k16, null)); +attempt("rc4 with iv", () => crypto.createCipheriv("rc4", k16, Buffer.alloc(8))); +attempt("seed-ecb", () => crypto.createCipheriv("seed-ecb", k16, null)); +attempt("unknown cipher", () => crypto.createCipheriv("aes-128-xyz", k16, Buffer.alloc(16))); + +attempt("cfb getAuthTag", () => { + const c = crypto.createCipheriv("aes-128-cfb", k16, Buffer.alloc(16)); + c.update("x"); + c.final(); + return c.getAuthTag(); +}); +attempt("ofb setAAD", () => crypto.createCipheriv("aes-128-ofb", k16, Buffer.alloc(16)).setAAD(Buffer.alloc(1))); +attempt("chacha20 setAuthTag", () => crypto.createDecipheriv("chacha20", k32, Buffer.alloc(16)).setAuthTag(Buffer.alloc(16))); +attempt("cfb update sizes", () => { + const c = crypto.createCipheriv("aes-128-cfb", k16, Buffer.alloc(16)); + return [c.update("abc").length, c.update("defghijklmnopqrstu").length, c.final().length].join(","); +}); +attempt("cfb1 setAutoPadding(false)", () => { + const c = crypto.createCipheriv("aes-128-cfb1", k16, Buffer.alloc(16)); + c.setAutoPadding(false); + return Buffer.concat([c.update("abc"), c.final()]); +}); +attempt("ecb unpadded short", () => { + const c = crypto.createCipheriv("aes-128-ecb", k16, null); + c.setAutoPadding(false); + c.update(Buffer.alloc(31, 7)); + return c.final(); +}); +attempt("des3 unpadded short", () => { + const c = crypto.createCipheriv("des-ede3-cbc", k24, Buffer.alloc(8)); + c.setAutoPadding(false); + c.update(Buffer.alloc(7)); + return c.final(); +}); +attempt("des3 bad decrypt", () => { + const d = crypto.createDecipheriv("des-ede3-cbc", k24, Buffer.alloc(8)); + d.update(Buffer.alloc(16, 9)); + return d.final(); +}); +attempt("ecb decrypt 15 bytes", () => { + const d = crypto.createDecipheriv("aes-128-ecb", k16, null); + d.update(Buffer.alloc(15)); + return d.final(); +}); +attempt("ofb update after final", () => { + const c = crypto.createCipheriv("aes-128-ofb", k16, Buffer.alloc(16)); + c.final(); + return c.update("x"); +}); diff --git a/tests/corpus/7152-crypto-prime.ts b/tests/corpus/7152-crypto-prime.ts new file mode 100644 index 000000000..79edcc502 --- /dev/null +++ b/tests/corpus/7152-crypto-prime.ts @@ -0,0 +1,96 @@ +// checkPrimeSync, checkPrime, generatePrimeSync and generatePrime: known +// primes, composites and strong pseudoprimes as bigints, Buffers, typed +// arrays, ArrayBuffers and DataViews, the checks option, and generated +// primes' length, top bits, primality, residues and safety, as +// ArrayBuffers and bigints, synchronously and through the callbacks (whose +// error parameter, typed Error | null, receives null where Node passes +// undefined). The small sizes and the add and rem forms have one possible +// result. +import { checkPrime, checkPrimeSync, generatePrime, generatePrimeSync } from "node:crypto"; + +const mersenne61 = 2n ** 61n - 1n; +const mersenne127 = 2n ** 127n - 1n; +const mersenne521 = 2n ** 521n - 1n; +const known: [string, bigint][] = [ + ["0", 0n], + ["1", 1n], + ["2", 2n], + ["3", 3n], + ["4", 4n], + ["9", 9n], + ["17863", 17863n], + ["17881", 17881n], + ["561 (Carmichael)", 561n], + ["2047 (strong pseudoprime to 2)", 2047n], + ["3215031751 (strong pseudoprime to 2, 3, 5 and 7)", 3215031751n], + ["2^61 - 1", mersenne61], + ["2^61 + 1", mersenne61 + 2n], + ["2^127 - 1", mersenne127], + ["(2^61 - 1)(2^127 - 1)", mersenne61 * mersenne127], + ["2^521 - 1", mersenne521], + ["2^521 + 1", mersenne521 + 2n], +]; +for (const [label, value] of known) console.log(label, checkPrimeSync(value)); + +console.log("Buffer 13", checkPrimeSync(Buffer.from([0, 0, 13]))); +console.log("Uint8Array 15", checkPrimeSync(new Uint8Array([15]))); +const words = new ArrayBuffer(8); +new DataView(words).setBigUint64(0, mersenne61); +console.log("ArrayBuffer 2^61 - 1", checkPrimeSync(words)); +console.log("DataView 2^61 - 1", checkPrimeSync(new DataView(words))); +console.log("empty", checkPrimeSync(new Uint8Array(0))); +console.log("checks 1", checkPrimeSync(97n, { checks: 1 })); +console.log("checks 200", checkPrimeSync(3215031751n, { checks: 200 })); + +const bitLength = (n: bigint): number => n.toString(2).length; +const fromBuffer = (buffer: ArrayBuffer): bigint => BigInt("0x" + Buffer.from(buffer).toString("hex")); + +for (const bits of [2, 3, 4]) console.log("size", bits, generatePrimeSync(bits, { bigint: true })); +const five = generatePrimeSync(5, { bigint: true }); +console.log("size 5", five === 29n || five === 31n); +for (const bits of [8, 16, 31, 32, 33, 64, 127, 256, 512]) { + const p = generatePrimeSync(bits, { bigint: true }); + console.log("size", bits, bitLength(p) === bits, p >> BigInt(bits - 2), checkPrimeSync(p)); +} +const buffer = generatePrimeSync(96); +console.log("ArrayBuffer", buffer instanceof ArrayBuffer, buffer.byteLength, checkPrimeSync(buffer), bitLength(fromBuffer(buffer))); +console.log("bigint false", generatePrimeSync(40, { bigint: false }).byteLength); + +for (const bits of [3, 6, 7]) console.log("safe", bits, generatePrimeSync(bits, { safe: true, bigint: true })); +for (const bits of [16, 64, 128]) { + const p = generatePrimeSync(bits, { safe: true, bigint: true }); + console.log("safe", bits, bitLength(p) === bits, p >> BigInt(bits - 2), checkPrimeSync(p), checkPrimeSync((p - 1n) / 2n)); +} + +console.log("add 200 rem 1", generatePrimeSync(8, { add: 200n, rem: 1n, bigint: true })); +console.log("add 128", generatePrimeSync(8, { add: 128n, bigint: true })); +console.log("add 16 rem 3", generatePrimeSync(5, { add: 16n, rem: 3n, bigint: true })); +const safe4 = generatePrimeSync(5, { add: 4n, safe: true, bigint: true }); +console.log("add 4 safe", safe4 === 23n || safe4 === 47n); +for (const [label, add, rem] of [ + ["add 12 rem 11", 12n, 11n], + ["add 6", 6n, undefined], + ["add 1000 rem 7", 1000n, 7n], +] as const) { + const p = generatePrimeSync(48, rem === undefined ? { add, bigint: true } : { add, rem, bigint: true }); + console.log(label, p % add, checkPrimeSync(p), bitLength(p) >= 48); +} +const viaBuffer = generatePrimeSync(64, { add: Buffer.from([0x01, 0x00]), rem: new Uint8Array([0x0f]), bigint: true }); +console.log("add and rem as bytes", viaBuffer % 256n, checkPrimeSync(viaBuffer)); +const safeAdd = generatePrimeSync(64, { add: 24n, rem: 23n, safe: true, bigint: true }); +console.log("safe add 24 rem 23", safeAdd % 24n, checkPrimeSync((safeAdd - 1n) / 2n)); +console.log("rem without add", generatePrimeSync(16, { rem: 5n, bigint: true }) < 65536n); + +generatePrime(64, (err, prime) => { + console.log("callback", err ?? "no error", prime instanceof ArrayBuffer, prime.byteLength, checkPrimeSync(prime)); + generatePrime(128, { safe: true, bigint: true }, (err2, safe) => { + console.log("callback safe", err2 ?? "no error", bitLength(safe), checkPrimeSync((safe - 1n) / 2n)); + checkPrime(safe, (err3, result) => { + console.log("checkPrime", err3 ?? "no error", result); + checkPrime(Buffer.from([1, 1]), { checks: 3 }, (err4, result2) => { + console.log("checkPrime 257", err4 ?? "no error", result2); + }); + }); + }); +}); +console.log("sync done"); diff --git a/tests/corpus/7153-crypto-prime-errors.cjs b/tests/corpus/7153-crypto-prime-errors.cjs new file mode 100644 index 000000000..ff32e6e09 --- /dev/null +++ b/tests/corpus/7153-crypto-prime-errors.cjs @@ -0,0 +1,80 @@ +// checkPrime, checkPrimeSync, generatePrime and generatePrimeSync errors from +// JavaScript: argument types and ranges, the options bag, add and rem, and +// OpenSSL's generation failures, synchronously and through the callbacks. +const crypto = require("crypto"); + +function attempt(label, run) { + try { + const value = run(); + console.log(label, "ok", typeof value === "bigint" ? `${value}n` : value instanceof ArrayBuffer ? `ArrayBuffer(${value.byteLength})` : value); + } catch (e) { + console.log(label, e.name, e.code, e.message, JSON.stringify(Object.keys(e))); + } +} + +attempt("check string", () => crypto.checkPrimeSync("7")); +attempt("check number", () => crypto.checkPrimeSync(7)); +attempt("check negative", () => crypto.checkPrimeSync(-7n)); +attempt("check large negative", () => crypto.checkPrimeSync(-(2n ** 40n))); +attempt("check options null", () => crypto.checkPrimeSync(7n, null)); +attempt("check options array", () => crypto.checkPrimeSync(7n, [])); +attempt("check options string", () => crypto.checkPrimeSync(7n, "x")); +attempt("check checks string", () => crypto.checkPrimeSync(7n, { checks: "1" })); +attempt("check checks fraction", () => crypto.checkPrimeSync(7n, { checks: 1.5 })); +attempt("check checks -1", () => crypto.checkPrimeSync(7n, { checks: -1 })); +attempt("check checks 2^31", () => crypto.checkPrimeSync(7n, { checks: 2 ** 31 })); +attempt("check checks -0", () => crypto.checkPrimeSync(7n, { checks: -0 })); +attempt("check checks 10", () => crypto.checkPrimeSync(7n, { checks: 10 })); +attempt("check no callback", () => crypto.checkPrime(7n)); +attempt("check callback string", () => crypto.checkPrime(7n, {}, "cb")); +attempt("check async string", () => crypto.checkPrime("7", () => {})); +attempt("check async options null", () => crypto.checkPrime(7n, null, () => {})); + +attempt("size string", () => crypto.generatePrimeSync("8")); +attempt("size fraction", () => crypto.generatePrimeSync(8.5)); +attempt("size 0", () => crypto.generatePrimeSync(0)); +attempt("size 2^31", () => crypto.generatePrimeSync(2 ** 31)); +attempt("size NaN", () => crypto.generatePrimeSync(NaN)); +attempt("size 1", () => crypto.generatePrimeSync(1)); +attempt("size 2 safe", () => crypto.generatePrimeSync(2, { safe: true })); +attempt("size 4 safe", () => crypto.generatePrimeSync(4, { safe: true })); +attempt("size 5 safe", () => crypto.generatePrimeSync(5, { safe: true })); +attempt("options null", () => crypto.generatePrimeSync(8, null)); +attempt("options number", () => crypto.generatePrimeSync(8, 1)); +attempt("safe string", () => crypto.generatePrimeSync(8, { safe: "yes" })); +attempt("bigint number", () => crypto.generatePrimeSync(8, { bigint: 1 })); +attempt("add number", () => crypto.generatePrimeSync(8, { add: 6 })); +attempt("add string", () => crypto.generatePrimeSync(8, { add: "6" })); +attempt("add negative", () => crypto.generatePrimeSync(8, { add: -6n })); +attempt("rem number", () => crypto.generatePrimeSync(8, { add: 6n, rem: 1 })); +attempt("rem negative", () => crypto.generatePrimeSync(8, { add: 6n, rem: -1n })); +attempt("add too long", () => crypto.generatePrimeSync(8, { add: 1024n })); +attempt("add zero", () => crypto.generatePrimeSync(8, { add: 0n })); +attempt("add zero bytes", () => crypto.generatePrimeSync(8, { add: new Uint8Array(0) })); +attempt("rem equal add", () => crypto.generatePrimeSync(8, { add: 6n, rem: 6n })); +attempt("rem above add", () => crypto.generatePrimeSync(8, { add: 6n, rem: 7n })); +attempt("rem without add", () => crypto.generatePrimeSync(8, { rem: 1000n, bigint: true }) < 256n); +attempt("add 1 bigint", () => crypto.generatePrimeSync(8, { add: 1n, bigint: true }) < 256n); +attempt("gen no callback", () => crypto.generatePrime(8)); +attempt("gen options callback", () => crypto.generatePrime(8, {})); +attempt("gen size string", () => crypto.generatePrime("8", () => {})); +attempt("gen add too long", () => crypto.generatePrime(8, { add: 1024n }, () => {})); +attempt("gen options null", () => crypto.generatePrime(8, null, () => {})); + +// The callback forms' generation errors and results, one after another. +crypto.generatePrime(1, function (err, prime) { + console.log("gen 1 callback", arguments.length, err.name, err.code, err.message, JSON.stringify(Object.keys(err)), prime); + crypto.generatePrime(8, { add: 0n }, function (err2, prime2) { + console.log("gen add 0 callback", arguments.length, err2.message, prime2); + crypto.generatePrime(3, { safe: true, bigint: true }, function (err3, prime3) { + console.log("gen 3 safe callback", arguments.length, err3, prime3); + crypto.checkPrime(7n, function (err4, result) { + console.log("check 7 callback", arguments.length, err4, result); + crypto.checkPrime(new Uint8Array([0, 0, 9]), { checks: 3 }, function (err5, result2) { + console.log("check 9 callback", arguments.length, err5, result2); + }); + }); + }); + }); +}); +console.log("sync end"); diff --git a/tests/fixtures/node-types/fenced.ts b/tests/fixtures/node-types/fenced.ts index 644e901f4..27e68a618 100644 --- a/tests/fixtures/node-types/fenced.ts +++ b/tests/fixtures/node-types/fenced.ts @@ -70,12 +70,12 @@ http2.createSecureServer({ allowHTTP1: true, cert: "pem", key: "pem", streamRese http2.connect("https://localhost"); /* The crypto surface beyond the lowered slice (randomness, the hash * chain, the introspection statics): RSA encryption names the lowered - * asymmetric surface, symmetric ciphers the missing cipher stack, prime - * checks theirs, and setFips the FIPS truth (getFips() lowers to 0). */ -import { checkPrimeSync, createCipheriv, publicEncrypt, setFips } from "node:crypto"; + * asymmetric surface, symmetric ciphers the missing cipher stack, the + * secure heap the generic hint, and setFips the FIPS truth (getFips() lowers to 0). */ +import { createCipheriv, publicEncrypt, secureHeapUsed, setFips } from "node:crypto"; publicEncrypt("key", Buffer.alloc(1)); createCipheriv("aes-128-ccm", Buffer.alloc(16), Buffer.alloc(12), { authTagLength: 16 }); -checkPrimeSync(7n); +secureHeapUsed(); setFips(false); fetch("https://example.invalid/", { integrity: "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", diff --git a/tests/harness/__snapshots__/node-types-fenced.txt b/tests/harness/__snapshots__/node-types-fenced.txt index 133978b5e..cabce49c4 100644 --- a/tests/harness/__snapshots__/node-types-fenced.txt +++ b/tests/harness/__snapshots__/node-types-fenced.txt @@ -117,7 +117,7 @@ fenced.ts:69:71 - error SC2020: 'createSecureServer option 'streamResetBurst' wi fenced.ts:76:1 - error SC2020: 'crypto.publicEncrypt' is typed by @types/node but has no scriptc lowering yet - 75 | import { checkPrimeSync, createCipheriv, publicEncrypt, setFips } from "node:crypto"; + 75 | import { createCipheriv, publicEncrypt, secureHeapUsed, setFips } from "node:crypto"; 76 | publicEncrypt("key", Buffer.alloc(1)); | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 77 | createCipheriv("aes-128-ccm", Buffer.alloc(16), Buffer.alloc(12), { authTagLength: 16 }); @@ -129,22 +129,22 @@ fenced.ts:77:16 - error SC2020: 'crypto.createCipheriv with cipher 'aes-128-ccm' 76 | publicEncrypt("key", Buffer.alloc(1)); 77 | createCipheriv("aes-128-ccm", Buffer.alloc(16), Buffer.alloc(12), { authTagLength: 16 }); | ^~~~~~~~~~~~~ - 78 | checkPrimeSync(7n); + 78 | secureHeapUsed(); - hint: the supported ciphers are aes-128-cbc, aes-192-cbc, aes-256-cbc, aes-128-ctr, aes-192-ctr, aes-256-ctr, aes-128-gcm, aes-192-gcm, aes-256-gcm, and chacha20-poly1305 + hint: the supported ciphers are aes-128, aes-192 and aes-256 in cbc, ctr, gcm, ecb, cfb, cfb8, cfb1 and ofb modes, des-ede3 in cbc, ecb, cfb, cfb8, cfb1 and ofb modes, des-ede in cbc, ecb, cfb and ofb modes, chacha20, and chacha20-poly1305 -fenced.ts:78:1 - error SC2020: 'crypto.checkPrimeSync' is typed by @types/node but has no scriptc lowering yet +fenced.ts:78:1 - error SC2020: 'crypto.secureHeapUsed' is typed by @types/node but has no scriptc lowering yet 77 | createCipheriv("aes-128-ccm", Buffer.alloc(16), Buffer.alloc(12), { authTagLength: 16 }); - 78 | checkPrimeSync(7n); - | ^~~~~~~~~~~~~~~~~~ + 78 | secureHeapUsed(); + | ^~~~~~~~~~~~~~~~ 79 | setFips(false); hint: the type checker sees everything @types/node declares, but only the supported surface compiles (https://scriptc.dev/limitations) fenced.ts:79:1 - error SC2020: 'crypto.setFips' is typed by @types/node but has no scriptc lowering yet - 78 | checkPrimeSync(7n); + 78 | secureHeapUsed(); 79 | setFips(false); | ^~~~~~~~~~~~~~ 80 | fetch("https://example.invalid/", { diff --git a/tests/harness/corpus-costs/7150-cipher-legacy-modes.ts.json b/tests/harness/corpus-costs/7150-cipher-legacy-modes.ts.json new file mode 100644 index 000000000..40196decf --- /dev/null +++ b/tests/harness/corpus-costs/7150-cipher-legacy-modes.ts.json @@ -0,0 +1 @@ +{ "plain": 6099, "san": 8251 } diff --git a/tests/harness/corpus-costs/7151-cipher-legacy-errors.cjs.json b/tests/harness/corpus-costs/7151-cipher-legacy-errors.cjs.json new file mode 100644 index 000000000..6a82dd081 --- /dev/null +++ b/tests/harness/corpus-costs/7151-cipher-legacy-errors.cjs.json @@ -0,0 +1 @@ +{ "plain": 7845, "san": 9581 } diff --git a/tests/harness/corpus-costs/7152-crypto-prime.ts.json b/tests/harness/corpus-costs/7152-crypto-prime.ts.json new file mode 100644 index 000000000..4df2be436 --- /dev/null +++ b/tests/harness/corpus-costs/7152-crypto-prime.ts.json @@ -0,0 +1 @@ +{ "plain": 7273, "san": 10284 } diff --git a/tests/harness/corpus-costs/7153-crypto-prime-errors.cjs.json b/tests/harness/corpus-costs/7153-crypto-prime-errors.cjs.json new file mode 100644 index 000000000..d20dc198a --- /dev/null +++ b/tests/harness/corpus-costs/7153-crypto-prime-errors.cjs.json @@ -0,0 +1 @@ +{ "plain": 5893, "san": 5691 } diff --git a/tests/harness/errors.test.ts b/tests/harness/errors.test.ts index b5fb028f9..c8b0b51b8 100644 --- a/tests/harness/errors.test.ts +++ b/tests/harness/errors.test.ts @@ -847,8 +847,9 @@ try { expect(r.exitCode).toBe(0); expect(r.stdout).toBe( "Error SC2020 The cipher 'aes-128-ccm' has no native implementation; the supported ciphers are " + - "aes-128-cbc, aes-192-cbc, aes-256-cbc, aes-128-ctr, aes-192-ctr, aes-256-ctr, aes-128-gcm, " + - "aes-192-gcm, aes-256-gcm and chacha20-poly1305\n" + + "aes-128, aes-192 and aes-256 in cbc, ctr, gcm, ecb, cfb, cfb8, cfb1 and ofb modes, des-ede3 in " + + "cbc, ecb, cfb, cfb8, cfb1 and ofb modes, des-ede in cbc, ecb, cfb and ofb modes, chacha20 and " + + "chacha20-poly1305\n" + "Error ERR_CRYPTO_UNKNOWN_CIPHER Unknown cipher\n", ); expect(r.stderr).toBe(""); From 9aa3b15174dc524eb1157429e133710a9b54a547 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 09:44:27 -0500 Subject: [PATCH 035/197] Support construct, writev, decodeStrings and defaultEncoding on streams --- .../builtin-apis/streams-and-readline.mdx | 2 + .../limitations/builtin-function-values.mdx | 3 +- packages/cli/test/native-link-info.test.ts | 4 +- .../ambient/scriptc-node-fallback.d.ts | 46 +- packages/compiler/src/backend/llvm/emitter.ts | 7 +- .../src/backend/llvm/expr-stream-callbacks.ts | 76 ++- .../compiler/src/backend/llvm/lib-shared.ts | 6 + .../compiler/src/backend/llvm/lib-stream.ts | 13 +- packages/compiler/src/backend/runtime-abi.ts | 4 +- .../src/backend/runtime-pack-targets.test.ts | 2 +- .../compiler/src/backend/runtime-pack.test.ts | 2 +- .../src/frontend/lowering/containers/bytes.ts | 3 +- .../src/frontend/lowering/lower-classes.ts | 13 +- .../src/frontend/lowering/lower-stream.ts | 255 ++++++++-- packages/compiler/src/ir/builtin-effects.ts | 1 + .../compiler/src/ir/builtin-signatures.ts | 6 + packages/compiler/src/ir/runtime-features.ts | 1 + packages/compiler/src/ir/validate.ts | 7 + .../test/native-codegen-integration.test.ts | 4 +- .../corpus/7246-stream-construct.ts.json | 6 + .../corpus/7247-stream-construct.mjs.json | 6 + .../tests/corpus/7248-stream-writev.ts.json | 6 + .../tests/corpus/7249-stream-writev.cjs.json | 6 + .../corpus/7250-stream-decode-strings.ts.json | 6 + .../7251-stream-write-encodings.mjs.json | 6 + .../runtime-darwin-arm64/scripts/build.mjs | 2 +- .../runtime-pack-common/scripts/build.mjs | 2 +- packages/runtime/src/scr_console.c | 2 +- packages/runtime/src/scr_core.h | 2 +- packages/runtime/src/scr_dyn_builtin_stream.c | 120 +++-- packages/runtime/src/scr_stream.c | 481 +++++++++++++++--- packages/runtime/src/scr_stream.h | 17 + tests/corpus/7246-stream-construct.ts | 156 ++++++ tests/corpus/7247-stream-construct.mjs | 101 ++++ tests/corpus/7248-stream-writev.ts | 124 +++++ tests/corpus/7249-stream-writev.cjs | 89 ++++ tests/corpus/7250-stream-decode-strings.ts | 89 ++++ tests/corpus/7251-stream-write-encodings.mjs | 65 +++ .../7246-stream-construct.ts.json | 1 + .../7247-stream-construct.mjs.json | 1 + .../corpus-costs/7248-stream-writev.ts.json | 1 + .../corpus-costs/7249-stream-writev.cjs.json | 1 + .../7250-stream-decode-strings.ts.json | 1 + .../7251-stream-write-encodings.mjs.json | 1 + .../self-hosting-native-toolchain.test.ts | 2 +- 45 files changed, 1543 insertions(+), 206 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7246-stream-construct.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7247-stream-construct.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7248-stream-writev.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7249-stream-writev.cjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7250-stream-decode-strings.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7251-stream-write-encodings.mjs.json create mode 100644 tests/corpus/7246-stream-construct.ts create mode 100644 tests/corpus/7247-stream-construct.mjs create mode 100644 tests/corpus/7248-stream-writev.ts create mode 100644 tests/corpus/7249-stream-writev.cjs create mode 100644 tests/corpus/7250-stream-decode-strings.ts create mode 100644 tests/corpus/7251-stream-write-encodings.mjs create mode 100644 tests/harness/corpus-costs/7246-stream-construct.ts.json create mode 100644 tests/harness/corpus-costs/7247-stream-construct.mjs.json create mode 100644 tests/harness/corpus-costs/7248-stream-writev.ts.json create mode 100644 tests/harness/corpus-costs/7249-stream-writev.cjs.json create mode 100644 tests/harness/corpus-costs/7250-stream-decode-strings.ts.json create mode 100644 tests/harness/corpus-costs/7251-stream-write-encodings.mjs.json diff --git a/docs/content/limitations/builtin-apis/streams-and-readline.mdx b/docs/content/limitations/builtin-apis/streams-and-readline.mdx index 8461cc71c..2a514a5a0 100644 --- a/docs/content/limitations/builtin-apis/streams-and-readline.mdx +++ b/docs/content/limitations/builtin-apis/streams-and-readline.mdx @@ -23,6 +23,8 @@ `stream.compose` accepts two or more streams, or one `Duplex`; functions, iterables, and web streams as stages are unsupported, and the `ERR_INVALID_ARG_VALUE` message for a stream that is no longer readable or writable names the stream's class instead of printing the stream. `pipeline` from `node:stream/promises` accepts a trailing `{ signal, end }` object, and `finished` accepts `readable`, `writable`, `error`, `cleanup`, and `signal` options, with boolean literals for the flags. Stream constructors accept a `signal` option in an options object literal, and `stream.addAbortSignal` accepts any stream. + The `construct` option and `_construct` method, `decodeStrings`, and `defaultEncoding` work on every stream class, and the `writev` option and `_writev` method on Writable and Duplex; `writev` on a Transform or PassThrough, assigning `_construct` after construction, and a non-literal `decodeStrings` or `defaultEncoding` in an options object literal are unsupported. + `setDefaultEncoding()` takes any encoding Node knows. A `transform` or `flush` option callback whose completion callback types `data` as `any`, as @types/node's `TransformCallback` does, may pass a Buffer, a string, or nothing; any other value throws a TypeError where the callback is called, instead of Node's `ERR_INVALID_ARG_TYPE` stream error. `node:readline` and `node:readline/promises` (also reached as `readline.promises`) interfaces, from `createInterface(options)` or `createInterface(input, output, completer, terminal)`, read `process.stdin` or a readable stream: `question`, `prompt`, `write` (with a key object literal of `name`, `ctrl`, `meta`, and `shift`), `pause`, `resume`, `'line'`/`'close'`/`'pause'`/`'resume'`/`'error'`/`'history'`/`'SIGINT'`/`'SIGTSTP'`/`'SIGCONT'` listeners, and `for await` compile. `on`, `once`, and `addListener` return the interface, so registrations chain. diff --git a/docs/content/limitations/builtin-function-values.mdx b/docs/content/limitations/builtin-function-values.mdx index c1bab6680..0fa62e558 100644 --- a/docs/content/limitations/builtin-function-values.mdx +++ b/docs/content/limitations/builtin-function-values.mdx @@ -28,7 +28,8 @@ The `captureRejections` option and `events.errorMonitor` listeners are not suppo The stream calls the object's `_read`, `_write`, `_final`, `_transform`, `_flush`, and `_destroy` with the object as `this`, and the stream prototypes' methods and accessors work on it. Its listeners live in the native stream rather than in `_events`, and `_readableState` and `_writableState` are read-only views that answer the commonly read fields. Typed code that receives such an object as a stream class uses its native stream, and piping a native stream into it works. -The options the native streams refuse (`decodeStrings: false`, `signal`, and others) are refused here too, as are `_writev`, `_construct`, symbol event names, `listeners()`, `rawListeners()`, `wrap()`, strings written to a byte-mode stream in encodings other than UTF-8, and pipes into a destination that overrides `write()` or `end()`. +The options the native streams refuse (`signal` and others) are refused here too, as are `_writev` on a Transform, `_final` on a Transform, symbol event names, `listeners()`, `rawListeners()`, `wrap()`, and pipes into a destination that overrides `write()` or `end()`. +`_construct` and `_writev` are read when `Readable.call(this, options)` or `Writable.call(this, options)` runs, as Node's constructors read them. Other class constructors passed to `util.inherits` or `util.deprecate` are refused. Deprecation warnings are emitted on the next tick and follow assignments to `process.noDeprecation` and `process.throwDeprecation`; compiled binaries take no Node command-line flags such as `--no-deprecation` or `--trace-deprecation`, and `process.traceDeprecation` is unsupported. A stored `util.deprecate` reports a `length` of 4, where Node reports 3. diff --git a/packages/cli/test/native-link-info.test.ts b/packages/cli/test/native-link-info.test.ts index 630ef7173..a7a5e718a 100644 --- a/packages/cli/test/native-link-info.test.ts +++ b/packages/cli/test/native-link-info.test.ts @@ -72,7 +72,7 @@ describe.runIf(supported)("macOS arm64 native link info", () => { minimum_os: "14.0", }, program: { object, entry_symbol: "main" }, - runtime_abi: { version: 26, marker: "scr_runtime_abi_v26" }, + runtime_abi: { version: 27, marker: "scr_runtime_abi_v27" }, runtime_pack: { kind: "precompiled", package: "@scriptc/runtime-darwin-arm64", @@ -165,6 +165,6 @@ describe.runIf(supported)("macOS arm64 native link info", () => { () => null, (failure: { stderr?: string }) => failure, ); - expect(error?.stderr).toContain("scr_runtime_abi_v26"); + expect(error?.stderr).toContain("scr_runtime_abi_v27"); }); }); diff --git a/packages/compiler/ambient/scriptc-node-fallback.d.ts b/packages/compiler/ambient/scriptc-node-fallback.d.ts index 2a3f11be9..abbd023d4 100644 --- a/packages/compiler/ambient/scriptc-node-fallback.d.ts +++ b/packages/compiler/ambient/scriptc-node-fallback.d.ts @@ -7090,7 +7090,11 @@ declare module "stream" { encoding: string, callback: (error?: Error | null) => void, ): void; - writev?(this: Writable, chunks: unknown[], callback: (error?: Error | null) => void): void; + writev?( + this: Writable, + chunks: Array<{ chunk: any; encoding: string }>, + callback: (error?: Error | null) => void, + ): void; final?(this: Writable, callback: (error?: Error | null) => void): void; destroy?(this: Writable, error: Error | null, callback: (error?: Error | null) => void): void; construct?(this: Writable, callback: (error?: Error | null) => void): void; @@ -7125,7 +7129,11 @@ declare module "stream" { readable?: boolean; writable?: boolean; defaultEncoding?: string; - writev?(this: Duplex, chunks: unknown[], callback: (error?: Error | null) => void): void; + writev?( + this: Duplex, + chunks: Array<{ chunk: any; encoding: string }>, + callback: (error?: Error | null) => void, + ): void; signal?: AbortSignal; } interface TransformOptions { @@ -7210,6 +7218,36 @@ declare module "stream" { ): void; } & ObjectModeFlag; + /* decodeStrings: false — a string write reaches write/transform as the + * string with its encoding (Buffers keep 'buffer'). */ + interface StringWritableOptions extends Omit { + decodeStrings: false; + write?( + this: Writable, + chunk: string | Buffer, + encoding: string, + callback: (error?: Error | null) => void, + ): void; + } + type StringDuplexOptions = Omit & { + decodeStrings: false; + write?( + this: Duplex, + chunk: string | Buffer, + encoding: string, + callback: (error?: Error | null) => void, + ): void; + }; + type StringTransformOptions = Omit & { + decodeStrings: false; + transform?( + this: Transform, + chunk: string | Buffer, + encoding: string, + callback: (error?: Error | null, data?: Buffer | string) => void, + ): void; + }; + /* The operators' options: concurrency and highWaterMark bound map's * in-flight callbacks and queued results. */ interface ArrayOptions { @@ -7340,6 +7378,7 @@ declare module "stream" { class Writable extends EventEmitter { constructor(opts: ObjectWritableOptions); + constructor(opts: StringWritableOptions); constructor(opts?: WritableOptions); /* The underscore-method surface (see Readable). */ _write(chunk: any, encoding: string, callback: (error?: Error | null) => void): void; @@ -7418,6 +7457,7 @@ declare module "stream" { * again. The lowering treats both halves as first-class. */ class Duplex extends Readable { constructor(opts: ObjectDuplexOptions); + constructor(opts: StringDuplexOptions); constructor(opts?: DuplexOptions); /* The underscore-method surface (see Readable). */ _write(chunk: any, encoding: string, callback: (error?: Error | null) => void): void; @@ -7517,6 +7557,7 @@ declare module "stream" { class Transform extends Duplex { constructor(opts: ObjectTransformOptions); + constructor(opts: StringTransformOptions); constructor(opts?: TransformOptions); /* The underscore-method surface (see Readable). */ _transform( @@ -7528,6 +7569,7 @@ declare module "stream" { } class PassThrough extends Transform { constructor(opts: ObjectTransformOptions); + constructor(opts: StringTransformOptions); constructor(opts?: TransformOptions); } diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index c1cf66af4..4cfa36b5b 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -6111,7 +6111,10 @@ export class LlEmitter { return streamDataAdapter(this, cbT); } - streamDoneFnFor(kind: "w" | "f" | "d" | "t" | "l", doneT: IrType & { kind: "func" }): string { + streamDoneFnFor( + kind: "w" | "f" | "d" | "t" | "l" | "c", + doneT: IrType & { kind: "func" }, + ): string { return streamDoneFnFor(this, kind, doneT); } @@ -6135,7 +6138,7 @@ export class LlEmitter { return zlibBytesThunkFor(this, cbT); } - streamCbThunkFor(kind: "r" | "w" | "f" | "d" | "t" | "l" | "e", cbT: IrType): string { + streamCbThunkFor(kind: "r" | "w" | "f" | "d" | "t" | "l" | "e" | "c" | "v", cbT: IrType): string { return streamCbThunkFor(this, kind, cbT); } diff --git a/packages/compiler/src/backend/llvm/expr-stream-callbacks.ts b/packages/compiler/src/backend/llvm/expr-stream-callbacks.ts index 0570d0ea7..f50ae95e3 100644 --- a/packages/compiler/src/backend/llvm/expr-stream-callbacks.ts +++ b/packages/compiler/src/backend/llvm/expr-stream-callbacks.ts @@ -212,7 +212,7 @@ export function streamDataAdapter( export function streamDoneFnFor( host: LlvmEmitterContext, - kind: "w" | "f" | "d" | "t" | "l", + kind: "w" | "f" | "d" | "t" | "l" | "c", doneT: IrType & { kind: "func" }, ): string { const key = `sd:${kind}:${typeKey(doneT)}`; @@ -256,19 +256,21 @@ export function streamDoneFnFor( } } const entry = - kind === "w" - ? "scr_stream_write_done" - : kind === "f" - ? "scr_stream_final_done" - : kind === "d" - ? "scr_stream_destroy_done" - : kind === "t" - ? valueData - ? "scr_stream_transform_done_value" - : "scr_stream_transform_done" - : valueData - ? "scr_stream_flush_done_value" - : "scr_stream_flush_done"; + kind === "c" + ? "scr_stream_construct_done" + : kind === "w" + ? "scr_stream_write_done" + : kind === "f" + ? "scr_stream_final_done" + : kind === "d" + ? "scr_stream_destroy_done" + : kind === "t" + ? valueData + ? "scr_stream_transform_done_value" + : "scr_stream_transform_done" + : valueData + ? "scr_stream_flush_done_value" + : "scr_stream_flush_done"; const twoSlot = (kind === "t" || kind === "l") && !valueData; host.declare( `declare void @${entry}(ptr, ptr${twoSlot ? ", ptr, ptr" : valueData ? ", ptr" : ""})`, @@ -623,7 +625,7 @@ export function zlibBytesThunkFor( export function streamCbThunkFor( host: LlvmEmitterContext, - kind: "r" | "w" | "f" | "d" | "t" | "l" | "e", + kind: "r" | "w" | "f" | "d" | "t" | "l" | "e" | "c" | "v", cbT: IrType, ): string { if (cbT.kind !== "func") @@ -638,9 +640,11 @@ export function streamCbThunkFor( ? ["ptr %s", "double %size"] : kind === "w" || kind === "t" ? ["ptr %s", "ptr %chunk"] - : kind === "d" || kind === "e" - ? ["ptr %s", "ptr %err"] - : ["ptr %s"]; + : kind === "v" + ? ["ptr %s", "ptr %chunks"] + : kind === "d" || kind === "e" + ? ["ptr %s", "ptr %err"] + : ["ptr %s"]; const declared = cbT.params; const hasThis = declared[0] !== undefined && declared[0].kind === "object"; if (declared.length === 0) { @@ -649,7 +653,9 @@ export function streamCbThunkFor( ); } const off = hasThis ? 1 : 0; - const full = (kind === "r" ? 1 : kind === "w" || kind === "t" ? 3 : kind === "d" ? 2 : 1) + off; + const full = + (kind === "r" ? 1 : kind === "w" || kind === "t" ? 3 : kind === "d" || kind === "v" ? 2 : 1) + + off; if (declared.length > full) { throw new InternalCompilerError( `llvm emitter bug: stream '${kind}' callback declares ${declared.length} params (frontend must fence)`, @@ -697,6 +703,31 @@ export function streamCbThunkFor( } continue; } + if (kind === "v" && pos === 0) { + // writev's chunks: the batch array (borrowed from the runtime), as + // a dyn value or converted to the declared type + if (p.kind === "dyn") { + host.declare(`declare ptr @scr_dyn_retain_v(ptr)`); + d.push(` %vc${i} = call ptr @scr_dyn_retain_v(ptr %chunks)`); + passed.push(`ptr %vc${i}`); + } else { + const helper = host.dyn.dynCheckHelper(p, false); + const ty = host.llType(p); + host.declare(`declare zeroext i1 @scr_exc_pending()`); + if (hasThis) host.declare(`declare void @scr_stream_release_v(ptr)`); + d.push( + ` %vt${i} = call ${ty === "i1" ? "zeroext i1" : ty} @${helper}(ptr %chunks, ptr null)`, + ` %vtbad${i} = call zeroext i1 @scr_exc_pending()`, + ` br i1 %vtbad${i}, label %vtfail${i}, label %vtok${i}`, + `vtfail${i}:`, + ...(hasThis ? [` call void @scr_stream_release_v(ptr %sr)`] : []), + ` ret void`, + `vtok${i}:`, + ); + passed.push(`${ty} %vt${i}`); + } + continue; + } const isChunkPos = (kind === "w" || kind === "t") && pos === 0; const isEncPos = (kind === "w" || kind === "t") && pos === 1; const isErrPos = (kind === "d" || kind === "e") && pos === 0; @@ -705,7 +736,7 @@ export function streamCbThunkFor( ? false : kind === "w" || kind === "t" ? pos === 2 - : kind === "f" || kind === "l" + : kind === "f" || kind === "l" || kind === "c" ? pos === 0 : pos === 1; if (p.kind === "dyn") { @@ -751,7 +782,7 @@ export function streamCbThunkFor( d.push(` br label %ego${i}`, `ego${i}:`, ` %edv${i} = load ptr, ptr %edslot${i}`); passed.push(`ptr %edv${i}`); } else if (isDonePos) { - const glue = `scr_stream_done_dyn_${kind}`; + const glue = `scr_stream_done_dyn_${kind === "v" ? "w" : kind}`; host.declare(`declare ptr @${glue}(ptr, ptr, ${host.sizeType})`); host.declare(`declare ptr @scr_dyn_new_func(ptr, ptr, i32, ptr, ptr)`); const clo = doneClosure(`@${glue}`, `dd${i}`); @@ -872,7 +903,8 @@ export function streamCbThunkFor( ); passed.push(`ptr %euv${i}`); } else if (isDonePos) { - const doneKind = kind as "w" | "f" | "d" | "t" | "l"; // "e" has no done position + // "e" has no done position; writev completes as a write + const doneKind = (kind === "v" ? "w" : kind) as "w" | "f" | "d" | "t" | "l" | "c"; if (p.kind !== "func") throw new InternalCompilerError("llvm emitter bug: stream done callback not a func"); const doneFn = host.streamDoneFnFor(doneKind, p); diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index 9a65d9a24..46bfc279e 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -1126,6 +1126,9 @@ export const LIB_FN_SYMS: Record = { "readable.pushStrEnc": "scr_stream_push_str_enc", "readable.pushEncoding": "scr_stream_set_push_encoding", "stream.objectMode": "scr_stream_object_mode", + "stream.decodeStrings": "scr_stream_decode_strings", + "writable.writeEncoding": "scr_stream_write_encoding", + "writable.setDefaultEncoding": "scr_stream_set_default_encoding", "stream.chunkDone": "scr_stream_chunk_done", "readable.pushNull": "scr_stream_push_null", "readable.pushDyn": "scr_stream_push_dyn", @@ -1729,6 +1732,9 @@ export const USES_TIMERS_LIB_FNS = new Set([ "stream.setDestroy", "stream.setTransform", "stream.setFlush", + "stream.setConstruct", + "stream.setWritev", + "stream.setWritevDefault", "process.activeResources", "stream.finished", "stream.finishedDyn", diff --git a/packages/compiler/src/backend/llvm/lib-stream.ts b/packages/compiler/src/backend/llvm/lib-stream.ts index 6d6c6f99c..4611b3bb8 100644 --- a/packages/compiler/src/backend/llvm/lib-stream.ts +++ b/packages/compiler/src/backend/llvm/lib-stream.ts @@ -187,18 +187,24 @@ export function emitStreamLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlV e.fn === "stream.setFinal" || e.fn === "stream.setDestroy" || e.fn === "stream.setTransform" || - e.fn === "stream.setFlush" + e.fn === "stream.setFlush" || + e.fn === "stream.setConstruct" || + e.fn === "stream.setWritev" || + e.fn === "stream.setWritevDefault" ) { // The underscore-method assignment surface: the runtime slot swaps // its closure (+1 moves in) and invoke thunk — the next // _read/_write/... dispatch uses it, Node's timing. - const kindOf: Record = { + const kindOf: Record = { "stream.setRead": "r", "stream.setWrite": "w", "stream.setFinal": "f", "stream.setDestroy": "d", "stream.setTransform": "t", "stream.setFlush": "l", + "stream.setConstruct": "c", + "stream.setWritev": "v", + "stream.setWritevDefault": "v", }; const symOf: Record = { "stream.setRead": "scr_stream_set_read", @@ -207,6 +213,9 @@ export function emitStreamLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlV "stream.setDestroy": "scr_stream_set_destroy", "stream.setTransform": "scr_stream_set_transform", "stream.setFlush": "scr_stream_set_flush", + "stream.setConstruct": "scr_stream_set_construct", + "stream.setWritev": "scr_stream_set_writev", + "stream.setWritevDefault": "scr_stream_set_writev_default", }; const recv = host.emitExpr(e.args[0]!); const cb = host.emitExpr(e.args[1]!); diff --git a/packages/compiler/src/backend/runtime-abi.ts b/packages/compiler/src/backend/runtime-abi.ts index 703e8fe48..3a41d9176 100644 --- a/packages/compiler/src/backend/runtime-abi.ts +++ b/packages/compiler/src/backend/runtime-abi.ts @@ -2,8 +2,8 @@ * of the link contract: a program object keeps this symbol undefined and the * matching runtime defines it, so incompatible manual links fail before the * program can start. */ -export const RUNTIME_ABI_VERSION = 26 as const; -export const RUNTIME_ABI_MARKER = "scr_runtime_abi_v26" as const; +export const RUNTIME_ABI_VERSION = 27 as const; +export const RUNTIME_ABI_MARKER = "scr_runtime_abi_v27" as const; /** Object consumption is useful today, but the complete scr_* surface may * still change before 1.0. The versioned marker prevents accidental mixing; diff --git a/packages/compiler/src/backend/runtime-pack-targets.test.ts b/packages/compiler/src/backend/runtime-pack-targets.test.ts index c884f1132..af61e5229 100644 --- a/packages/compiler/src/backend/runtime-pack-targets.test.ts +++ b/packages/compiler/src/backend/runtime-pack-targets.test.ts @@ -52,7 +52,7 @@ async function fixture(target: NativeTargetSpec, workers = false) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 26, marker: "scr_runtime_abi_v26" }, + runtime_abi: { version: 27, marker: "scr_runtime_abi_v27" }, compiler: { command: "fixture", identity: "fixture", target: target.llvmTriple }, macros: { executable: [], excluded: ["SCR_LIB"], sanitizer: "external-toolchain-required" }, flavors: { diff --git a/packages/compiler/src/backend/runtime-pack.test.ts b/packages/compiler/src/backend/runtime-pack.test.ts index 3a0b7870a..d09509911 100644 --- a/packages/compiler/src/backend/runtime-pack.test.ts +++ b/packages/compiler/src/backend/runtime-pack.test.ts @@ -127,7 +127,7 @@ async function fixture(target: NativeTargetSpec = MACOS_ARM64_TARGET) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 26, marker: "scr_runtime_abi_v26" }, + runtime_abi: { version: 27, marker: "scr_runtime_abi_v27" }, compiler: { command: "clang", identity: "fixture clang", diff --git a/packages/compiler/src/frontend/lowering/containers/bytes.ts b/packages/compiler/src/frontend/lowering/containers/bytes.ts index fd4420b69..995cb177a 100644 --- a/packages/compiler/src/frontend/lowering/containers/bytes.ts +++ b/packages/compiler/src/frontend/lowering/containers/bytes.ts @@ -1414,7 +1414,8 @@ const BUF_ENCODINGS: Record = { * does not know — the ladder callers (stream options) turn unknown * literals into Node's runtime ERR_UNKNOWN_ENCODING throw. */ export function knownBufEncoding(name: string): string | undefined { - return own(BUF_ENCODINGS, name); + // Buffer.isEncoding: any letter case + return own(BUF_ENCODINGS, name) ?? own(BUF_ENCODINGS, name.toLowerCase()); } /** The literal encoding argument of a Buffer surface, normalized — or a diff --git a/packages/compiler/src/frontend/lowering/lower-classes.ts b/packages/compiler/src/frontend/lowering/lower-classes.ts index d8ccc15b7..682b9bde6 100644 --- a/packages/compiler/src/frontend/lowering/lower-classes.ts +++ b/packages/compiler/src/frontend/lowering/lower-classes.ts @@ -133,6 +133,7 @@ import { STREAM_PROP_MEMBERS, STREAM_COMPLETION_CALLBACK_INDEX, UNDERSCORE_METHODS, + streamExtraCallbacks, lowerStreamNew, lowerStreamSuperCall, streamCtorShape, @@ -2746,11 +2747,11 @@ export function collectClassShapeInner( `overriding the stream member '${name}' (the runtime owns the stream surface)`, ); } - if (name === "_writev" || name === "_construct") { + if (name === "_writev" && !streamExtraCallbacks(streamBase.def.name).includes("writev")) { lowerer.unsupported( "SC1090", memberName, - `declaring '${name}' on a stream subclass (${name === "_writev" ? "batched writes are" : "deferred construction is"} not lowered — writes deliver one chunk at a time)`, + `declaring '_writev' on a ${streamBase.def.name.slice(1)} subclass (batched writes are supported on Writable and Duplex subclasses)`, ); } const accepted = streamCtorShape(streamBase.def.name).accepted; @@ -8987,8 +8988,9 @@ export function superCallStmt( // The SYNTHESIZED constructor of a ctor-less stream subclass: // super() with default options (underscore methods still bind; a // construction passing options requires a declared constructor — - // lowerNew fences that). Zero options ⇒ exactly one init stmt. - return lowerStreamSuperCall( + // lowerNew fences that). Zero options ⇒ the init stmt, then the + // class's _writev/_construct follow-ups. + const stmts = lowerStreamSuperCall( lowerer, info, base, @@ -8996,7 +8998,8 @@ export function superCallStmt( thisLocal, loc, info.decl ?? info.ctor!, - )[0]!; + ); + return stmts.length === 1 ? stmts[0]! : { kind: "block", body: stmts, loc }; } lowerer.noteSuperConstructorEdge(`%${base.def.name}.constructor`); const call: IrStmt = { diff --git a/packages/compiler/src/frontend/lowering/lower-stream.ts b/packages/compiler/src/frontend/lowering/lower-stream.ts index 6540b3a99..20f156124 100644 --- a/packages/compiler/src/frontend/lowering/lower-stream.ts +++ b/packages/compiler/src/frontend/lowering/lower-stream.ts @@ -176,10 +176,25 @@ export const UNDERSCORE_METHODS: ReadonlyMap = new Map([ ["flush", "_flush"], ]); +/** The option callbacks applied after construction rather than through + * the constructor's callback slots (option name → method name): construct + * on every stream class, and writev on the writable classes whose + * machinery hands it batches (Writable and Duplex). */ +export const STREAM_EXTRA_CALLBACKS: ReadonlyMap = new Map([ + ["construct", "_construct"], + ["writev", "_writev"], +]); + +export function streamExtraCallbacks(cls: string): readonly string[] { + return cls === "%Writable" || cls === "%Duplex" ? ["construct", "writev"] : ["construct"]; +} + /** The completion callback's parameter position in each underscore * method that takes one (cbTuples' "fn" positions). */ export const STREAM_COMPLETION_CALLBACK_INDEX: ReadonlyMap = new Map([ ["_write", 2], + ["_writev", 1], + ["_construct", 0], ["_final", 0], ["_destroy", 1], ["_transform", 2], @@ -317,9 +332,11 @@ function lowerStreamCallback( // (`string | null | undefined`) converts from the encoding argument's // value. if ( - ((i === 0 && (chunkValues || widensInto(lowerer, fullTuple[0]!, funcType.params[0]!, loc))) || + ((((i === 0 && + (chunkValues || widensInto(lowerer, fullTuple[0]!, funcType.params[0]!, loc))) || i === 1) && - (which === "write" || which === "transform") && + (which === "write" || which === "transform")) || + (i === 0 && which === "writev")) && funcType.params[i]!.kind !== "func" && canConvertToDyn( funcType.params[i]!, @@ -429,6 +446,16 @@ interface StreamOptions { * right after construction. */ readableObject: boolean; writableObject: boolean; + /** decodeStrings: false (string writes stay strings), applied with + * stream.decodeStrings right after construction. */ + decodeStringsOff: boolean; + /** A writable side's non-utf8 `defaultEncoding` as written, applied + * with writable.writeEncoding. */ + writeEncoding: string | null; + /** The construct and writev callbacks (options or the class's + * _construct/_writev), applied with stream.setWritev and + * stream.setConstruct right after construction. */ + extras: { name: string; cb: IrExpr }[]; /** Present callbacks in canonical order with their flag bits. */ flags: number; cbs: IrExpr[]; @@ -470,14 +497,19 @@ function streamObjectModes( return out; } -/** Whether an options literal sets the writable side's object mode with a - * `true` literal (objectMode, or writableObjectMode on a duplex stream). - * Reports nothing: streamObjectModes fences the other value forms. */ +/** Whether an options literal makes the writable side hold values: its + * object mode set with a `true` literal (objectMode, or writableObjectMode + * on a duplex stream), or decodeStrings: false (strings stay strings). + * Reports nothing: the options parse fences the other value forms. */ function literalWritesValues(opts: ts.ObjectLiteralExpression, duplexish: boolean): boolean { for (const prop of opts.properties?.nodes ?? []) { if (!ts.isPropertyAssignment(prop)) continue; const name = ts.isIdentifier(prop.name) || ts.isStringLiteral(prop.name) ? prop.name.text : null; + if (name === "decodeStrings") { + if (prop.initializer.kind !== ts.SyntaxKind.TrueKeyword) return true; + continue; + } if (name !== "objectMode" && !(duplexish && name === "writableObjectMode")) continue; if (prop.initializer.kind !== ts.SyntaxKind.FalseKeyword) return true; } @@ -494,6 +526,8 @@ function cbTuples(lowerer: Lowerer): Record { read: [F64], write: [BYTES, STRING, "fn"], final: ["fn"], + construct: ["fn"], + writev: [DYN, "fn"], destroy: [errorOrNull(lowerer), "fn"], transform: [BYTES, STRING, "fn"], flush: ["fn"], @@ -734,9 +768,14 @@ function parseStreamOptions( signal: null, readableObject: false, writableObject: false, + decodeStringsOff: false, + writeEncoding: null, + extras: [], flags: 0, cbs: [], }; + const extraNames = streamExtraCallbacks(`%${ctorName}`); + const writableClass = ctorName !== "Readable"; const followed = followOptionsArg(lowerer, arg); const optsLit = followed === null ? undefined : followed; if (optsLit !== undefined && !ts.isObjectLiteralExpression(optsLit)) { @@ -756,6 +795,12 @@ function parseStreamOptions( ); out.readableObject = objectModes.readable; out.writableObject = objectModes.writable; + // the chunk an option callback receives can be another value than a + // Buffer (literalWritesValues), read before any callback lowers + const writesValues = + optsLit !== undefined && + ts.isObjectLiteralExpression(optsLit) && + literalWritesValues(optsLit, ctorName !== "Readable" && ctorName !== "Writable"); if (out.readableObject) out.hwmR = numLit(16, loc); if (out.writableObject) out.hwmW = numLit(16, loc); const tuples = cbTuples(lowerer); @@ -772,7 +817,7 @@ function parseStreamOptions( lowerer.unsupported("SC1090", prop, `computed or spread members in ${ctorName} options`); } // Callback members: shorthand methods and function-valued properties. - if (accepted.includes(name)) { + if (accepted.includes(name) || extraNames.includes(name)) { let fnNode: ts.MethodDeclaration | ts.FunctionExpression | ts.ArrowFunction | null = null; if (ts.isMethodDeclaration(prop)) fnNode = prop; else if ( @@ -823,7 +868,7 @@ function parseStreamOptions( }); present.set( name, - lowerStreamCallback(lowerer, name, fnNode, thisType, tuple, out.writableObject), + lowerStreamCallback(lowerer, name, fnNode, thisType, tuple, writesValues), ); } finally { if (pinned) lowerer.stageSourceTypes.delete(pinned); @@ -872,11 +917,17 @@ function parseStreamOptions( // read up front (streamObjectModes) break; case "decodeStrings": { + // WritableState's `options.decodeStrings === false`; a Readable + // ignores the option if (value.kind === ts.SyntaxKind.TrueKeyword) break; // Node's default + if (value.kind === ts.SyntaxKind.FalseKeyword) { + if (writableClass) out.decodeStringsOff = true; + break; + } lowerer.noLowering( - `${ctorName} with decodeStrings: false`, + `${ctorName} with a runtime 'decodeStrings' value`, prop, - "chunks always decode to Buffers here (Node's default)", + "write 'decodeStrings: true' or 'decodeStrings: false'", ); break; } @@ -911,15 +962,20 @@ function parseStreamOptions( out.badEncoding = t.value; break; } - if (t.isStringLiteralType() && ctorName === "Readable") { - out.pushEncoding = knownBufEncoding(t.value)!; + if (t.isStringLiteralType()) { + // the readable side decodes push(string) with it, and the + // writable side encodes write(string) with it (the name as + // written reaches _write when decodeStrings is false) + if (ctorName !== "Writable" && !out.readableObject) + out.pushEncoding = knownBufEncoding(t.value)!; + if (writableClass) out.writeEncoding = t.value; break; } } lowerer.noLowering( - `the ${ctorName} option 'defaultEncoding'`, + `the ${ctorName} option 'defaultEncoding' with a runtime value`, prop, - "utf8 (the default) is the supported write-side string encoding", + "write the encoding as a string literal", ); break; case "signal": { @@ -980,18 +1036,13 @@ function parseStreamOptions( ); break; } - case "construct": - lowerer.noLowering( - `the ${ctorName} option 'construct'`, - prop, - "deferred construction (_construct) is not lowered yet", - ); - break; case "writev": lowerer.noLowering( `the ${ctorName} option 'writev'`, prop, - "batched writes (_writev) are not lowered yet — writes deliver one chunk at a time", + ctorName === "Readable" + ? "a Readable has no writable side" + : "batched writes on a Transform would bypass its transform — writev is supported on Writable and Duplex", ); break; default: @@ -1008,6 +1059,10 @@ function parseStreamOptions( out.cbs.push(cb); } } + for (const name of extraNames) { + const cb = present.get(name) ?? fallbackCb?.(name) ?? null; + if (cb) out.extras.push({ name, cb }); + } return out; } @@ -1029,6 +1084,9 @@ function streamCtorArgs( signal: IrExpr | null; readableObject: boolean; writableObject: boolean; + decodeStringsOff: boolean; + writeEncoding: string | null; + extras: { name: string; cb: IrExpr }[]; } { const shape = streamCtorShape(cls); const o = parseStreamOptions( @@ -1060,9 +1118,72 @@ function streamCtorArgs( signal: o.signal, readableObject: o.readableObject, writableObject: o.writableObject, + decodeStringsOff: o.decodeStringsOff, + writeEncoding: o.writeEncoding, + extras: o.extras, }; } +/** The follow-ups a stream's options apply over the fresh stream `recv` + * after the chained ones: decodeStrings, the writable defaultEncoding, + * then writev and construct (Node's constructors schedule construct + * last). */ +function streamOptionFollowUps( + parsed: { + decodeStringsOff: boolean; + writeEncoding: string | null; + extras: { name: string; cb: IrExpr }[]; + }, + recv: () => IrExpr, + thisType: IrType, + loc: SrcLoc, + writevFn: IrLibFn = "stream.setWritev", +): IrStmt[] { + const out: IrStmt[] = []; + if (parsed.decodeStringsOff) { + out.push({ + kind: "exprStmt", + expr: { + kind: "libCall", + fn: "stream.decodeStrings", + args: [recv(), boolLit(false, loc)], + type: thisType, + loc, + }, + loc, + }); + } + if (parsed.writeEncoding !== null) { + out.push({ + kind: "exprStmt", + expr: { + kind: "libCall", + fn: "writable.writeEncoding", + args: [recv(), strLit(parsed.writeEncoding, loc)], + type: thisType, + loc, + }, + loc, + }); + } + for (const name of ["writev", "construct"]) { + const extra = parsed.extras.find((x) => x.name === name); + if (!extra) continue; + out.push({ + kind: "exprStmt", + expr: { + kind: "libCall", + fn: name === "writev" ? writevFn : "stream.setConstruct", + args: [recv(), extra.cb], + type: VOID, + loc, + }, + loc, + }); + } + return out; +} + /** `new Readable(opts?)` and friends — called from lowerNew once the * callee resolved to a builtin stream class. */ export function lowerStreamNew(lowerer: Lowerer, expr: ts.NewExpression, info: ClassInfo): IrExpr { @@ -1137,12 +1258,29 @@ export function lowerStreamNew(lowerer: Lowerer, expr: ts.NewExpression, info: C loc, }; } - if (parsed.signal === null) return built; - // { signal }: addAbortSignal over the fresh stream (answers it +1). + if (parsed.signal !== null) { + // { signal }: addAbortSignal over the fresh stream (answers it +1). + built = { + kind: "libCall", + fn: "stream.addAbortSignal", + args: [built, parsed.signal], + type: thisType, + loc, + }; + } + if (!parsed.decodeStringsOff && parsed.writeEncoding === null && parsed.extras.length === 0) + return built; + const fresh = lowerer.declareHiddenLocal("%stream", thisType); + const followUps = streamOptionFollowUps( + parsed, + () => varRef(fresh.id, thisType, loc), + thisType, + loc, + ); return { - kind: "libCall", - fn: "stream.addAbortSignal", - args: [built, parsed.signal], + kind: "seqExpr", + stmts: [{ kind: "varDecl", localId: fresh.id, init: built, loc }, ...followUps], + result: varRef(fresh.id, thisType, loc), type: thisType, loc, }; @@ -1236,10 +1374,12 @@ function streamMethodWrapper( } else if (!typeEquals(got.type, want)) { // A write or transform encoding of another type (`string | null | // undefined`) converts from the encoding argument's value: an - // object-mode write passes undefined, null or the name. + // object-mode write passes undefined, null or the name. A chunk of + // another type (`string | Buffer` under decodeStrings: false) + // converts from the chunk's value. if ( - i === 1 && - (which === "write" || which === "transform") && + (i === 1 || (i === 0 && (chunkValues || which === "writev"))) && + (which === "write" || which === "transform" || which === "writev") && got.type.kind !== "func" && canConvertToDyn( got.type, @@ -1262,8 +1402,7 @@ function streamMethodWrapper( // or throws a TypeError there; a byte-mode side's chunk is a Buffer. if ( i === 0 && - chunkValues && - (which === "write" || which === "transform") && + (which === "writev" || (chunkValues && (which === "write" || which === "transform"))) && got.type.kind !== "func" && canConvertToDyn( got.type, @@ -1380,7 +1519,7 @@ export function lowerStreamSuperCall( (!ts.isObjectLiteralExpression(optsNode) || literalWritesValues(optsNode, cls !== "%Readable" && cls !== "%Writable")); const fallback = (name: string): IrExpr | null => { - const methodName = UNDERSCORE_METHODS.get(name); + const methodName = UNDERSCORE_METHODS.get(name) ?? STREAM_EXTRA_CALLBACKS.get(name); return methodName ? streamMethodWrapper(lowerer, info, name, methodName, thisType, loc, blame, chunkValues) : null; @@ -1401,6 +1540,13 @@ export function lowerStreamSuperCall( cbs.push(cb); } } + // The class's _writev and _construct ride the same way: an options + // writev or construct (applied by initDyn) wins. + const extras: { name: string; cb: IrExpr }[] = []; + for (const name of streamExtraCallbacks(cls)) { + const cb = fallback(name); + if (cb) extras.push({ name, cb }); + } return [ { kind: "exprStmt", @@ -1413,6 +1559,13 @@ export function lowerStreamSuperCall( }, loc, }, + ...streamOptionFollowUps( + { decodeStringsOff: false, writeEncoding: null, extras }, + thisRef, + thisType, + loc, + "stream.setWritevDefault", + ), ]; } } @@ -1499,6 +1652,7 @@ export function lowerStreamSuperCall( loc, }); } + out.push(...streamOptionFollowUps(parsed, thisRef, thisType, loc)); return out; } @@ -1542,18 +1696,18 @@ export function lowerStreamUnderscoreAssign( const loc = locOf(expr); const cls = base.def.name; const clsName = cls.slice(1); - if (methodName === "_writev" || methodName === "_construct") { + if (methodName === "_construct") { + // Node's constructor reads _construct once; a later assignment has no + // effect there lowerer.noLowering( `assigning '${methodName}' on a ${clsName}`, expr.left, - methodName === "_writev" - ? "batched writes (_writev) are not lowered yet — writes deliver one chunk at a time" - : "deferred construction (_construct) is not lowered yet", + "declare _construct on the class or pass the construct option", ); } - const which = optionByMethod.get(methodName)!; + const which = methodName === "_writev" ? "writev" : optionByMethod.get(methodName)!; const { accepted } = streamCtorShape(cls); - if (!accepted.includes(which)) { + if (!accepted.includes(which) && !streamExtraCallbacks(cls).includes(which)) { lowerer.noLowering( `assigning '${methodName}' on a ${clsName}`, expr.left, @@ -1589,6 +1743,7 @@ export function lowerStreamUnderscoreAssign( destroy: "stream.setDestroy", transform: "stream.setTransform", flush: "stream.setFlush", + writev: "stream.setWritev", }; return { kind: "exprStmt", @@ -4616,18 +4771,20 @@ export function lowerStreamMethodCall( } if (member === "setDefaultEncoding") { - // The write-side default: utf8 IS the default — admit it; other - // encodings would change what write(string) means. + // The write-side default: lowercased and validated (ERR_UNKNOWN_ENCODING, + // thrown) at run time; answers the stream. requireSide(writableSide, "setDefaultEncoding"); - const t = args[0] ? lowerer.typeOf(args[0]) : null; - if (t?.isStringLiteralType() && (t.value === "utf8" || t.value === "utf-8")) { - return lowerer.lowerExpr(access.expression); - } - lowerer.noLowering( - "setDefaultEncoding", - call, - "utf8 (the default, as a compile-time literal) is the supported write-side string encoding", - ); + if (args.length > 1) + lowerer.noLowering(`setDefaultEncoding with ${args.length} arguments`, call); + const receiver = lowerer.lowerExpr(access.expression); + const enc = args[0] ? lowerer.lowerExprExpecting(args[0], DYN) : dynUndefinedExpr(loc); + return { + kind: "libCall", + fn: "writable.setDefaultEncoding", + args: [receiver, enc], + type: receiver.type, + loc, + }; } return null; diff --git a/packages/compiler/src/ir/builtin-effects.ts b/packages/compiler/src/ir/builtin-effects.ts index 16db45b6c..6ddee1597 100644 --- a/packages/compiler/src/ir/builtin-effects.ts +++ b/packages/compiler/src/ir/builtin-effects.ts @@ -379,6 +379,7 @@ export const MAY_THROW_LIB_FNS: ReadonlySet = new Set([ "writable.writeEnc", "writable.end", "writable.uncork", + "writable.setDefaultEncoding", "stream.destroy", "stream.destroyErr", "stream.iteratorClose", diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index fcdbe8694..e561c0dbf 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -3930,6 +3930,9 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ "stream.setDestroy": { argTypes: [null, null], result: VOID }, "stream.setTransform": { argTypes: [null, null], result: VOID }, "stream.setFlush": { argTypes: [null, null], result: VOID }, + "stream.setConstruct": { argTypes: [null, null], result: VOID }, + "stream.setWritev": { argTypes: [null, null], result: VOID }, + "stream.setWritevDefault": { argTypes: [null, null], result: VOID }, /** The dyn-options twins (a checked-dynamic options record — the JS * lane's `super(options)` forwarding and `new Readable(dynVar)`): the * option walk runs at RUNTIME with Node's reading rules. newDyn: @@ -4056,6 +4059,9 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ /** The objectMode option's sides (readable, writable), applied right * after construction; answers the receiver (+1, the chaining shape). */ "stream.objectMode": { argTypes: [null, BOOL, BOOL], result: VOID }, + "stream.decodeStrings": { argTypes: [null, BOOL], result: VOID }, + "writable.writeEncoding": { argTypes: [null, STRING], result: VOID }, + "writable.setDefaultEncoding": { argTypes: [null, DYN], result: VOID }, /** Whether a readable.nextChunkDyn step answered the end (the done * marker; an object-mode stream can yield undefined). */ "stream.chunkDone": { argTypes: [DYN], result: BOOL }, diff --git a/packages/compiler/src/ir/runtime-features.ts b/packages/compiler/src/ir/runtime-features.ts index 5b984f318..555f5d306 100644 --- a/packages/compiler/src/ir/runtime-features.ts +++ b/packages/compiler/src/ir/runtime-features.ts @@ -218,6 +218,7 @@ const STREAM_LIB_FNS: ReadonlySet = new Set([ "child.stdioStream", "stream.chunkDone", "stream.objectMode", + "stream.decodeStrings", "console.bind", "crypto.hashStream", "crypto.hashStreamPiped", diff --git a/packages/compiler/src/ir/validate.ts b/packages/compiler/src/ir/validate.ts index bb36c7007..68bc61b21 100644 --- a/packages/compiler/src/ir/validate.ts +++ b/packages/compiler/src/ir/validate.ts @@ -5494,6 +5494,10 @@ function validateFunction( e.fn === "stream.destroy" || e.fn === "stream.destroyErr" || e.fn === "stream.objectMode" || + e.fn === "stream.decodeStrings" || + e.fn === "stream.setConstruct" || + e.fn === "stream.setWritev" || + e.fn === "stream.setWritevDefault" || e.fn === "stream.iteratorClose" || e.fn === "stream.prop" || e.fn === "stream.errored" || @@ -5785,6 +5789,9 @@ function validateFunction( e.fn === "readable.setEncoding" || e.fn === "readable.pushEncoding" || e.fn === "stream.objectMode" || + e.fn === "stream.decodeStrings" || + e.fn === "writable.writeEncoding" || + e.fn === "writable.setDefaultEncoding" || e.fn === "stream.destroy" || e.fn === "stream.destroyErr" || e.fn === "stream.iteratorClose" diff --git a/packages/compiler/test/native-codegen-integration.test.ts b/packages/compiler/test/native-codegen-integration.test.ts index af8d9b724..35a747d60 100644 --- a/packages/compiler/test/native-codegen-integration.test.ts +++ b/packages/compiler/test/native-codegen-integration.test.ts @@ -378,7 +378,7 @@ done: expect(await symbols(helperObject, ["-u"])).toEqual(await symbols(clangObject, ["-u"])); expect(await symbols(helperObject, ["-gU"])).toEqual(await symbols(clangObject, ["-gU"])); expect(await symbols(helperObject, ["-gU"])).toEqual(["0000000000000000 T _main"]); - expect(await symbols(helperObject, ["-u"])).toContain("_scr_runtime_abi_v26"); + expect(await symbols(helperObject, ["-u"])).toContain("_scr_runtime_abi_v27"); const clangExe = join(dir, "clang-program"); const linkDriver = join(dir, "clang-driver.c"); @@ -712,7 +712,7 @@ done: .split("\n"); expect(undefinedSymbols).toContain("_sf_scale"); expect(undefinedSymbols).toContain("_sf_callback_mix"); - expect(undefinedSymbols).toContain("_scr_runtime_abi_v26"); + expect(undefinedSymbols).toContain("_scr_runtime_abi_v27"); }); test("helper and clang object paths retain the same outbound FFI C ABI", async () => { diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7246-stream-construct.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7246-stream-construct.ts.json new file mode 100644 index 000000000..f1dcb625f --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7246-stream-construct.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7246-stream-construct.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7247-stream-construct.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7247-stream-construct.mjs.json new file mode 100644 index 000000000..00c6c7f0c --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7247-stream-construct.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7247-stream-construct.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7248-stream-writev.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7248-stream-writev.ts.json new file mode 100644 index 000000000..0faf76b0f --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7248-stream-writev.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7248-stream-writev.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7249-stream-writev.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7249-stream-writev.cjs.json new file mode 100644 index 000000000..5374e215e --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7249-stream-writev.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7249-stream-writev.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7250-stream-decode-strings.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7250-stream-decode-strings.ts.json new file mode 100644 index 000000000..67d2b759b --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7250-stream-decode-strings.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7250-stream-decode-strings.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7251-stream-write-encodings.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7251-stream-write-encodings.mjs.json new file mode 100644 index 000000000..d4c01fab9 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7251-stream-write-encodings.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7251-stream-write-encodings.mjs" + ], + "diags": [] +} diff --git a/packages/runtime-darwin-arm64/scripts/build.mjs b/packages/runtime-darwin-arm64/scripts/build.mjs index 012b86013..631750cdf 100644 --- a/packages/runtime-darwin-arm64/scripts/build.mjs +++ b/packages/runtime-darwin-arm64/scripts/build.mjs @@ -340,7 +340,7 @@ async function build() { package: packageManifest.name, version: packageManifest.version, target: RUNTIME_PACK_MATRIX.target, - runtime_abi: { version: 26, marker: "scr_runtime_abi_v26" }, + runtime_abi: { version: 27, marker: "scr_runtime_abi_v27" }, compiler: { command: compiler, identity: compilerVersion, diff --git a/packages/runtime-pack-common/scripts/build.mjs b/packages/runtime-pack-common/scripts/build.mjs index 5b8f3563f..0e81033ad 100644 --- a/packages/runtime-pack-common/scripts/build.mjs +++ b/packages/runtime-pack-common/scripts/build.mjs @@ -437,7 +437,7 @@ async function build() { package: packageManifest.name, version: packageManifest.version, target: matrix.target, - runtime_abi: { version: 26, marker: "scr_runtime_abi_v26" }, + runtime_abi: { version: 27, marker: "scr_runtime_abi_v27" }, compiler: { command: compiler, identity: compilerVersion, diff --git a/packages/runtime/src/scr_console.c b/packages/runtime/src/scr_console.c index 24cd8093b..a3ac43cc0 100644 --- a/packages/runtime/src/scr_console.c +++ b/packages/runtime/src/scr_console.c @@ -148,7 +148,7 @@ static void scr_enable_utf8_console_output(void) { } #endif -void scr_runtime_abi_v26(void) {} +void scr_runtime_abi_v27(void) {} void scr_init(void) { #ifdef SCR_WORKERS diff --git a/packages/runtime/src/scr_core.h b/packages/runtime/src/scr_core.h index 791c600c7..fea020d16 100644 --- a/packages/runtime/src/scr_core.h +++ b/packages/runtime/src/scr_core.h @@ -81,7 +81,7 @@ void scr_init(void); /* Program objects emitted by the bundled LLVM helper reference this symbol. * Its versioned spelling makes a mismatched manual runtime link fail before * the program can start. */ -void scr_runtime_abi_v26(void); +void scr_runtime_abi_v27(void); void scr_runtime_workers_v8(void); /* ── the trap funnel (scr_console.c; scr_library.c under -DSCR_LIB) ────── diff --git a/packages/runtime/src/scr_dyn_builtin_stream.c b/packages/runtime/src/scr_dyn_builtin_stream.c index cdd815642..384e8d337 100644 --- a/packages/runtime/src/scr_dyn_builtin_stream.c +++ b/packages/runtime/src/scr_dyn_builtin_stream.c @@ -108,10 +108,6 @@ static ScrDyn *bcs_undefined(void) { return scr_dyn_retain(scr_dyn_undefined()); } -static ScrDyn *bcs_str(const char *text) { - return bcs_key(text); -} - static void bcs_unsupported(const char *message) { scr_throw_error_msg_code(SCR_ERR_ERROR, message, strlen(message), "SC2020"); } @@ -231,7 +227,7 @@ static ScrStream *bcs_need(ScrDyn *self, const char *reading) { /* ── completion callbacks ───────────────────────────────────────────── */ -enum { BCS_DONE_WRITE, BCS_DONE_FINAL, BCS_DONE_DESTROY, BCS_DONE_TRANSFORM, BCS_DONE_FLUSH }; +enum { BCS_DONE_WRITE, BCS_DONE_FINAL, BCS_DONE_DESTROY, BCS_DONE_TRANSFORM, BCS_DONE_FLUSH, BCS_DONE_CONSTRUCT }; static void bcs_run_destroy_cbs(ScrStream *s, ScrError *err) { BcsRec *rec = bcs_rec(s); @@ -266,6 +262,9 @@ static ScrDyn *bcs_done_call(ScrClosure *closure, ScrDyn *const *args, size_t ar case BCS_DONE_FINAL: scr_stream_final_done(s, err); break; + case BCS_DONE_CONSTRUCT: + scr_stream_construct_done(s, err); + break; case BCS_DONE_DESTROY: { ScrError *kept = err ? scr_error_retain(err) : NULL; scr_stream_destroy_done(s, err); @@ -401,6 +400,30 @@ static void bcs_plain_call(ScrClosure *cb, ScrStream *s, const char *method, int static bool bcs_is_function(ScrDyn *object, const char *name, bool *failed); +/* constructNT: `try { stream._construct(cb) } catch (err) { onConstruct(err) }`. */ +static void bcs_construct_inv(ScrClosure *cb, ScrStream *s) { + ScrDyn *object = scr_box_get_ref(cb->caps[0]); + ScrDyn *callback = bcs_done(s, BCS_DONE_CONSTRUCT); + ScrDyn *result = bcs_invoke(object, "_construct", &callback, 1); + scr_dyn_release(callback); + if (result) scr_dyn_release(result); + else if (scr_exc_pending()) scr_stream_construct_done(s, bcs_take_error()); + scr_dyn_release(object); +} + +/* clearBuffer's batch: this._writev(chunks, cb); a throw reaches the + * write() caller, as in Node. */ +static void bcs_writev_inv(ScrClosure *cb, ScrStream *s, ScrDyn *chunks) { + ScrDyn *object = scr_box_get_ref(cb->caps[0]); + ScrDyn *args[2]; + args[0] = scr_dyn_retain(chunks); + args[1] = bcs_done(s, BCS_DONE_WRITE); + ScrDyn *result = bcs_invoke(object, "_writev", args, 2); + for (int i = 0; i < 2; i++) scr_dyn_release(args[i]); + if (result) scr_dyn_release(result); + scr_dyn_release(object); +} + /* prefinish reads `stream._final` when the writable side finishes, so a * method assigned after the constructor runs; without one the finish goes * on as Node's does when `typeof stream._final !== 'function'`. */ @@ -875,14 +898,14 @@ static ScrDyn *bcs_construct(int kind, ScrDyn *self, ScrDyn *const *args, size_t } bool failed = false; bool writes = kind != BCS_READABLE; - if (writes && bcs_is_function(self, "_writev", &failed)) { - bcs_unsupported("_writev on a constructor-function stream is not supported yet"); - return NULL; - } - if (!failed && bcs_is_function(self, "_construct", &failed)) { - bcs_unsupported("_construct on a constructor-function stream is not supported yet"); + bool transformish = kind == BCS_TRANSFORM || kind == BCS_PASSTHROUGH; + /* Node's constructors read _writev and _construct as they run */ + bool has_writev = writes && bcs_is_function(self, "_writev", &failed); + if (has_writev && transformish) { + bcs_unsupported("_writev on a constructor-function Transform is not supported yet"); return NULL; } + bool has_construct = !failed && bcs_is_function(self, "_construct", &failed); if (failed) return NULL; ScrDyn *opts = bcs_arg(args, argc, 0); ScrStream *s = kind == BCS_READABLE ? scr_stream_new_readable_dyn(opts) @@ -936,6 +959,9 @@ static ScrDyn *bcs_construct(int kind, ScrDyn *self, ScrDyn *const *args, size_t } } scr_stream_set_destroy(s, bcs_hook(self, (void *)bcs_destroy_inv), bcs_destroy_inv); + if (has_writev) scr_stream_set_writev(s, bcs_hook(self, (void *)bcs_writev_inv), bcs_writev_inv); + if (has_construct) + scr_stream_set_construct(s, bcs_hook(self, (void *)bcs_construct_inv), bcs_construct_inv); bcs_hide(self, s); if (!failed) failed = !bcs_adopt_listeners(s, self); if (!failed && kind != BCS_WRITABLE) failed = !bcs_state_view(self, s, true); @@ -1020,17 +1046,6 @@ static ScrClosure *bcs_callback(ScrDyn *fn, ScrDyn *self, ScrStream *s) { return closure; } -/* Strings in another encoding than utf8 have no write path here. */ -static bool bcs_utf8_only(const ScrDyn *encoding) { - if (encoding->kind != SCR_DYN_STR) return true; - ScrStr *canonical = bcs_encoding(encoding); - if (!canonical) return false; - bool utf8 = bcs_is(canonical, "utf8"); - scr_str_release(canonical); - if (!utf8) bcs_unsupported("writing strings in an encoding other than utf8 to a constructor-function stream is not supported yet"); - return utf8; -} - static ScrDyn *bcs_write(ScrStream *s, ScrDyn *self, ScrDyn *const *args, size_t argc) { ScrDyn *chunk = bcs_arg(args, argc, 0); ScrDyn *encoding = bcs_arg(args, argc, 1); @@ -1039,14 +1054,9 @@ static ScrDyn *bcs_write(ScrStream *s, ScrDyn *self, ScrDyn *const *args, size_t cb = encoding; encoding = scr_stream_writes_values(s) ? scr_dyn_new_null() : scr_dyn_undefined(); } - bool below; - if (scr_stream_writes_values(s)) { - /* an object-mode side hands the encoding argument to _write */ - below = scr_stream_write_enc(s, chunk, encoding, bcs_callback(cb, self, s)); - } else { - if (chunk->kind == SCR_DYN_STR && !bcs_utf8_only(encoding)) return NULL; - below = scr_stream_write_dyn(s, chunk, bcs_callback(cb, self, s)); - } + /* an object-mode side hands the encoding argument to _write; a byte-mode + * side encodes a string with it (or the default encoding) */ + bool below = scr_stream_write_enc(s, chunk, encoding, bcs_callback(cb, self, s)); if (scr_exc_pending()) return NULL; return scr_dyn_new_bool(below); } @@ -1064,24 +1074,17 @@ static ScrDyn *bcs_end(ScrStream *s, ScrDyn *self, ScrDyn *const *args, size_t a encoding = scr_dyn_new_null(); } ScrStream *chained = NULL; - if (!bcs_nullish(chunk) && scr_stream_writes_values(s)) { - /* end(chunk) writes the value with the encoding argument */ + if (!bcs_nullish(chunk)) { + /* end(chunk) writes the chunk with the encoding argument first */ + if (scr_stream_prop(s, "writableObjectMode") == 0 && chunk->kind != SCR_DYN_STR && + chunk->kind != SCR_DYN_BYTES) { + scr_dyn_arg_type_fail("chunk", "of type string or an instance of Buffer, TypedArray, or DataView", chunk); + return NULL; + } scr_stream_write_enc(s, chunk, encoding, NULL); if (scr_exc_pending()) return NULL; - chained = scr_stream_end(s, NULL, NULL, bcs_callback(cb, self, s)); - } else if (chunk->kind == SCR_DYN_STR && !bcs_utf8_only(encoding)) { - return NULL; - } else if (bcs_nullish(chunk)) { - chained = scr_stream_end(s, NULL, NULL, bcs_callback(cb, self, s)); - } else if (chunk->kind == SCR_DYN_STR) { - chained = scr_stream_end(s, NULL, chunk->v.str, bcs_callback(cb, self, s)); - } else if (chunk->kind == SCR_DYN_BYTES) { - ScrBytes *bytes = scr_bytes_raw_view(chunk->v.bytes); - chained = scr_stream_end(s, bytes, NULL, bcs_callback(cb, self, s)); - scr_bytes_release(bytes); - } else { - scr_dyn_arg_type_fail("chunk", "of type string or an instance of Buffer, TypedArray, or DataView", chunk); } + chained = scr_stream_end(s, NULL, NULL, bcs_callback(cb, self, s)); if (chained) scr_stream_release(chained); if (scr_exc_pending()) return NULL; return scr_dyn_retain(self); @@ -1164,8 +1167,26 @@ static ScrDyn *bcs_method_call(ScrClosure *closure, ScrDyn *const *args, size_t switch (method) { case BCS_M_READ_DEFAULT: return bcs_not_implemented("_read()"); - case BCS_M_WRITE_DEFAULT: + case BCS_M_WRITE_DEFAULT: { + /* Writable.prototype._write: this._writev([{ chunk, encoding }], cb) */ + ScrDyn *self = scr_dyn_this_get(); + bool failed = false; + if (bcs_is_function(self, "_writev", &failed)) { + ScrDyn *chunks = scr_dyn_new_arr(); + ScrDyn *entry = scr_dyn_new_obj(); + scr_dyn_obj_set(entry, "chunk", 5, scr_dyn_retain(bcs_arg(args, argc, 0))); + scr_dyn_obj_set(entry, "encoding", 8, scr_dyn_retain(bcs_arg(args, argc, 1))); + scr_dyn_arr_push(chunks, entry); + ScrDyn *pass[2] = {chunks, bcs_arg(args, argc, 2)}; + ScrDyn *result = bcs_invoke(self, "_writev", pass, 2); + scr_dyn_release(chunks); + scr_dyn_release(self); + return result; + } + scr_dyn_release(self); + if (failed) return NULL; return bcs_not_implemented("_write()"); + } case BCS_M_TRANSFORM_DEFAULT: return bcs_not_implemented("_transform()"); case BCS_M_DESTROY_DEFAULT: @@ -1317,9 +1338,12 @@ static ScrDyn *bcs_method_call(ScrClosure *closure, ScrDyn *const *args, size_t scr_stream_uncork(s); result = bcs_undefined(); break; - case BCS_M_SET_DEFAULT_ENCODING: - if (bcs_utf8_only(first)) result = scr_dyn_retain(self); + case BCS_M_SET_DEFAULT_ENCODING: { + ScrStream *chained = scr_stream_set_default_encoding(s, first); + if (chained) scr_stream_release(chained); + if (!scr_exc_pending()) result = scr_dyn_retain(self); break; + } default: bcs_unsupported("Readable.prototype.wrap on a constructor-function stream is not supported yet"); break; diff --git a/packages/runtime/src/scr_stream.c b/packages/runtime/src/scr_stream.c index be5ccb910..691b6af8e 100644 --- a/packages/runtime/src/scr_stream.c +++ b/packages/runtime/src/scr_stream.c @@ -186,6 +186,19 @@ struct ScrStreamState { * and transform steps receive, are ScrDyn* values (in the ScrBytes* * slots), each counting 1 toward length. */ bool object_mode; + /* The queue holds ScrDyn* values: on an object-mode side, and with + * decodeStrings: false, where strings stay strings (their UTF-16 + * length counts) and Buffers are buffer values. Each entry's encoding + * argument rides in ScrWEntry.enc. */ + bool values, decode_strings_off; + /* The defaultEncoding option or setDefaultEncoding(), as given; NULL + * is utf8. What write(string) encodes with, and the encoding _write + * receives for a string when decodeStrings is false. */ + ScrStr *default_enc; + /* _writev: more than one buffered write reaches it as one batch, and a + * stream without _write sends each write through it. */ + ScrClosure *writev_cb; + ScrStreamWritevInv writev_inv; bool ending, finished, prefinished, final_called, finish_scheduled; bool final_pending; /* _final called, its callback not yet run */ ScrClosure *inflight_cb; /* owned or NULL */ @@ -252,6 +265,7 @@ struct ScrStreamState { * path and _destroy wait. A destroy() in that window marks the stream * destroyed and runs its _destroy once construction completes. */ bool constructing, destroy_deferred; + bool construct_finished; /* onConstruct ran: a second callback is ERR_MULTIPLE_CALLBACK */ ScrError *destroy_err; /* owned: the deferred destroy's error */ ScrClosure *construct_cb; ScrStreamPlainInv construct_inv; @@ -340,18 +354,25 @@ static ScrDyn *scr_stream_entry_box(void *entry, ScrEntryKind kind) { return d; } -/* Writable-queue chunks: Buffers, or dyn values on an object-mode side. */ +/* Writable-queue chunks: Buffers, or dyn values (w.values). A value + * counts 1 on an object-mode side, else its length (a string's UTF-16 + * length, as Node's chunk.length). */ static size_t scr_stream_wchunk_len(const ScrStreamState *st, ScrBytes *chunk) { - return st->w.object_mode ? 1 : chunk->len; + if (st->w.object_mode) return 1; + if (!st->w.values) return chunk->len; + ScrDyn *v = (ScrDyn *)chunk; + if (v->kind == SCR_DYN_STR) return (size_t)scr_str_utf16_len(v->v.str); + if (v->kind == SCR_DYN_BYTES) return v->v.bytes->len; + return 1; } static ScrBytes *scr_stream_wchunk_retain(const ScrStreamState *st, ScrBytes *chunk) { - if (st->w.object_mode) return (ScrBytes *)scr_dyn_retain((ScrDyn *)chunk); + if (st->w.values) return (ScrBytes *)scr_dyn_retain((ScrDyn *)chunk); return scr_bytes_retain(chunk); } static void scr_stream_wchunk_release(const ScrStreamState *st, ScrBytes *chunk) { - if (st->w.object_mode) scr_dyn_release((ScrDyn *)chunk); + if (st->w.values) scr_dyn_release((ScrDyn *)chunk); else scr_bytes_release(chunk); } @@ -377,6 +398,7 @@ static void scr_stream_state_drop(ScrStreamState *st, bool gc) { if (st->destroy_err) scr_error_release(st->destroy_err); if (!gc) { if (st->construct_cb) scr_closure_release(st->construct_cb); + if (st->w.writev_cb) scr_closure_release(st->w.writev_cb); if (st->r.read_cb) scr_closure_release(st->r.read_cb); if (st->w.write_cb) scr_closure_release(st->w.write_cb); if (st->w.final_cb) scr_closure_release(st->w.final_cb); @@ -390,13 +412,14 @@ static void scr_stream_state_drop(ScrStreamState *st, bool gc) { for (size_t i = 0; i < st->drain_srcs.n; i++) scr_stream_release(st->drain_srcs.src[i]); } for (size_t i = st->w.head; i < st->w.n; i++) { - bool traced = st->w.object_mode && scr_stream_dyn_entry_traced((ScrDyn *)st->w.q[i].chunk); + bool traced = st->w.values && scr_stream_dyn_entry_traced((ScrDyn *)st->w.q[i].chunk); if (!gc || !traced) scr_stream_wchunk_release(st, st->w.q[i].chunk); if (!gc && st->w.q[i].cb) scr_closure_release(st->w.q[i].cb); ScrDyn *enc = st->w.q[i].enc; if (enc && (!gc || !scr_stream_dyn_entry_traced(enc))) scr_dyn_release(enc); } free(st->w.q); + if (st->w.default_enc) scr_str_release(st->w.default_enc); free(st->w.end_cbs); free(st->fin.cb); free(st->fin.inv); @@ -426,6 +449,7 @@ void scr_stream_st_trace(ScrStreamState *st, ScrTraceVisit visit, void *ctx) { if (st->construct_cb) visit(st->construct_cb, ctx); if (st->w.write_cb) visit(st->w.write_cb, ctx); if (st->w.final_cb) visit(st->w.final_cb, ctx); + if (st->w.writev_cb) visit(st->w.writev_cb, ctx); if (st->w.inflight_cb) visit(st->w.inflight_cb, ctx); if (st->transform_cb) visit(st->transform_cb, ctx); if (st->flush_cb) visit(st->flush_cb, ctx); @@ -434,7 +458,7 @@ void scr_stream_st_trace(ScrStreamState *st, ScrTraceVisit visit, void *ctx) { for (size_t i = 0; i < st->fin.n; i++) visit(st->fin.cb[i], ctx); for (size_t i = st->w.head; i < st->w.n; i++) { if (st->w.q[i].cb) visit(st->w.q[i].cb, ctx); - if (st->w.object_mode && scr_stream_dyn_entry_traced((ScrDyn *)st->w.q[i].chunk)) + if (st->w.values && scr_stream_dyn_entry_traced((ScrDyn *)st->w.q[i].chunk)) visit(st->w.q[i].chunk, ctx); if (st->w.q[i].enc && scr_stream_dyn_entry_traced(st->w.q[i].enc)) visit(st->w.q[i].enc, ctx); } @@ -1032,23 +1056,10 @@ static void scr_stream_resume_kick(ScrStream *s) { static ScrError *scr_stream_dyn_err(ScrDyn *const *args, size_t argc) { if (argc < 1) return NULL; const ScrDyn *a = args[0]; - if (a->kind == SCR_DYN_UNDEF || a->kind == SCR_DYN_NULL) return NULL; - if (a->kind == SCR_DYN_STR) { - ScrError *e = scr_error_new(SCR_ERR_ERROR, a->v.str); - return e; - } - if (a->kind == SCR_DYN_OBJ) { - ScrDyn *m = scr_dyn_obj_get((ScrDyn *)a, "message", 7); - ScrStr *msg = m && m->kind == SCR_DYN_STR ? scr_str_retain(m->v.str) : scr_str_new("", 0); - ScrError *e = scr_error_new(SCR_ERR_ERROR, msg); - scr_str_release(msg); - return e; - } - /* any other truthy value: Node would wrap it — approximate with "" */ - ScrStr *empty = scr_str_new("", 0); - ScrError *e = scr_error_new(SCR_ERR_ERROR, empty); - scr_str_release(empty); - return e; + if (!scr_dyn_truthy(a)) return NULL; + if (a->kind == SCR_DYN_STR) return scr_error_new(SCR_ERR_ERROR, a->v.str); + /* the error value itself (the same object reaches 'error') */ + return scr_error_from_dyn(a); } static ScrStream *scr_stream_dyn_recv(ScrClosure *clo) { @@ -1071,6 +1082,13 @@ ScrDyn *scr_stream_done_dyn_f(ScrClosure *clo, ScrDyn *const *args, size_t argc) return scr_stream_done_dyn_ret(); } +ScrDyn *scr_stream_done_dyn_c(ScrClosure *clo, ScrDyn *const *args, size_t argc) { + ScrStream *s = scr_stream_dyn_recv(clo); + scr_stream_construct_done(s, scr_stream_dyn_err(args, argc)); + scr_stream_release(s); + return scr_stream_done_dyn_ret(); +} + ScrDyn *scr_stream_done_dyn_d(ScrClosure *clo, ScrDyn *const *args, size_t argc) { ScrStream *s = scr_stream_dyn_recv(clo); scr_stream_destroy_done(s, scr_stream_dyn_err(args, argc)); @@ -1865,6 +1883,8 @@ static void scr_stream_wq_push(ScrStreamState *st, ScrBytes *chunk /*moves*/, Sc static void scr_stream_transform_callback(ScrStream *s); static void scr_stream_do_write_inner(ScrStream *s, ScrBytes *chunk /*moves*/, ScrClosure *cb /*moves*/); +static ScrDyn *scr_stream_wentry_value(const ScrStreamState *st, ScrBytes *chunk); +static ScrDyn *scr_stream_wentry_encoding(const ScrStreamState *st, ScrDyn *enc); /* One write reaching _write/_transform; the callback reads the write's * encoding argument through w.cur_enc while it runs. */ @@ -1891,7 +1911,7 @@ static void scr_stream_do_write_inner(ScrStream *s, ScrBytes *chunk /*moves*/, S if (st->is_transform || st->passthrough) st->xf_length = st->r.length; if (st->passthrough && st->transform_cb == NULL) { /* identity: push through, complete synchronously */ - if (st->w.object_mode) scr_stream_push_dyn(s, (ScrDyn *)chunk); + if (st->w.values) scr_stream_push_dyn(s, (ScrDyn *)chunk); else scr_stream_add_chunk(s, chunk, false); scr_stream_wchunk_release(st, chunk); if (!scr_exc_pending()) scr_stream_transform_callback(s); @@ -1912,6 +1932,19 @@ static void scr_stream_do_write_inner(ScrStream *s, ScrBytes *chunk /*moves*/, S st->w.wsync = false; return; } + if (st->w.write_cb == NULL && st->w.writev_cb) { + /* Writable.prototype._write: this._writev([{ chunk, encoding }], cb) */ + ScrDyn *chunks = scr_dyn_new_arr(); + ScrDyn *entry = scr_dyn_new_obj(); + scr_dyn_obj_set(entry, "chunk", 5, scr_stream_wentry_value(st, chunk)); + scr_dyn_obj_set(entry, "encoding", 8, scr_stream_wentry_encoding(st, st->w.cur_enc)); + scr_dyn_arr_push(chunks, entry); + scr_stream_wchunk_release(st, chunk); + st->w.writev_inv(st->w.writev_cb, s, chunks); + scr_dyn_release(chunks); + st->w.wsync = false; + return; + } if (st->w.write_cb == NULL) { scr_stream_wchunk_release(st, chunk); ScrError *e = scr_stream_mkerr("ERR_METHOD_NOT_IMPLEMENTED", "The _write() method is not implemented"); @@ -2031,11 +2064,107 @@ static SCR_COLD void scr_stream_writev_buffer(ScrStream *s) { scr_stream_do_write(s, all, cbs, NULL); } +/* A write's chunk and encoding as _writev receives them (+1 each). */ +static ScrDyn *scr_stream_wentry_value(const ScrStreamState *st, ScrBytes *chunk) { + if (st->w.values) return scr_dyn_retain((ScrDyn *)chunk); + return chunk->is_buffer ? scr_dyn_new_buffer(chunk) : scr_dyn_new_bytes(chunk); +} + +static ScrDyn *scr_stream_wentry_encoding(const ScrStreamState *st, ScrDyn *enc) { + if (!st->w.values) { + ScrStr *buffer = scr_str_new("buffer", 6); + ScrDyn *d = scr_dyn_new_str(buffer); + scr_str_release(buffer); + return d; + } + return scr_dyn_retain(enc ? enc : scr_dyn_undefined()); +} + +/* A batch entry's callback: the write's callback, or Node's nop. */ +static ScrDyn *scr_stream_writev_entry_cb_call(ScrClosure *clo, ScrDyn *const *args, size_t argc) { + (void)args; + (void)argc; + if (clo->ncaps > 0) { + ScrClosure *cb = scr_box_get_ref(clo->caps[0]); + ((void (*)(ScrClosure *))cb->fn)(cb); + scr_closure_release(cb); + } + return scr_dyn_retain(scr_dyn_undefined()); +} + +static ScrDyn *scr_stream_writev_entry_cb(ScrClosure *cb /*borrowed or NULL*/) { + ScrClosure *c = scr_closure_new((void *)&scr_stream_writev_entry_cb_call, cb ? 1 : 0); + if (cb) { + c->caps[0] = scr_box_new_obj(&scr_closure_retain_v, &scr_closure_release_v, &scr_closure_trace_v); + scr_box_set_ref(c->caps[0], scr_closure_retain(cb)); + } + return scr_dyn_new_func(c, &scr_stream_writev_entry_cb_call, 1, "(error)", cb ? "" : "nop"); +} + +/* Calls _writev with chunks as the in-flight write of len, whose + * completion runs cb (moves; NULL when every write had none). */ +static void scr_stream_writev_call(ScrStream *s, ScrDyn *chunks, size_t len, ScrClosure *cb) { + ScrStreamState *st = s->st; + st->w.writing = true; + st->w.wsync = true; + st->w.inflight_len = len; + st->w.inflight_cb = cb; + st->w.writev_inv(st->w.writev_cb, s, chunks); + st->w.wsync = false; +} + +/* Node's clearBuffer with _writev and more than one buffered write: the + * writes go out as one [{ chunk, encoding, callback }] batch (its + * allBuffers says whether every encoding is 'buffer'), and the batch's + * completion runs the writes' callbacks in order. */ +static void scr_stream_writev_batch(ScrStream *s) { + ScrStreamState *st = s->st; + size_t total = 0, ncbs = 0; + for (size_t i = st->w.head; i < st->w.n; i++) + if (st->w.q[i].cb) ncbs++; + ScrClosure *cbs = ncbs > 0 ? scr_closure_new((void *)&scr_stream_writev_cbs, ncbs) : NULL; + ScrDyn *chunks = scr_dyn_new_arr(); + bool all_buffers = true; + size_t k = 0; + for (size_t i = st->w.head; i < st->w.n; i++) { + ScrWEntry e = st->w.q[i]; + total += scr_stream_wchunk_len(st, e.chunk); + ScrDyn *entry = scr_dyn_new_obj(); + ScrDyn *encoding = scr_stream_wentry_encoding(st, e.enc); + if (!(encoding->kind == SCR_DYN_STR && encoding->v.str->len == 6 && + memcmp(encoding->v.str->data, "buffer", 6) == 0)) + all_buffers = false; + scr_dyn_obj_set(entry, "chunk", 5, scr_stream_wentry_value(st, e.chunk)); + scr_dyn_obj_set(entry, "encoding", 8, encoding); + scr_dyn_obj_set(entry, "callback", 8, scr_stream_writev_entry_cb(e.cb)); + scr_dyn_arr_push(chunks, entry); + scr_stream_wchunk_release(st, e.chunk); + if (e.enc) scr_dyn_release(e.enc); + if (e.cb) { + cbs->caps[k] = scr_box_new_obj(&scr_closure_retain_v, &scr_closure_release_v, + &scr_closure_trace_v); + scr_box_set_ref(cbs->caps[k], e.cb); /* moves */ + k++; + } + } + scr_dyn_arr_named_put(chunks, "allBuffers", 10, scr_dyn_new_bool(all_buffers)); + st->w.head = 0; + st->w.n = 0; + scr_stream_writev_call(s, chunks, total, cbs); + scr_dyn_release(chunks); +} + /* The buffered writes, one at a time (Node's clearBuffer without _writev). */ static void scr_stream_clear_buffer_each(ScrStream *s) { ScrStreamState *st = s->st; if (st->w.buffer_processing) return; st->w.buffer_processing = true; + if (st->w.writev_cb && !st->w.writing && st->w.n - st->w.head > 1 && st->w.corked == 0 && + !st->destroyed && !st->constructing && !scr_exc_pending()) { + scr_stream_writev_batch(s); + st->w.buffer_processing = false; + return; + } while (!st->w.writing && st->w.head < st->w.n && st->w.corked == 0 && !st->destroyed && !scr_exc_pending()) { ScrWEntry e = st->w.q[st->w.head++]; @@ -2528,6 +2657,27 @@ static void scr_stream_dynopt_transform_inv(ScrClosure *cb, ScrStream *s, ScrByt if (r != NULL) scr_dyn_release(r); } +static void scr_stream_dynopt_construct_inv(ScrClosure *cb, ScrStream *s) { + ScrDyn *fn = scr_stream_dynopt_target(cb); + ScrDyn *a = scr_stream_dynopt_done(s, &scr_stream_done_dyn_c, 1, "(error)"); + ScrDyn *r = scr_dyn_call(fn, &a, 1, "construct"); + scr_dyn_release(a); + scr_dyn_release(fn); + if (r != NULL) scr_dyn_release(r); +} + +static void scr_stream_dynopt_writev_inv(ScrClosure *cb, ScrStream *s, ScrDyn *chunks) { + ScrDyn *fn = scr_stream_dynopt_target(cb); + ScrDyn *args[2]; + args[0] = scr_dyn_retain(chunks); + args[1] = scr_stream_dynopt_done(s, &scr_stream_done_dyn_w, 1, "(error)"); + ScrDyn *r = scr_dyn_call(fn, args, 2, "writev"); + scr_dyn_release(args[0]); + scr_dyn_release(args[1]); + scr_dyn_release(fn); + if (r != NULL) scr_dyn_release(r); +} + static void scr_stream_dynopt_final_inv(ScrClosure *cb, ScrStream *s) { ScrDyn *fn = scr_stream_dynopt_target(cb); ScrDyn *a = scr_stream_dynopt_done(s, &scr_stream_done_dyn_f, 1, "(error)"); @@ -2570,9 +2720,13 @@ typedef struct { bool auto_destroy, emit_close, allow_half_open; bool readable_side, writable_side; bool readable_object, writable_object; /* objectMode and the sided keys */ + bool decode_strings_off; /* decodeStrings: false */ ScrStr *enc; /* owned canonical name, or NULL */ + /* defaultEncoding other than utf8: as given (borrowed) and canonical + * (owned), or NULL */ + ScrStr *default_enc, *default_canonical; /* BORROWED callable fields (owned by the options record). */ - ScrDyn *read, *write, *final_fn, *destroy, *transform, *flush; + ScrDyn *read, *write, *final_fn, *destroy, *transform, *flush, *construct, *writev; } ScrStreamDynOpts; /* Node's `=== false` comparisons (autoDestroy/emitClose/allowHalfOpen and @@ -2647,7 +2801,10 @@ static bool scr_stream_parse_dyn_opts(const ScrDyn *opts, char base, ScrStreamDy out->writable_side = true; out->enc = NULL; out->readable_object = out->writable_object = false; + out->decode_strings_off = false; + out->default_enc = out->default_canonical = NULL; out->read = out->write = out->final_fn = out->destroy = out->transform = out->flush = NULL; + out->construct = out->writev = NULL; if (opts == NULL || opts->kind != SCR_DYN_OBJ) return true; /* Node's `if (options)` */ const bool duplexish = base == 'd' || base == 't' || base == 'p'; const bool transformish = base == 't' || base == 'p'; @@ -2658,10 +2815,9 @@ static bool scr_stream_parse_dyn_opts(const ScrDyn *opts, char base, ScrStreamDy out->readable_object = object || (duplexish && scr_stream_dynopt_truthy(opts, "readableObjectMode", 18)); out->writable_object = object || (duplexish && scr_stream_dynopt_truthy(opts, "writableObjectMode", 18)); } - if (scr_stream_dynopt_false(scr_dyn_obj_get((ScrDyn *)opts, "decodeStrings", 13))) { - static const char m[] = "decodeStrings: false is not supported yet (chunks always decode to Buffers)"; - return scr_stream_dynopt_unsupported(m, sizeof m - 1); - } + /* WritableState: decodeStrings === false keeps string writes */ + out->decode_strings_off = + base != 'r' && scr_stream_dynopt_false(scr_dyn_obj_get((ScrDyn *)opts, "decodeStrings", 13)); { ScrDyn *v = scr_dyn_obj_get((ScrDyn *)opts, "signal", 6); if (v != NULL && v->kind != SCR_DYN_NULL && v->kind != SCR_DYN_UNDEF) { @@ -2669,21 +2825,22 @@ static bool scr_stream_parse_dyn_opts(const ScrDyn *opts, char base, ScrStreamDy return scr_stream_dynopt_unsupported(m, sizeof m - 1); } } - if (scr_stream_dynopt_fn(opts, "construct", 9) != NULL) { - static const char m[] = "the stream option 'construct' is not supported yet (deferred construction has no lowering)"; - return scr_stream_dynopt_unsupported(m, sizeof m - 1); - } - if (scr_stream_dynopt_fn(opts, "writev", 6) != NULL) { - static const char m[] = "the stream option 'writev' is not supported yet (writes deliver one chunk at a time)"; + out->construct = scr_stream_dynopt_fn(opts, "construct", 9); + if (base != 'r') out->writev = scr_stream_dynopt_fn(opts, "writev", 6); + if (transformish && out->writev != NULL) { + static const char m[] = "the stream option 'writev' on a Transform is not supported yet (batches would bypass the transform)"; return scr_stream_dynopt_unsupported(m, sizeof m - 1); } { + /* a known encoding or ERR_UNKNOWN_ENCODING (the states' constructors) */ ScrDyn *v = scr_dyn_obj_get((ScrDyn *)opts, "defaultEncoding", 15); if (v != NULL && v->kind == SCR_DYN_STR && !(v->v.str->len == 4 && memcmp(v->v.str->data, "utf8", 4) == 0) && !(v->v.str->len == 5 && memcmp(v->v.str->data, "utf-8", 5) == 0)) { - static const char m[] = "the stream option 'defaultEncoding' is not supported yet (utf8, the default, is the write-side string encoding)"; - return scr_stream_dynopt_unsupported(m, sizeof m - 1); + ScrStr *canonical = scr_bytes_normalize_encoding(v->v.str); + if (canonical == NULL) return false; + out->default_enc = v->v.str; + out->default_canonical = canonical; } } if (transformish) { @@ -2752,8 +2909,33 @@ static ScrClosure *scr_stream_dynopt_pick(ScrDyn *opt, void *dyn_inv, ScrClosure return fb; } -static void scr_stream_dynopt_apply_object(ScrStream *s, const ScrStreamDynOpts *o) { - if (s != NULL && s->st != NULL) scr_stream_set_object_mode(s, o->readable_object, o->writable_object); +static void scr_stream_dynopt_construct_inv(ScrClosure *cb, ScrStream *s); +static void scr_stream_dynopt_writev_inv(ScrClosure *cb, ScrStream *s, ScrDyn *chunks); + +/* The options a dyn constructor applies over the fresh stream: the object + * modes, decodeStrings, defaultEncoding (the push decoding of a readable + * side and the writes of a writable side), writev, and construct, which + * Node's constructor schedules last. Releases the owned canonical name. */ +static void scr_stream_dynopt_apply_object(ScrStream *s, ScrStreamDynOpts *o) { + if (s != NULL && s->st != NULL) { + scr_stream_set_object_mode(s, o->readable_object, o->writable_object); + if (o->decode_strings_off) scr_stream_set_decode_strings(s, false); + if (o->default_enc) { + scr_stream_set_write_encoding(s, o->default_enc); + if (s->st->has_r && !s->st->r.object_entries) + scr_stream_release(scr_stream_set_push_encoding(s, o->default_canonical)); + } + if (o->writev && s->st->has_w) + scr_stream_set_writev(s, scr_stream_dynopt_clo(o->writev, (void *)&scr_stream_dynopt_writev_inv), + &scr_stream_dynopt_writev_inv); + if (o->construct) + scr_stream_set_construct(s, scr_stream_dynopt_clo(o->construct, (void *)&scr_stream_dynopt_construct_inv), + &scr_stream_dynopt_construct_inv); + } + if (o->default_canonical) { + scr_str_release(o->default_canonical); + o->default_canonical = NULL; + } } static void scr_stream_dynopt_apply_enc(ScrStream *s, ScrStr *enc) { @@ -5542,7 +5724,69 @@ void scr_stream_set_object_mode(ScrStream *s, bool readable, bool writable) { st->r.object_entries = true; st->r.object_mode_opt = true; } - if (writable && st->has_w) st->w.object_mode = true; + if (writable && st->has_w) st->w.object_mode = st->w.values = true; +} + +/* decodeStrings: false — string writes stay strings (scr_stream_write_string). */ +void scr_stream_set_decode_strings(ScrStream *s, bool decode) { + ScrStreamState *st = s->st; + if (decode || !st->has_w) return; + st->w.decode_strings_off = true; + st->w.values = true; +} + +ScrStream *scr_stream_decode_strings(ScrStream *s, bool decode) { + scr_stream_set_decode_strings(s, decode); + return scr_stream_retain(s); +} + +static void scr_stream_store_default_enc(ScrStreamState *st, ScrStr *enc /*moves*/) { + if (st->w.default_enc) scr_str_release(st->w.default_enc); + st->w.default_enc = NULL; + if ((enc->len == 4 && memcmp(enc->data, "utf8", 4) == 0) || + (enc->len == 5 && memcmp(enc->data, "utf-8", 5) == 0)) { + scr_str_release(enc); /* Node's kDefaultUTF8Encoding */ + return; + } + st->w.default_enc = enc; +} + +/* The defaultEncoding option on the writable side, a known encoding as + * given (borrowed; validated by the caller). */ +void scr_stream_set_write_encoding(ScrStream *s, ScrStr *enc) { + if (s->st->has_w) scr_stream_store_default_enc(s->st, scr_str_retain(enc)); +} + +ScrStream *scr_stream_write_encoding(ScrStream *s, ScrStr *enc) { + scr_stream_set_write_encoding(s, enc); + return scr_stream_retain(s); +} + +/* setDefaultEncoding(encoding): the name lowercased, a known encoding or + * ERR_UNKNOWN_ENCODING (thrown); answers the stream (+1). */ +ScrStream *scr_stream_set_default_encoding(ScrStream *s, ScrDyn *enc) { + ScrStr *raw = enc->kind == SCR_DYN_STR ? scr_str_retain(enc->v.str) : scr_dyn_string_coerce_js(enc); + ScrStr *lower = raw; + if (enc->kind == SCR_DYN_STR) { + char *buf = malloc(raw->len ? raw->len : 1); + if (!buf) scr_stream_oom(); + for (size_t i = 0; i < raw->len; i++) { + char c = raw->data[i]; + buf[i] = c >= 'A' && c <= 'Z' ? (char)(c - 'A' + 'a') : c; + } + lower = scr_str_new(buf, raw->len); + free(buf); + scr_str_release(raw); + } + ScrStr *canonical = scr_bytes_normalize_encoding(lower); + if (canonical == NULL) { + scr_str_release(lower); + return scr_stream_retain(s); + } + scr_str_release(canonical); + if (s->st->has_w) scr_stream_store_default_enc(s->st, lower); + else scr_str_release(lower); + return scr_stream_retain(s); } /* stream.objectMode: scr_stream_set_object_mode, answering the receiver @@ -5556,7 +5800,7 @@ ScrStream *scr_stream_object_mode(ScrStream *s, bool readable, bool writable) { * object-mode writable side): the compiler's chunk thunks read the chunk * by this. */ bool scr_stream_writes_values(ScrStream *s) { - return s->st->has_w && s->st->w.object_mode; + return s->st->has_w && s->st->w.values; } /* emit('data', value) with a checked-dynamic chunk: the 'data' slots by @@ -6050,6 +6294,60 @@ static bool scr_stream_write_value(ScrStream *s, ScrDyn *v, ScrClosure *cb) { return scr_stream_write_chunk_enc(s, (ScrBytes *)v, cb, scr_stream_cb_enc(cb)); } +/* A Buffer write on a byte-mode side: with decodeStrings: false the + * queue holds values, so the Buffer is boxed with its 'buffer' encoding. + * Borrows chunk. */ +static bool scr_stream_write_buffer(ScrStream *s, ScrBytes *chunk, ScrClosure *cb) { + ScrStreamState *st = s->st; + if (!st->has_w || !st->w.values) return scr_stream_write_chunk(s, chunk, cb); + ScrDyn *v = chunk->is_buffer ? scr_dyn_new_buffer(chunk) : scr_dyn_new_bytes(chunk); + ScrStr *buffer = scr_str_new("buffer", 6); + ScrDyn *enc = scr_dyn_new_str(buffer); + scr_str_release(buffer); + bool ret = scr_stream_write_chunk_enc(s, (ScrBytes *)v, cb, enc); + scr_dyn_release(enc); + scr_dyn_release(v); + return ret; +} + +/* A string write on a byte-mode side, with the encoding argument as + * given (borrowed; NULL takes the default encoding): decodeStrings + * encodes it (Buffer.from(chunk, encoding)), and with decodeStrings: + * false it stays a string that _write receives with that encoding. */ +static bool scr_stream_write_string(ScrStream *s, ScrStr *str, ScrStr *name, ScrClosure *cb) { + ScrStreamState *st = s->st; + ScrStr *given = name ? scr_str_retain(name) : st->has_w && st->w.default_enc ? scr_str_retain(st->w.default_enc) : NULL; + if (st->has_w && st->w.decode_strings_off) { + ScrStr *e = given ? given : scr_str_new("utf8", 4); + ScrDyn *v = scr_dyn_new_str(str); + ScrDyn *enc = scr_dyn_new_str(e); + scr_str_release(e); + bool ret = scr_stream_write_chunk_enc(s, (ScrBytes *)v, cb, enc); + scr_dyn_release(enc); + scr_dyn_release(v); + return ret; + } + ScrBytes *b; + if (given) { + ScrStr *canonical = scr_bytes_normalize_encoding(given); + scr_str_release(given); + if (canonical == NULL) { + if (cb) scr_closure_release(cb); + return false; + } + b = scr_bytes_from_str(str, canonical); + scr_str_release(canonical); + } else { + b = scr_bytes_new(SCR_BYTES_U8, (double)str->len); + memcpy(b->data, str->data, str->len); + if (!str->ascii) scr_wtf8_to_utf8((char *)b->data, str->len); + } + scr_stream_buffer(b); + bool ret = scr_stream_write_buffer(s, b, cb); + scr_bytes_release(b); + return ret; +} + bool scr_stream_write(ScrStream *s, ScrBytes *chunk, ScrClosure *cb) { if (s->st->has_w && s->st->w.object_mode) { ScrDyn *v = chunk->is_buffer ? scr_dyn_new_buffer(chunk) : scr_dyn_new_bytes(chunk); @@ -6057,7 +6355,7 @@ bool scr_stream_write(ScrStream *s, ScrBytes *chunk, ScrClosure *cb) { scr_dyn_release(v); return ret; } - return scr_stream_write_chunk(s, chunk, cb); + return scr_stream_write_buffer(s, chunk, cb); } /* write(null) reached at runtime (a nullable-union chunk's null arm): @@ -6077,13 +6375,7 @@ bool scr_stream_write_str(ScrStream *s, ScrStr *str, ScrClosure *cb) { scr_dyn_release(v); return ret; } - ScrBytes *b = scr_bytes_new(SCR_BYTES_U8, (double)str->len); - memcpy(b->data, str->data, str->len); - if (!str->ascii) scr_wtf8_to_utf8((char *)b->data, str->len); - scr_stream_buffer(b); - bool ret = scr_stream_write_chunk(s, b, cb); - scr_bytes_release(b); - return ret; + return scr_stream_write_string(s, str, NULL, cb); } /* write(chunk, encoding, cb) with the encoding argument as passed (NULL @@ -6103,20 +6395,15 @@ bool scr_stream_write_enc(ScrStream *s, const ScrDyn *chunk, ScrDyn *enc, ScrClo name = scr_dyn_string_coerce_js(enc); if (name && !(name->len == 6 && memcmp(name->data, "buffer", 6) == 0)) { ScrStr *canonical = scr_bytes_normalize_encoding(name); - scr_str_release(name); if (canonical == NULL) { + scr_str_release(name); if (cb) scr_closure_release(cb); return false; } - if (chunk->kind != SCR_DYN_STR) { - scr_str_release(canonical); - return scr_stream_write_dyn(s, chunk, cb); - } - ScrBytes *b = scr_bytes_from_str(chunk->v.str, canonical); scr_str_release(canonical); - scr_stream_buffer(b); - bool ret = scr_stream_write_chunk(s, b, cb); - scr_bytes_release(b); + bool ret = chunk->kind == SCR_DYN_STR ? scr_stream_write_string(s, chunk->v.str, name, cb) + : scr_stream_write_dyn(s, chunk, cb); + scr_str_release(name); return ret; } if (name && chunk->kind == SCR_DYN_STR) { @@ -6535,8 +6822,41 @@ static void scr_stream_emit_readable_now(ScrStream *s) { /* ── tick dispatch ────────────────────────────────────────────────────── */ /* onConstruct + the readable/writable onConstructed hooks. err moves. */ +/* aggregateTwoErrors(inner, outer) for streams without native ops: both + * present and distinct make an AggregateError([outer, inner]) with + * outer's message and code. Borrows both; +1 or NULL. */ +static ScrError *scr_stream_aggregate_errors(ScrError *inner, ScrError *outer) { + if (inner && outer && inner != outer) { + ScrDyn *message = scr_dyn_new_str(outer->message); + ScrError *agg = scr_aggregate_new(message, scr_dyn_undefined()); + scr_dyn_release(message); + if (agg == NULL) return scr_error_retain(outer); + ScrDyn *outer_value = scr_dyn_from_error(outer); + ScrDyn *list = scr_dyn_new_arr(); + scr_dyn_arr_push(list, scr_dyn_retain(outer_value)); + scr_dyn_arr_push(list, scr_dyn_from_error(inner)); + scr_aggregate_set_errors(agg, list); + scr_dyn_release(list); + ScrDyn *code = scr_dyn_obj_read(outer_value, "code", 4); + if (code && code->kind == SCR_DYN_STR) scr_error_set_code(agg, code->v.str->data); + scr_dyn_release(code); + scr_dyn_release(outer_value); + return agg; + } + ScrError *one = outer ? outer : inner; + return one ? scr_error_retain(one) : NULL; +} + static void scr_stream_constructed(ScrStream *s, ScrError *err) { ScrStreamState *st = s->st; + if (st->construct_finished) { + /* onConstruct called again: errorOrDestroy(err ?? ERR_MULTIPLE_CALLBACK) */ + if (!err) err = scr_stream_mkerr("ERR_MULTIPLE_CALLBACK", "Callback called multiple times"); + scr_stream_error_or_destroy(s, err); + scr_error_release(err); + return; + } + st->construct_finished = true; st->constructing = false; if (st->construct_cb) { scr_closure_release(st->construct_cb); @@ -6549,8 +6869,9 @@ static void scr_stream_constructed(ScrStream *s, ScrError *err) { /* aggregateTwoErrors(constructError, destroyError): the owner builds * the AggregateError when both are present and distinct */ ScrError *use; - if (err && derr && err != derr && st->native_ops && st->native_ops->aggregate) { - use = st->native_ops->aggregate(err, derr); + if (err && derr && err != derr) { + use = st->native_ops && st->native_ops->aggregate ? st->native_ops->aggregate(err, derr) + : scr_stream_aggregate_errors(err, derr); } else { use = derr ? derr : err; if (use) scr_error_retain(use); @@ -6589,6 +6910,39 @@ void scr_stream_construct_done(ScrStream *s, ScrError *err /*moves*/) { void scr_stream_set_writev_concat(ScrStream *s) { s->st->writev_concat = true; } +/* A user construct option or _construct method: Node schedules it from the + * constructor (construct(stream, cb)), and the hook's callback reports + * through scr_stream_construct_done. */ +void scr_stream_set_construct(ScrStream *s, ScrClosure *cb /*moves*/, ScrStreamPlainInv inv) { + if (s->st->construct_cb || s->st->construct_finished) { + scr_closure_release(cb); + return; + } + scr_stream_construct(s, cb, inv); +} + +/* A class's _writev under options that may carry their own writev (the + * option, an instance property in Node, wins). */ +void scr_stream_set_writev_default(ScrStream *s, ScrClosure *cb /*moves*/, ScrStreamWritevInv inv) { + if (s->st->has_w && s->st->w.writev_cb == NULL) { + scr_stream_set_writev(s, cb, inv); + return; + } + scr_closure_release(cb); +} + +/* A user writev option or _writev method. */ +void scr_stream_set_writev(ScrStream *s, ScrClosure *cb /*moves*/, ScrStreamWritevInv inv) { + ScrStreamState *st = s->st; + if (!st->has_w) { + scr_closure_release(cb); + return; + } + if (st->w.writev_cb) scr_closure_release(st->w.writev_cb); + st->w.writev_cb = cb; + st->w.writev_inv = inv; +} + void scr_stream_fail(ScrStream *s, ScrError *err) { scr_stream_error_or_destroy(s, err); } static void scr_stream_plain_inv(ScrClosure *cb, va_list ap) { @@ -6780,6 +7134,7 @@ static void scr_stream_run_tick(ScrStreamTick *t) { if (st->r.read_cb) { scr_closure_release(st->r.read_cb); st->r.read_cb = NULL; } if (st->w.write_cb) { scr_closure_release(st->w.write_cb); st->w.write_cb = NULL; } if (st->w.final_cb) { scr_closure_release(st->w.final_cb); st->w.final_cb = NULL; } + if (st->w.writev_cb) { scr_closure_release(st->w.writev_cb); st->w.writev_cb = NULL; } if (st->transform_cb) { scr_closure_release(st->transform_cb); st->transform_cb = NULL; } if (st->flush_cb) { scr_closure_release(st->flush_cb); st->flush_cb = NULL; } if (st->destroy_cb) { scr_closure_release(st->destroy_cb); st->destroy_cb = NULL; } @@ -6834,7 +7189,11 @@ static void scr_stream_run_tick(ScrStreamTick *t) { scr_stream_error_buffer(s); break; case SCR_ST_CONSTRUCT: { - if (st->construct_cb) st->construct_inv(st->construct_cb, s); + if (st->construct_cb) { + st->construct_inv(st->construct_cb, s); + /* a throw inside _construct reaches onConstruct on a tick */ + if (scr_exc_pending()) scr_stream_construct_done(s, scr_stream_take_error()); + } break; } case SCR_ST_CONSTRUCTED: { diff --git a/packages/runtime/src/scr_stream.h b/packages/runtime/src/scr_stream.h index 8b8a72406..c78a03e1a 100644 --- a/packages/runtime/src/scr_stream.h +++ b/packages/runtime/src/scr_stream.h @@ -65,6 +65,8 @@ typedef void (*ScrStreamReadInv)(ScrClosure *cb, ScrStream *s, double size); typedef void (*ScrStreamChunkInv)(ScrClosure *cb, ScrStream *s, ScrBytes *chunk); typedef void (*ScrStreamPlainInv)(ScrClosure *cb, ScrStream *s); typedef void (*ScrStreamErrInv)(ScrClosure *cb, ScrStream *s, ScrError *err); +/* writev(chunks, callback): chunks is the batch's dyn array (borrowed). */ +typedef void (*ScrStreamWritevInv)(ScrClosure *cb, ScrStream *s, struct ScrDyn *chunks); /* Constructors (+1 result; option callbacks MOVE; NULL = absent). hwm < 0 * means the byte default (65536 — Node 24's getDefaultHighWaterMark). */ @@ -512,6 +514,21 @@ void scr_stream_flush_done_value(ScrStream *s, ScrError *err, struct ScrDyn *v); void scr_stream_construct(ScrStream *s, ScrClosure *cb, ScrStreamPlainInv inv); void scr_stream_construct_done(ScrStream *s, ScrError *err); void scr_stream_set_writev_concat(ScrStream *s); +/* The user construct option / _construct method (scheduled now; cb + * moves), and the writev option / _writev method (cb moves). */ +void scr_stream_set_construct(ScrStream *s, ScrClosure *cb, ScrStreamPlainInv inv); +void scr_stream_set_writev(ScrStream *s, ScrClosure *cb, ScrStreamWritevInv inv); +void scr_stream_set_writev_default(ScrStream *s, ScrClosure *cb, ScrStreamWritevInv inv); +/* decodeStrings (false keeps string writes as strings), the writable + * defaultEncoding option (a known encoding, borrowed), and + * setDefaultEncoding (validated; answers the stream +1). The chaining + * forms answer the stream (+1). */ +void scr_stream_set_decode_strings(ScrStream *s, bool decode); +ScrStream *scr_stream_decode_strings(ScrStream *s, bool decode); +void scr_stream_set_write_encoding(ScrStream *s, ScrStr *enc); +ScrStream *scr_stream_write_encoding(ScrStream *s, ScrStr *enc); +ScrStream *scr_stream_set_default_encoding(ScrStream *s, struct ScrDyn *enc); +ScrDyn *scr_stream_done_dyn_c(ScrClosure *clo, ScrDyn *const *args, size_t argc); /* The native state's extra operations: view adds its members to the * stream's untyped view and close is the view's close() (both NULL when * absent); aggregate builds aggregateTwoErrors' +1 AggregateError from the diff --git a/tests/corpus/7246-stream-construct.ts b/tests/corpus/7246-stream-construct.ts new file mode 100644 index 000000000..7cfb82e49 --- /dev/null +++ b/tests/corpus/7246-stream-construct.ts @@ -0,0 +1,156 @@ +// Deferred construction in typed code: the construct option and the +// _construct method on Readable, Writable, Duplex and Transform. Writes +// and reads wait for the callback, an error from it destroys the stream, +// a destroy while constructing reports both errors as an AggregateError, +// a throw inside construct is its error, and a second callback is +// ERR_MULTIPLE_CALLBACK. +import { EventEmitter } from "node:events"; +import { Duplex, Readable, Transform, Writable } from "node:stream"; + +const log: string[] = []; + +function ticks(count: number): void { + let i = 0; + const step = (): void => { + log.push("t" + i); + if (++i < count) process.nextTick(step); + }; + process.nextTick(step); +} + +class Store extends Writable { + private ready = false; + _construct(callback: (error?: Error | null) => void): void { + log.push("store construct"); + setTimeout(() => { + this.ready = true; + log.push("store ready"); + callback(); + }, 2); + } + _write(chunk: Buffer, _enc: string, cb: (error?: Error | null) => void): void { + log.push(`store write ${chunk.toString()} ready=${this.ready}`); + cb(); + } + _final(cb: (error?: Error | null) => void): void { + log.push("store final"); + cb(); + } +} + +class Pair extends Duplex { + constructor() { + super({ readableObjectMode: true }); + } + _construct(callback: (error?: Error | null) => void): void { + log.push("pair construct"); + process.nextTick(callback); + } + _read(): void {} + _write(chunk: Buffer, _enc: string, cb: (error?: Error | null) => void): void { + this.push({ got: chunk.toString() }); + cb(); + } + _final(cb: (error?: Error | null) => void): void { + this.push(null); + cb(); + } +} + +function closed(stream: EventEmitter): Promise { + return new Promise((resolve) => stream.on("close", () => resolve())); +} + +async function main(): Promise { + ticks(4); + const store = new Store(); + log.push("write " + store.write("a")); + store.end("b"); + store.on("finish", () => log.push("store finish")); + await closed(store); + + const source = new Readable({ + construct(cb: (error?: Error | null) => void) { + log.push("source construct"); + process.nextTick(cb); + }, + read() { + log.push("source read"); + this.push("x"); + this.push(null); + }, + }); + source.on("data", (c: Buffer) => log.push("source data " + c.toString())); + await closed(source); + + const pair = new Pair(); + pair.on("data", (v: { got: string }) => log.push("pair data " + v.got)); + pair.write("p"); + pair.end(); + await closed(pair); + + const upper = new Transform({ + construct(cb: (error?: Error | null) => void) { + log.push("upper construct"); + cb(); + }, + transform(c: Buffer, _e: string, cb: (error: Error | null, data?: Buffer) => void) { + cb(null, Buffer.from(c.toString().toUpperCase())); + }, + }); + upper.on("data", (c: Buffer) => log.push("upper " + c.toString())); + upper.end("shout"); + await closed(upper); + + const failing = new Writable({ + construct(cb: (error?: Error | null) => void) { + cb(new Error("no target")); + }, + write(_c: Buffer, _e: string, cb: (error?: Error | null) => void) { + cb(); + }, + }); + failing.on("error", (e: Error) => log.push("failing " + e.message)); + await closed(failing); + + const raced = new Writable({ + construct(cb: (error?: Error | null) => void) { + setTimeout(() => cb(new Error("late")), 2); + }, + write(_c: Buffer, _e: string, cb: (error?: Error | null) => void) { + cb(); + }, + destroy(err: Error | null, cb: (error?: Error | null) => void) { + log.push("raced destroy " + (err ? err.message : "none")); + cb(err); + }, + }); + raced.on("error", (e: Error) => log.push(`raced ${e.name} ${e.message}`)); + raced.destroy(new Error("early")); + await closed(raced); + + const thrown = new Writable({ + construct() { + throw new Error("thrown in construct"); + }, + write(_c: Buffer, _e: string, cb: (error?: Error | null) => void) { + cb(); + }, + }); + thrown.on("error", (e: Error) => log.push("thrown " + e.message)); + await closed(thrown); + + const twice = new Writable({ + construct(cb: (error?: Error | null) => void) { + cb(); + cb(); + }, + write(_c: Buffer, _e: string, cb: (error?: Error | null) => void) { + cb(); + }, + }); + twice.on("error", (e: Error) => log.push("twice " + (e as NodeJS.ErrnoException).code)); + await closed(twice); + console.log(log.join("\n")); +} +main(); diff --git a/tests/corpus/7247-stream-construct.mjs b/tests/corpus/7247-stream-construct.mjs new file mode 100644 index 000000000..355116fa7 --- /dev/null +++ b/tests/corpus/7247-stream-construct.mjs @@ -0,0 +1,101 @@ +// Deferred construction in JavaScript: construct in an options literal, +// in an options value read at run time, and _construct on a constructor +// function's prototype, with the write, read, error, destroy-while- +// constructing, throwing and double-callback paths. +import * as stream from "node:stream"; +import { Writable, Readable } from "node:stream"; +import * as util from "node:util"; +const log = []; +const tick = (n) => { let i = 0; const f = () => { log.push("t" + i); if (++i < n) process.nextTick(f); }; process.nextTick(f); }; +tick(4); +const wopts = { + construct(cb) { log.push("construct"); setTimeout(() => { log.push("constructed"); cb(); }, 2); }, + write(c, e, cb) { log.push("write " + c + "/" + e); cb(); }, + final(cb) { log.push("final"); cb(); }, +}; +const w = new Writable(dyn(wopts)); +log.push("write ret " + w.write("a")); +w.end("b"); +w.on("finish", () => log.push("finish")); +w.on("close", () => { log.push("close"); next(); }); +function next() { + const ropts = { construct(cb) { log.push("r construct"); process.nextTick(cb); }, read() { log.push("read"); r.push("x"); r.push(null); } }; + const r = new Readable(dyn(ropts)); + r.on("data", (c) => log.push("data " + c)); + r.on("end", () => { log.push("r end"); errs(); }); +} +function errs() { + const eo = { construct(cb) { cb(new Error("boom")); }, write(c, e2, cb) { cb(); } }; + const e = new Writable(dyn(eo)); + e.on("error", (err) => log.push("error " + err.message)); + e.on("close", () => { log.push("e close"); destroyDuring(); }); +} +function destroyDuring() { + const dopts = { construct(cb) { setTimeout(() => cb(new Error("late")), 2); }, write(c, e, cb) { cb(); }, destroy(err, cb) { log.push("destroy " + (err && err.message)); cb(err); } }; + const d = new Writable(dyn(dopts)); + d.on("error", (err) => log.push("d error " + err.name + " " + err.message + " " + (err.errors ? err.errors.map((x) => x.message).join("+") : ""))); + d.on("close", () => { log.push("d close"); throwing(); }); + d.destroy(new Error("early")); +} +function throwing() { + const to = { construct(cb) { throw new Error("thrown"); }, write(c, e, cb) { cb(); } }; + const t = new Writable(dyn(to)); + t.on("error", (err) => log.push("t error " + err.message)); + t.on("close", () => { log.push("t close"); twice(); }); +} +function twice() { + const o = { construct(cb) { cb(); cb(); }, write(c, e, cb) { cb(); } }; + const t = new Writable(dyn(o)); + t.on("error", (err) => log.push("twice error " + err.code)); + t.on("close", () => { log.push("twice close"); viaLiteral(); }); + setTimeout(() => t.destroy(), 5); +} +function dyn(o) { + return o; +} +function viaLiteral() { + const lit = new Writable({ + construct(cb) { + log.push("literal construct " + (this === lit)); + setImmediate(cb); + }, + write(chunk, enc, cb) { + log.push(`literal write ${chunk}/${enc}`); + cb(); + }, + }); + lit.end("z", () => { + log.push("literal end"); + viaConstructor(); + }); +} +function viaConstructor() { + function Bad() { + stream.Readable.call(this); + } + util.inherits(Bad, stream.Readable); + Bad.prototype._construct = function () { + throw new Error("ctor threw"); + }; + Bad.prototype._read = function () {}; + function Late(opts) { + stream.Writable.call(this, opts); + } + util.inherits(Late, stream.Writable); + Late.prototype._construct = function (cb) { + log.push("late construct " + (this instanceof Late)); + setTimeout(cb, 1); + }; + Late.prototype._write = function (chunk, enc, cb) { + log.push("late write " + chunk); + cb(); + }; + const late = new Late(); + late.write("q"); + late.end(() => { + log.push("late end"); + const bad = new Bad(); + bad.on("error", (e) => log.push("bad " + e.message)); + bad.on("close", () => console.log(log.join("\n"))); + }); +} diff --git a/tests/corpus/7248-stream-writev.ts b/tests/corpus/7248-stream-writev.ts new file mode 100644 index 000000000..2df298d6d --- /dev/null +++ b/tests/corpus/7248-stream-writev.ts @@ -0,0 +1,124 @@ +// Batched writes in typed code: the writev option and the _writev method +// on Writable and Duplex. Writes buffered behind a write in flight or a +// cork reach writev as one [{ chunk, encoding }] batch whose callback +// runs the writes' callbacks in order; a stream with only writev sends +// each lone write as a batch of one; a later `stream._writev = fn` +// takes over; an object-mode side batches its values. +import { EventEmitter } from "node:events"; +import { Duplex, Writable } from "node:stream"; + +interface Entry { + chunk: Buffer; + encoding: string; +} + +interface ValueEntry { + chunk: unknown; + encoding: string | undefined; +} + +const log: string[] = []; + +function finished(stream: EventEmitter): Promise { + return new Promise((resolve) => stream.on("finish", () => resolve())); +} + +class Batcher extends Writable { + _write(chunk: Buffer, _enc: string, cb: (error?: Error | null) => void): void { + log.push("batcher write " + chunk.toString()); + setTimeout(cb, 1); + } + _writev(chunks: Entry[], cb: (error?: Error | null) => void): void { + log.push(`batcher writev ${chunks.map((c) => c.chunk.toString() + "/" + c.encoding).join(",")}`); + setTimeout(cb, 1); + } +} + +class Echo extends Duplex { + _read(): void {} + _write(chunk: Buffer, _enc: string, cb: (error?: Error | null) => void): void { + this.push(chunk); + cb(); + } + _writev(chunks: Entry[], cb: (error?: Error | null) => void): void { + this.push(Buffer.concat(chunks.map((c) => c.chunk))); + cb(); + } + _final(cb: (error?: Error | null) => void): void { + this.push(null); + cb(); + } +} + +async function main(): Promise { + const batcher = new Batcher(); + batcher.write("a", () => log.push("a done")); + batcher.write("b", () => log.push("b done")); + batcher.write("c"); + batcher.write(Buffer.from("d"), () => log.push("d done")); + batcher.end(() => log.push("batcher end")); + await finished(batcher); + + const corked = new Writable({ + writev(chunks: Entry[], cb: (error?: Error | null) => void) { + log.push(`corked ${chunks.length}: ${chunks.map((c) => c.chunk.toString("hex")).join(" ")}`); + cb(); + }, + }); + corked.cork(); + corked.write("x"); + corked.write("ff", "hex"); + corked.write("y"); + corked.uncork(); + corked.write("z", () => log.push("z done")); + corked.end(); + await finished(corked); + + const late = new Writable({ + write(c: Buffer, _e: string, cb: () => void) { + log.push("late write " + c.toString()); + cb(); + }, + }); + late._writev = (chunks: Entry[], cb: (error?: Error | null) => void) => { + log.push("late writev " + chunks.length); + cb(); + }; + late.cork(); + late.write("1"); + late.write("2"); + late.uncork(); + late.end(); + await finished(late); + + const echo = new Echo(); + const echoed: string[] = []; + echo.on("data", (c: Buffer) => echoed.push(c.toString())); + echo.cork(); + echo.write("e1"); + echo.write("e2"); + process.nextTick(() => { + echo.uncork(); + echo.end("e3"); + }); + await new Promise((resolve) => echo.on("end", resolve)); + log.push("echoed " + echoed.join(",")); + + const values = new Writable({ + objectMode: true, + writev(chunks: ValueEntry[], cb: (error?: Error | null) => void) { + log.push( + "values " + chunks.map((c) => `${JSON.stringify(c.chunk)}/${String(c.encoding)}`).join(","), + ); + cb(); + }, + }); + values.cork(); + values.write({ n: 1 }); + values.write([2], "utf8"); + values.uncork(); + values.end(); + await finished(values); + console.log(log.join("\n")); +} +main(); diff --git a/tests/corpus/7249-stream-writev.cjs b/tests/corpus/7249-stream-writev.cjs new file mode 100644 index 000000000..0e41fe1c8 --- /dev/null +++ b/tests/corpus/7249-stream-writev.cjs @@ -0,0 +1,89 @@ +// Batched writes in JavaScript: the writev option in a literal and in an +// options value, and _writev on a constructor function's prototype. The +// batch entries carry chunk, encoding and callback (Node's nop when the +// write passed none), allBuffers says whether every chunk is a Buffer, +// and a stream with only _writev sends a lone write as { chunk, encoding }. +const stream = require("node:stream"); +const util = require("node:util"); + +const log = []; +const describe = (chunks) => + chunks + .map((c) => { + const chunk = Buffer.isBuffer(c.chunk) ? "B:" + c.chunk.toString() : `${typeof c.chunk}:${c.chunk}`; + const callback = typeof c.callback === "function" ? c.callback.name || "fn" : String(c.callback); + return `${chunk}/${c.encoding}/${callback}`; + }) + .join(",") + + ` all=${chunks.allBuffers} keys=${Object.keys(chunks[0]).join(".")}`; + +function viaLiteral(next) { + const w = new stream.Writable({ + write(chunk, enc, cb) { + log.push("write " + chunk); + setTimeout(cb, 1); + }, + writev(chunks, cb) { + log.push("writev " + describe(chunks)); + setTimeout(cb, 1); + }, + }); + w.write("a", () => log.push("cb a")); + w.write("b", () => log.push("cb b")); + w.write("c"); + w.write(Buffer.from("d"), () => log.push("cb d")); + w.end(() => log.push("end")); + w.on("finish", () => { + log.push("finish"); + next(); + }); +} + +function dyn(value) { + return value; +} + +function viaValue(next) { + const options = { + writev(chunks, cb) { + log.push("value " + describe(chunks)); + cb(); + }, + }; + const v = new stream.Writable(dyn(options)); + v.cork(); + v.write("x"); + v.write("6869", "hex"); + v.uncork(); + v.write("z", () => log.push("z cb")); + v.end(); + v.on("finish", () => { + log.push("value finish"); + next(); + }); +} + +function Sink(opts) { + stream.Writable.call(this, opts); +} +util.inherits(Sink, stream.Writable); +Sink.prototype._writev = function (chunks, cb) { + log.push("sink " + (this instanceof Sink) + " " + describe(chunks)); + cb(); +}; + +function viaConstructor(next) { + const s = new Sink({ decodeStrings: false }); + s.cork(); + s.write("a"); + s.write("6869", "hex"); + s.write(Buffer.from("b")); + s.uncork(); + s.write("lone"); + s.end(() => { + log.push("sink end"); + next(); + }); +} + +viaLiteral(() => viaValue(() => viaConstructor(() => console.log(log.join("\n"))))); diff --git a/tests/corpus/7250-stream-decode-strings.ts b/tests/corpus/7250-stream-decode-strings.ts new file mode 100644 index 000000000..57e84aff2 --- /dev/null +++ b/tests/corpus/7250-stream-decode-strings.ts @@ -0,0 +1,89 @@ +// decodeStrings: false and the write-side default encoding in typed code: +// a string write reaches _write and _transform as the string with its +// encoding (its UTF-16 length counts toward writableLength), Buffers keep +// 'buffer', defaultEncoding decides how write(string) encodes (and the +// name _write sees when strings stay strings), and setDefaultEncoding() +// lowercases, validates and chains. +import { PassThrough, Transform, Writable } from "node:stream"; + +const log: string[] = []; + +function describe(chunk: string | Buffer, enc: string): string { + return typeof chunk === "string" ? `string:${chunk}/${enc}` : `B:${chunk.toString("hex")}/${enc}`; +} + +class Keeper extends Writable { + seen: string[] = []; + constructor() { + super({ decodeStrings: false, highWaterMark: 4 }); + } + _write(chunk: string | Buffer, enc: string, cb: (error?: Error | null) => void): void { + this.seen.push(describe(chunk, enc)); + setTimeout(cb, 1); + } +} + +async function main(): Promise { + const keeper = new Keeper(); + log.push("ret " + keeper.write("héllo")); + log.push("length " + keeper.writableLength); + keeper.write("6869", "hex"); + keeper.write(Buffer.from("zz")); + keeper.end("end", "latin1"); + await new Promise((resolve) => keeper.on("finish", resolve)); + log.push("keeper " + keeper.seen.join(" ")); + + const hex = new Writable({ + defaultEncoding: "hex", + write(chunk: Buffer, enc: string, cb: (error?: Error | null) => void) { + log.push(`hex ${chunk.toString()} ${enc}`); + cb(); + }, + }); + hex.write("6869"); + hex.write("41", "utf8"); + hex.setDefaultEncoding("BASE64").write("aGk="); + try { + hex.setDefaultEncoding("nope"); + } catch (e) { + log.push(`setDefaultEncoding ${(e as NodeJS.ErrnoException).code} ${(e as Error).message}`); + } + log.push("chains " + (hex.setDefaultEncoding("utf8") === hex)); + hex.end("ok"); + + const both = new Writable({ + decodeStrings: false, + defaultEncoding: "LATIN1", + write(chunk: string | Buffer, enc: string, cb: (error?: Error | null) => void) { + log.push("both " + describe(chunk, enc)); + cb(); + }, + }); + both.write("x"); + both.setDefaultEncoding("UTF-8"); + both.write("y"); + both.end(); + + const t = new Transform({ + decodeStrings: false, + transform(chunk: string | Buffer, enc: string, cb: (error: Error | null, data?: string | Buffer) => void) { + log.push("transform " + describe(chunk, enc)); + cb(null, chunk); + }, + }); + t.on("data", (c: Buffer) => log.push("transform out " + c.toString("hex"))); + t.write("abc"); + t.end("ff", "hex"); + + const p = new PassThrough({ decodeStrings: false }); + p.on("data", (c: Buffer) => log.push("pass " + c.toString())); + p.end("q"); + + try { + new Writable({ defaultEncoding: "bogus" }); + } catch (e) { + log.push("constructor " + (e as NodeJS.ErrnoException).code); + } + setTimeout(() => console.log(log.join("\n")), 10); +} +main(); diff --git a/tests/corpus/7251-stream-write-encodings.mjs b/tests/corpus/7251-stream-write-encodings.mjs new file mode 100644 index 000000000..5972f6684 --- /dev/null +++ b/tests/corpus/7251-stream-write-encodings.mjs @@ -0,0 +1,65 @@ +// Write encodings in JavaScript: decodeStrings: false and defaultEncoding +// in options literals and options values, setDefaultEncoding(), and +// strings written in other encodings to constructor-function streams. +import * as stream from "node:stream"; +import { PassThrough, Transform, Writable } from "node:stream"; +import * as util from "node:util"; + +const log = []; +const describe = (c, e) => `${Buffer.isBuffer(c) ? "B:" + c.toString("hex") : typeof c + ":" + c}/${e}`; +const dyn = (value) => value; + +const keep = new Writable({ decodeStrings: false, highWaterMark: 4, write(c, e, cb) { log.push("keep " + describe(c, e)); cb(); } }); +log.push("ret " + keep.write("héllo") + " length " + keep.writableLength); +keep.write("6869", "hex"); +keep.write(Buffer.from("zz")); +keep.end("end", "latin1"); + +const valueOpts = { decodeStrings: false, defaultEncoding: "HEX", write(c, e, cb) { log.push("value " + describe(c, e)); cb(); } }; +const fromValue = new Writable(dyn(valueOpts)); +fromValue.write("ab"); +fromValue.write(Buffer.from("c")); + +const hex = new Writable({ defaultEncoding: "hex", write(c, e, cb) { log.push("hex " + describe(c, e)); cb(); } }); +hex.write("6869"); +hex.write("41", "utf8"); +hex.setDefaultEncoding("BASE64"); +hex.write("aGk="); +try { + hex.setDefaultEncoding("nope"); +} catch (e) { + log.push("setDefaultEncoding " + e.code + " " + e.message); +} +log.push("chains " + (hex.setDefaultEncoding("utf8") === hex)); + +const t = new Transform({ decodeStrings: false, transform(c, e, cb) { log.push("t " + describe(c, e)); cb(null, c); } }); +t.on("data", (c) => log.push("t data " + describe(c, "-"))); +t.write("abc"); +t.write("ff", "hex"); +const p = new PassThrough({ decodeStrings: false }); +p.on("data", (c) => log.push("p data " + describe(c, "-"))); +p.write("q"); + +const om = new Writable({ objectMode: true, defaultEncoding: "hex", write(c, e, cb) { log.push("om " + describe(c, e)); cb(); } }); +om.write("s"); +try { + new Writable({ defaultEncoding: "bogus" }); +} catch (e) { + log.push("constructor " + e.code); +} + +function Sink() { + stream.Writable.call(this); +} +util.inherits(Sink, stream.Writable); +Sink.prototype._write = function (c, e, cb) { + log.push("sink " + describe(c, e)); + cb(); +}; +const sink = new Sink(); +sink.write("6869", "hex"); +sink.setDefaultEncoding("latin1"); +sink.write("é"); +sink.end("YQ==", "base64"); + +setTimeout(() => console.log(log.join("\n")), 5); diff --git a/tests/harness/corpus-costs/7246-stream-construct.ts.json b/tests/harness/corpus-costs/7246-stream-construct.ts.json new file mode 100644 index 000000000..d94320c7e --- /dev/null +++ b/tests/harness/corpus-costs/7246-stream-construct.ts.json @@ -0,0 +1 @@ +{ "plain": 2327, "san": 3064 } diff --git a/tests/harness/corpus-costs/7247-stream-construct.mjs.json b/tests/harness/corpus-costs/7247-stream-construct.mjs.json new file mode 100644 index 000000000..a74da86c8 --- /dev/null +++ b/tests/harness/corpus-costs/7247-stream-construct.mjs.json @@ -0,0 +1 @@ +{ "plain": 2834, "san": 3538 } diff --git a/tests/harness/corpus-costs/7248-stream-writev.ts.json b/tests/harness/corpus-costs/7248-stream-writev.ts.json new file mode 100644 index 000000000..83ccdb03d --- /dev/null +++ b/tests/harness/corpus-costs/7248-stream-writev.ts.json @@ -0,0 +1 @@ +{ "plain": 2294, "san": 3026 } diff --git a/tests/harness/corpus-costs/7249-stream-writev.cjs.json b/tests/harness/corpus-costs/7249-stream-writev.cjs.json new file mode 100644 index 000000000..2fc75720e --- /dev/null +++ b/tests/harness/corpus-costs/7249-stream-writev.cjs.json @@ -0,0 +1 @@ +{ "plain": 2640, "san": 2765 } diff --git a/tests/harness/corpus-costs/7250-stream-decode-strings.ts.json b/tests/harness/corpus-costs/7250-stream-decode-strings.ts.json new file mode 100644 index 000000000..b6c0a39a0 --- /dev/null +++ b/tests/harness/corpus-costs/7250-stream-decode-strings.ts.json @@ -0,0 +1 @@ +{ "plain": 2016, "san": 2519 } diff --git a/tests/harness/corpus-costs/7251-stream-write-encodings.mjs.json b/tests/harness/corpus-costs/7251-stream-write-encodings.mjs.json new file mode 100644 index 000000000..7e459d978 --- /dev/null +++ b/tests/harness/corpus-costs/7251-stream-write-encodings.mjs.json @@ -0,0 +1 @@ +{ "plain": 2161, "san": 2661 } diff --git a/tests/harness/self-hosting-native-toolchain.test.ts b/tests/harness/self-hosting-native-toolchain.test.ts index 176caa516..c0176eb93 100644 --- a/tests/harness/self-hosting-native-toolchain.test.ts +++ b/tests/harness/self-hosting-native-toolchain.test.ts @@ -108,7 +108,7 @@ for (const backend of ["llvm"] as const) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 26, marker: "scr_runtime_abi_v26" }, + runtime_abi: { version: 27, marker: "scr_runtime_abi_v27" }, compiler: { command: "fixture", identity: "fixture", target: target.llvmTriple }, macros: { executable: [], excluded: [], sanitizer: "external-toolchain-required" }, flavors: { From 4e06e72b75fbc1d2095291cf38df48fc46515594 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 09:54:40 -0500 Subject: [PATCH 036/197] Key flatMap union helpers by their arm plan and copy array elements as stored The union-returning flatMap helper was interned by element and result types alone, so two callbacks with one union result type, one with an empty `never[]` arm and one with a populated array arm, shared the first helper and the second dropped its elements. The key now includes the per-arm plan. Array arms copy their elements as stored, a present undefined included, so an array arm's element type must be the result's, and a unit-typed result (`undefined`) appends itself. --- .../lowering/containers/array-callbacks.ts | 34 +++++++++++++------ tests/corpus/7029-flatmap-union-results.ts | 13 +++++++ 2 files changed, 37 insertions(+), 10 deletions(-) diff --git a/packages/compiler/src/frontend/lowering/containers/array-callbacks.ts b/packages/compiler/src/frontend/lowering/containers/array-callbacks.ts index 3de1a4e6b..02d0bc3e7 100644 --- a/packages/compiler/src/frontend/lowering/containers/array-callbacks.ts +++ b/packages/compiler/src/frontend/lowering/containers/array-callbacks.ts @@ -1588,9 +1588,10 @@ interface FlatMapUnionPlan { arms: ("spread" | "empty" | "value")[]; } -/** The plan for a callback returning `T | U[]`: every non-array arm and - * every array arm's element must convert into the result's element type - * without a check. Null when an arm does not. */ +/** The plan for a callback returning `T | U[]`: every non-array arm must + * convert into the result's element type without a check, and every + * array arm's element type must be the result's. Null when an arm does + * not. */ function flatMapUnionPlan( lowerer: Lowerer, call: ts.CallExpression, @@ -1623,7 +1624,9 @@ function flatMapUnionPlan( const plan: FlatMapUnionPlan["arms"] = []; for (const arm of arms) { if (arm.kind === "array" && emptyArms.some((e) => typeEquals(e, arm))) plan.push("empty"); - else if (arm.kind === "array" && converts(arm.elem)) plan.push("spread"); + // An array arm copies its elements as they are stored, a present + // undefined included, so its element type is the result's. + else if (arm.kind === "array" && typeEquals(arm.elem, outElem)) plan.push("spread"); else if (arm.kind !== "array" && converts(arm)) plan.push("value"); else return null; } @@ -1642,7 +1645,9 @@ function flatMapUnionHelper( loc: SrcLoc, present: boolean, ): string { - const key = `flatMapUnion:${typeKey(elem)}:${typeKey(fnRet)}:${typeKey(plan.outElem)}:${arity}${present ? ":present" : ""}`; + // The plan is part of the key: one union result type can carry an empty + // `never[]` arm in one callback and a populated array arm in another. + const key = `flatMapUnion:${typeKey(elem)}:${typeKey(fnRet)}:${typeKey(plan.outElem)}:${plan.arms.join(",")}:${arity}${present ? ":present" : ""}`; const existing = lowerer.arrHofHelpers.get(key); if (existing) return existing; const name = `%arr.flatMap.${lowerer.arrHofHelpers.size}`; @@ -1683,7 +1688,19 @@ function flatMapUnionHelper( lowerer, outRef(), outLength(), - lowerer.coerceToExpected(narrowed, plan.outElem), + lowerer.coerceToExpected( + // A unit arm's value is the unit itself, which has no + // narrowed representation. + isUnitType(arm) + ? { + kind: "unitLit", + unit: arm.kind === "nullT" ? "null" : "undefined", + type: arm, + loc, + } + : narrowed, + plan.outElem, + ), plan.outElem, loc, ), @@ -1749,10 +1766,7 @@ function flatMapUnionHelper( lowerer, outRef(), outLength(), - lowerer.coerceToExpected( - varRef(itemId, arrayValueType(lowerer, innerT.elem), loc), - plan.outElem, - ), + varRef(itemId, arrayValueType(lowerer, innerT.elem), loc), plan.outElem, loc, ), diff --git a/tests/corpus/7029-flatmap-union-results.ts b/tests/corpus/7029-flatmap-union-results.ts index 83c9583de..efeae4d8f 100644 --- a/tests/corpus/7029-flatmap-union-results.ts +++ b/tests/corpus/7029-flatmap-union-results.ts @@ -29,3 +29,16 @@ const out = words.flatMap((s) => { return s.length > 0 ? s.toUpperCase() : []; }); console.log(out, calls); + +// One union result type with an empty arm in one callback and a populated +// array arm in another. +const counts = [1, 2, 3]; +console.log(counts.flatMap((n) => (n % 2 ? n : [])), counts.flatMap((n) => (n % 2 ? n : [n, n]))); + +// Array results keep present undefined elements, and an undefined result +// appends itself. +const withGaps: (number | undefined)[] = [4, undefined, 6]; +const maybe: (number | undefined)[] = [1, 2, undefined]; +console.log(maybe.flatMap((n) => (n === 2 ? withGaps : n))); +const gaps: (string | undefined)[] = ["x", undefined]; +console.log(["a", "b"].flatMap((s) => (s === "b" ? gaps : s))); From c9ae4d80196eeabdacc7574f3dd216bf0422f32e Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 09:55:30 -0500 Subject: [PATCH 037/197] Look up spawnSync commands on the replacement environment's PATH With an `env` option, spawnSync searched the parent's PATH for a command without a '/'. Node (libuv) searches the PATH of the child's environment, or the default path when that environment has none, so a command only on the child's PATH ran as ENOENT and one only on the parent's PATH ran where Node reports ENOENT. The lookup now follows libuv: each PATH entry in order, an empty entry as the current directory, and EACCES when a candidate exists but cannot run. --- packages/runtime/src/scr_child.c | 62 +++++++++++++++++++++++++++-- tests/corpus/7030-spawn-sync-env.ts | 19 +++++++++ 2 files changed, 78 insertions(+), 3 deletions(-) diff --git a/packages/runtime/src/scr_child.c b/packages/runtime/src/scr_child.c index 387885c42..df60fcb26 100644 --- a/packages/runtime/src/scr_child.c +++ b/packages/runtime/src/scr_child.c @@ -2907,6 +2907,7 @@ void scr_children_poll(void) { #include #include #include +#include #include #include #include @@ -3111,8 +3112,55 @@ static ScrSpawnRes *scr_spawn_res_new(bool has_status, double status, static char **scr_exec_envp(ScrArr *pairs); static void scr_exec_envp_free(char **envp); +/* The executable `file` names when `envp` replaces the child's + * environment, found as libuv finds it: a name with a '/' is used as + * given; otherwise each entry of the replacement's PATH (an empty entry + * is the current directory), or of the default path when the replacement + * has no PATH, is tried in order. Returns a malloc'd path, or NULL with + * *err set: EACCES when a candidate exists but cannot run, else ENOENT. */ +static char *scr_spawn_path_lookup(const char *file, char *const *envp, int *err) { + if (strchr(file, '/')) { + char *copy = strdup(file); + if (!copy) scr_child_oom(); + return copy; + } + const char *path = "/usr/bin:/bin"; + for (char *const *e = envp; *e; e++) { + if (strncmp(*e, "PATH=", 5) == 0) { + path = *e + 5; + break; + } + } + size_t file_len = strlen(file); + bool denied = false; + for (const char *dir = path;;) { + const char *end = strchr(dir, ':'); + size_t dir_len = end ? (size_t)(end - dir) : strlen(dir); + const char *prefix = dir_len ? dir : "."; + size_t prefix_len = dir_len ? dir_len : 1; + char *candidate = malloc(prefix_len + 1 + file_len + 1); + if (!candidate) scr_child_oom(); + memcpy(candidate, prefix, prefix_len); + candidate[prefix_len] = '/'; + memcpy(candidate + prefix_len + 1, file, file_len + 1); + struct stat st; + if (stat(candidate, &st) == 0) { + if (S_ISREG(st.st_mode) && access(candidate, X_OK) == 0) return candidate; + denied = true; + } else if (errno == EACCES) { + denied = true; + } + free(candidate); + if (!end) break; + dir = end + 1; + } + *err = denied ? EACCES : ENOENT; + return NULL; +} + /* env_pairs: NULL inherits the parent's environment; else [k,v,...] - * replaces it, as Node's `env` option does. */ + * replaces it, as Node's `env` option does, and the command is looked up + * on the replacement's PATH (scr_spawn_path_lookup). */ static ScrSpawnRes *scr_spawn_sync_core(ScrStr *cmd, ScrArr *args, double timeout_ms, int killsig, int in_mode, int out_mode, int err_mode, ScrArr *env_pairs) { @@ -3152,8 +3200,16 @@ static ScrSpawnRes *scr_spawn_sync_core(ScrStr *cmd, ScrArr *args, double timeou char **argv = scr_child_argv(cmd, args); char **envp = env_pairs != NULL ? scr_exec_envp(env_pairs) : NULL; pid_t pid = -1; - int spawn_err = posix_spawnp(&pid, cmd->data, &fa, NULL, argv, - envp != NULL ? envp : environ); + int spawn_err; + if (envp != NULL) { + char *resolved = scr_spawn_path_lookup(cmd->data, envp, &spawn_err); + if (resolved) { + spawn_err = posix_spawn(&pid, resolved, &fa, NULL, argv, envp); + free(resolved); + } + } else { + spawn_err = posix_spawnp(&pid, cmd->data, &fa, NULL, argv, environ); + } posix_spawn_file_actions_destroy(&fa); free(argv); scr_exec_envp_free(envp); diff --git a/tests/corpus/7030-spawn-sync-env.ts b/tests/corpus/7030-spawn-sync-env.ts index e613c81f0..0ab001b8f 100644 --- a/tests/corpus/7030-spawn-sync-env.ts +++ b/tests/corpus/7030-spawn-sync-env.ts @@ -1,6 +1,9 @@ // spawnSync's env option replaces the child's environment, alone or with // the other options. import { spawnSync } from "node:child_process"; +import { chmodSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; const merged = spawnSync("sh", ["-c", 'echo "$GREETING $HOME_SET"'], { env: { ...process.env, GREETING: "hello", HOME_SET: process.env.HOME ? "yes" : "no" }, @@ -26,3 +29,19 @@ console.log(quiet.status, quiet.stderr.toString().trim(), quiet.stdout.toString( const missing = spawnSync("/nonexistent/tool", [], { env: { A: "1" } }); console.log(missing.status, missing.error?.message); + +// The command is found on the replacement environment's PATH, or on the +// default path when the replacement has none, not on the parent's PATH. +const dir = mkdtempSync(join(tmpdir(), "scr-spawn-path-")); +writeFileSync(join(dir, "child-only-tool"), "#!/bin/sh\necho child-only\n"); +chmodSync(join(dir, "child-only-tool"), 0o755); +writeFileSync(join(dir, "not-executable"), "#!/bin/sh\necho never\n"); +const viaPath = spawnSync("child-only-tool", [], { env: { PATH: `/nonexistent:${dir}` }, encoding: "utf8" }); +console.log(viaPath.status, viaPath.stdout.trim()); +const defaultPath = spawnSync("sh", ["-c", "echo default"], { env: { A: "1" }, encoding: "utf8" }); +console.log(defaultPath.status, defaultPath.stdout.trim()); +const notOnPath = spawnSync("sh", ["-c", "echo never"], { env: { PATH: dir }, encoding: "utf8" }); +console.log(notOnPath.status, notOnPath.error?.message); +const denied = spawnSync("not-executable", [], { env: { PATH: dir }, encoding: "utf8" }); +console.log(denied.status, denied.error?.message); +rmSync(dir, { recursive: true }); From 05acb4d24783f63cabf49cdc177d138fa5ea0f70 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 09:58:47 -0500 Subject: [PATCH 038/197] Update the DiffieHellman runtime test for the prime search test_dh.c includes scr_dh.c and still called sd_generate_safe_prime without its new top_two argument, so dh.test.ts failed to compile. It now passes false, as the DiffieHellman paths do, and a new case checks generatePrime's search: the sizes and add forms with one possible result, top-two-bit safe-prime starts, and checks above OpenSSL's round count. --- packages/runtime/test/test_dh.c | 54 ++++++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/packages/runtime/test/test_dh.c b/packages/runtime/test/test_dh.c index aa8f175a3..f459f1047 100644 --- a/packages/runtime/test/test_dh.c +++ b/packages/runtime/test/test_dh.c @@ -199,7 +199,7 @@ static void test_generate(void) { mbedtls_mpi_lset(&g, 2); for (size_t i = 0; i < sizeof adds / sizeof adds[0]; i++) { for (int trial = 0; trial < 2; trial++) { - bool ok = sd_generate_safe_prime(&p, 512, adds[i], adds[i] - 1); + bool ok = sd_generate_safe_prime(&p, 512, adds[i], adds[i] - 1, false); mbedtls_mpi_uint r = 0; ok = ok && mbedtls_mpi_bitlen(&p) == 512 && mbedtls_mpi_mod_int(&r, &p, adds[i]) == 0 && r == adds[i] - 1; mbedtls_mpi_copy(&q, &p); @@ -217,12 +217,64 @@ static void test_generate(void) { mbedtls_mpi_free(&g); } +/* generatePrime's search: the sizes and add forms with one possible + * result, top-two-bit safe-prime starts, and checks above OpenSSL's + * round count. */ +static void test_generate_prime(void) { + static const struct { + size_t bits; + bool safe; + int add, rem; + int want; + } cases[] = { + {2, false, 0, 0, 3}, {3, false, 0, 0, 7}, {4, false, 0, 0, 13}, {3, true, 0, 0, 7}, + {6, true, 0, 0, 59}, {7, true, 0, 0, 107}, {8, false, 200, 1, 401}, {8, false, 128, 0, 257}, + {5, false, 16, 3, 19}, + }; + mbedtls_mpi p, add, rem, q; + mbedtls_mpi_init(&p); + mbedtls_mpi_init(&add); + mbedtls_mpi_init(&rem); + mbedtls_mpi_init(&q); + for (size_t i = 0; i < sizeof cases / sizeof cases[0]; i++) { + mbedtls_mpi_lset(&add, cases[i].add); + mbedtls_mpi_lset(&rem, cases[i].rem); + bool ok = sd_generate_prime(&p, cases[i].bits, cases[i].safe, cases[i].add ? &add : NULL, + cases[i].rem ? &rem : NULL) && + mbedtls_mpi_cmp_int(&p, cases[i].want) == 0; + char what[96]; + snprintf(what, sizeof what, "generatePrime(%zu, safe %d, add %d, rem %d) is %d", cases[i].bits, cases[i].safe, + cases[i].add, cases[i].rem, cases[i].want); + check(ok, what); + } + for (int trial = 0; trial < 2; trial++) { + bool ok = sd_generate_safe_prime(&p, 128, 12, 11, true) && mbedtls_mpi_bitlen(&p) == 128 && + mbedtls_mpi_get_bit(&p, 126) == 1; + mbedtls_mpi_copy(&q, &p); + mbedtls_mpi_shift_r(&q, 1); + ok = ok && mbedtls_mpi_is_prime_ext(&p, 40, sd_random, NULL) == 0 && + mbedtls_mpi_is_prime_ext(&q, 40, sd_random, NULL) == 0; + check(ok, "a 128-bit safe prime with its top two bits set"); + } + ScDhSmallPrimes *small = sd_small_primes(); + mbedtls_mpi_read_string(&q, 10, "3215031751"); + check(!sd_is_prime_checks(small, &q, 200), "200 rounds reject a strong pseudoprime to 2, 3, 5 and 7"); + mbedtls_mpi_read_string(&q, 10, "2305843009213693951"); + check(sd_is_prime_checks(small, &q, 200), "200 rounds accept 2^61 - 1"); + free(small); + mbedtls_mpi_free(&p); + mbedtls_mpi_free(&add); + mbedtls_mpi_free(&rem); + mbedtls_mpi_free(&q); +} + int main(void) { test_mont_pow(); test_is_prime(); test_check_params(); test_long_exponent(); test_generate(); + test_generate_prime(); fprintf(stderr, "%ld/%ld cases passed\n", total - failed, total); return failed ? 1 : 0; } From c43d4002e71f3c87a8b7bee79c12c02793bf1c2b Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 09:59:57 -0500 Subject: [PATCH 039/197] Skip only always-throwing calls as record producers in the key-order witness The witness skipped every libCall whose signature result is VOID. VOID is also the signature of runtime calls that take their record type from the use site and build the record at run time, such as net.serverAddress (`server.address()`). A literal of the same shape then set the shape's key order, and the runtime record enumerated in the literal's order instead of Node's (`port, family, address` for `address, family, port`). The witness now skips only the calls in ALWAYS_THROW_LIB_FNS, which never produce a value. The set moves from validate.ts's result-type check to builtin-effects.ts, and both use it. --- .../lowering/key-order-witness.test.ts | 30 +++++++++++++++++++ .../frontend/lowering/key-order-witness.ts | 11 +++---- packages/compiler/src/ir/builtin-effects.ts | 22 ++++++++++++++ packages/compiler/src/ir/validate.ts | 29 ++---------------- 4 files changed, 61 insertions(+), 31 deletions(-) diff --git a/packages/compiler/src/frontend/lowering/key-order-witness.test.ts b/packages/compiler/src/frontend/lowering/key-order-witness.test.ts index de16e5d61..8632bad47 100644 --- a/packages/compiler/src/frontend/lowering/key-order-witness.test.ts +++ b/packages/compiler/src/frontend/lowering/key-order-witness.test.ts @@ -157,6 +157,36 @@ test("a synthesized construction pins today's table", () => { expect(f.order(ab)).toEqual(["a", "b"]); }); +test("a runtime builder typed by its use site pins today's table", () => { + const f = fixture(); + const ab = f.record( + [ + ["a", F64], + ["b", F64], + ], + ["a", "b"], + ); + // net.serverAddress's signature result is VOID: the call takes its record + // type from its use site, and the runtime builds the record in that order. + const built: IrExpr = { kind: "libCall", fn: "net.serverAddress", args: [], type: ab, loc }; + f.run([observe(f.literal(ab, ["b", "a"])), observe(built)]); + expect(f.order(ab)).toEqual(["a", "b"]); +}); + +test("an always-throwing call typed as the record leaves its literals' order", () => { + const f = fixture(); + const ab = f.record( + [ + ["a", F64], + ["b", F64], + ], + ["a", "b"], + ); + const thrown: IrExpr = { kind: "libCall", fn: "error.nodeThrow", args: [], type: ab, loc }; + f.run([observe(f.literal(ab, ["b", "a"])), observe(thrown)]); + expect(f.order(ab)).toEqual(["b", "a"]); +}); + test("a checked conversion alone pins today's table", () => { const f = fixture(); const ab = f.record( diff --git a/packages/compiler/src/frontend/lowering/key-order-witness.ts b/packages/compiler/src/frontend/lowering/key-order-witness.ts index 7ce8ae074..4adfb87ab 100644 --- a/packages/compiler/src/frontend/lowering/key-order-witness.ts +++ b/packages/compiler/src/frontend/lowering/key-order-witness.ts @@ -45,7 +45,7 @@ import type { IrExpr, IrFunction, IrRecordShape, IrStmt, IrType } from "../../ir import { shapeHasAccessorSlots, typeEquals, UNDEFINED_T } from "../../ir/ir.js"; import { everyExprChild, everyStmtChild, everyTypeChild } from "../../ir/traverse.js"; import type { Lowerer } from "./lowerer.js"; -import { LIB_FN_SIGS } from "../../ir/builtin-signatures.js"; +import { ALWAYS_THROW_LIB_FNS } from "../../ir/builtin-effects.js"; /** One property of an object literal, in source order. */ export type KeyOrderSegment = @@ -313,10 +313,11 @@ export function applyKeyOrderWitnesses(lowerer: Lowerer, fns: readonly IrFunctio // Runtime surfaces (console, util.format, workers) that take a // record may walk its keys; runtime builders make new records. for (const arg of e.args) observedRoots.push(arg.type); - // A void runtime call typed by its use site is a throw standing in - // for a value (`error.nodeThrow` in a narrowing ternary): it never - // produces one. - if (LIB_FN_SIGS[e.fn].result.kind !== "void") producedBy(e.type, true); + // An always-throwing call stands in for a value (`error.nodeThrow` + // in a narrowing ternary) and never produces one. Other calls + // whose signature result is VOID are typed by their use site and + // build the record at run time (`server.address()`). + if (!ALWAYS_THROW_LIB_FNS.has(e.fn)) producedBy(e.type, true); return; case "intrinsic": // Intrinsics hand back values built elsewhere, except allSettled's diff --git a/packages/compiler/src/ir/builtin-effects.ts b/packages/compiler/src/ir/builtin-effects.ts index 16db45b6c..c2194a13c 100644 --- a/packages/compiler/src/ir/builtin-effects.ts +++ b/packages/compiler/src/ir/builtin-effects.ts @@ -1001,3 +1001,25 @@ export const MAY_THROW_LIB_FNS: ReadonlySet = new Set([ "crypto.createSign", "crypto.createVerify", ]); + +/** libCall members that always throw. Their result type is the replaced + * expression's own and is never materialized: global.undefRead's is the + * declared type of the undefined global; the others stand in for a value + * of the expression they replace. The fs Chk ladders qualify: every + * validation failure throws Node's typed error, and a full pass throws the + * trailing compiler-rendered fence. */ +export const ALWAYS_THROW_LIB_FNS: ReadonlySet = new Set([ + "global.undefRead", + "error.nodeThrow", + "error.argTypeThrow", + "error.propTypeThrow", + "fs.mkdtempChk", + "fs.readFileChk", + "fs.opendirChk", + "fs.watchFileChk", + "fs.lchmodChk", + "fs.readChk", + "fs.streamOptsChk", + "net.connectOptsChk", + "tls.caCertsChk", +]); diff --git a/packages/compiler/src/ir/validate.ts b/packages/compiler/src/ir/validate.ts index bb36c7007..163c2a41c 100644 --- a/packages/compiler/src/ir/validate.ts +++ b/packages/compiler/src/ir/validate.ts @@ -61,6 +61,7 @@ import { unionWideningTags } from "./analysis.js"; import { alwaysReturns } from "./control-flow.js"; import { dynamicOperationOf, functionTier } from "./tiers.js"; import { LIB_FN_SIGS, pipelinePumpedRuns } from "./builtin-signatures.js"; +import { ALWAYS_THROW_LIB_FNS } from "./builtin-effects.js"; /** The promisified crypto calls' value kinds besides checked-dynamic. */ const CRYPTO_PROMISE_VALUES: Readonly> = { @@ -5345,32 +5346,8 @@ function validateFunction( const callback = e.args[e.args.length - 1]?.type; if (callback?.kind !== "func") err(`libCall ${e.fn} callback must be a function`, e.loc); } - if (e.fn === "global.undefRead") { - // Always throws — the result type is whatever the declared type - // of the undefined global mapped to (never materialized). - return true; - } - if ( - e.fn === "error.nodeThrow" || - e.fn === "error.argTypeThrow" || - e.fn === "error.propTypeThrow" || - e.fn === "fs.mkdtempChk" || - e.fn === "fs.readFileChk" || - e.fn === "fs.opendirChk" || - e.fn === "fs.watchFileChk" || - e.fn === "fs.lchmodChk" || - e.fn === "fs.readChk" || - e.fn === "fs.streamOptsChk" || - e.fn === "net.connectOptsChk" || - e.fn === "tls.caCertsChk" - ) { - // Always throws — the result type is the replaced expression's - // own (never materialized; the global.undefRead pattern). The - // fs Chk ladders qualify: every validation failure throws - // Node's typed error, and a full pass throws the trailing - // compiler-rendered fence. - return true; - } + // Always throws: the result type is never materialized. + if (ALWAYS_THROW_LIB_FNS.has(e.fn)) return true; if (e.fn === "error.new" || e.fn === "error.newOptions") { // Which builtin the runtime constructs is named by the result type. if (!isBuiltinErrorObject(e.type)) { From affe386bbf62de66812386b9fa07fbba1d8f37fd Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 10:09:43 -0500 Subject: [PATCH 040/197] Throw for every object on the right of an arrow instanceof, and give blocked bodies their this - An arrow function or method on the right of an untyped `instanceof` has no prototype. Node throws "Function has non-object prototype 'undefined' in instanceof check" for any object, but the runtime threw only for plain objects, arrays and functions, and answered false for typed arrays, native handles, promises and typed references. It now answers false only for primitives. - Coverage lowers the body of a signature-blocked function for its diagnostics. A declared `this` parameter now gives that body its receiver, as lowerFunction does, so `this.label` reports nothing. Before, each use reported SC1080. --- .../src/frontend/lowering/lower-functions.ts | 3 +++ packages/runtime/src/scr_json.c | 10 +++++++--- tests/corpus/7005-dyn-constructor-instanceof/main.ts | 3 ++- .../node_modules/tag-types/index.d.ts | 1 + .../node_modules/tag-types/index.js | 10 ++++++++++ tests/coverage-fixtures/blocked-signature-body.ts | 4 ++++ tests/harness/coverage.test.ts | 12 ++++++++---- 7 files changed, 35 insertions(+), 8 deletions(-) diff --git a/packages/compiler/src/frontend/lowering/lower-functions.ts b/packages/compiler/src/frontend/lowering/lower-functions.ts index ea7d7957b..c1cca9c64 100644 --- a/packages/compiler/src/frontend/lowering/lower-functions.ts +++ b/packages/compiler/src/frontend/lowering/lower-functions.ts @@ -893,6 +893,9 @@ export function blockedBodyDiagnostics( lowerer.fnStack.push(ctx); let internal = false; try { + // A declared `this` parameter gives the body its receiver, as + // lowerFunction declares it. + if (decl.parameters.nodes.some(isThisParameter)) declareCallReceiver(lowerer, decl, false); for (const param of decl.parameters.nodes) { if (isThisParameter(param)) continue; const declaredParam = lowersCleanly(lowerer, () => { diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index 906be0ccd..fbfcade66 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -3968,10 +3968,14 @@ bool scr_bytes_instanceof_at(const ScrDyn *value, const ScrDyn *callee, const Sc bool undefined_prototype = prototype->kind == SCR_DYN_UNDEF; scr_dyn_release(prototype); /* A function the program compiled with no prototype (an arrow function - * or a method): Node's TypeError for an object, false for a primitive. - * The runtime's own constructors carry "native:" signatures. */ + * or a method): false for a primitive, and Node's TypeError for every + * object, typed arrays, native handles, promises and typed references + * included. The runtime's own constructors carry "native:" signatures. */ if (undefined_prototype && (!callee->v.fn.sig || strncmp(callee->v.fn.sig, "native:", 7) != 0)) { - if (value->kind != SCR_DYN_OBJ && value->kind != SCR_DYN_ARR && value->kind != SCR_DYN_FUNC) return false; + if (value->kind == SCR_DYN_UNDEF || value->kind == SCR_DYN_NULL || value->kind == SCR_DYN_BOOL || + value->kind == SCR_DYN_NUM || value->kind == SCR_DYN_STR || value->kind == SCR_DYN_BIGINT || + value->kind == SCR_DYN_SYMBOL) + return false; static const char message[] = "Function has non-object prototype 'undefined' in instanceof check"; scr_throw_error_msg(SCR_ERR_TYPE, message, sizeof message - 1); return false; diff --git a/tests/corpus/7005-dyn-constructor-instanceof/main.ts b/tests/corpus/7005-dyn-constructor-instanceof/main.ts index c32a820c2..21fcbbd26 100644 --- a/tests/corpus/7005-dyn-constructor-instanceof/main.ts +++ b/tests/corpus/7005-dyn-constructor-instanceof/main.ts @@ -3,9 +3,10 @@ // an ES5 constructor's prototype, at module load and in calls, and throws a // TypeError for an arrow function, which has no prototype, when the value // is an object. -import { arrowCheck, describe, rejects, tags } from "tag-types"; +import { arrowCheck, arrowObjects, describe, rejects, tags } from "tag-types"; console.log(tags()); console.log(describe("{}"), describe("[1]"), describe("null"), describe('"s"')); console.log(rejects('{"tag":"!x"}')); console.log(arrowCheck("{}"), arrowCheck("1")); +console.log(arrowObjects()); diff --git a/tests/corpus/7005-dyn-constructor-instanceof/node_modules/tag-types/index.d.ts b/tests/corpus/7005-dyn-constructor-instanceof/node_modules/tag-types/index.d.ts index adeea728e..5b451adfa 100644 --- a/tests/corpus/7005-dyn-constructor-instanceof/node_modules/tag-types/index.d.ts +++ b/tests/corpus/7005-dyn-constructor-instanceof/node_modules/tag-types/index.d.ts @@ -2,3 +2,4 @@ export declare function describe(json: string): string; export declare function tags(): string; export declare function rejects(json: string): string; export declare function arrowCheck(json: string): string; +export declare function arrowObjects(): string; diff --git a/tests/corpus/7005-dyn-constructor-instanceof/node_modules/tag-types/index.js b/tests/corpus/7005-dyn-constructor-instanceof/node_modules/tag-types/index.js index 20eee362f..81165bd58 100644 --- a/tests/corpus/7005-dyn-constructor-instanceof/node_modules/tag-types/index.js +++ b/tests/corpus/7005-dyn-constructor-instanceof/node_modules/tag-types/index.js @@ -59,3 +59,13 @@ module.exports.arrowCheck = function (json) { return error.name + ": " + error.message; } }; +// Every object reaches the TypeError, typed arrays, promises and Maps too. +module.exports.arrowObjects = function () { + return [new Uint8Array(2), Promise.resolve(1), new Map(), [1]].map(function (value) { + try { + return String(value instanceof make); + } catch (error) { + return error.name; + } + }).join(","); +}; diff --git a/tests/coverage-fixtures/blocked-signature-body.ts b/tests/coverage-fixtures/blocked-signature-body.ts index 7f8db4eb8..2c37484e0 100644 --- a/tests/coverage-fixtures/blocked-signature-body.ts +++ b/tests/coverage-fixtures/blocked-signature-body.ts @@ -13,3 +13,7 @@ function unused(target: WeakRef): number { return extra === undefined ? 0 : 1; } console.log(measure(new WeakRef({}), "abc")); +// A declared `this` parameter gives the body its receiver. +function described(this: { label: string }, target: WeakRef): string { + return this.label + String(target); +} diff --git a/tests/harness/coverage.test.ts b/tests/harness/coverage.test.ts index 40b55b078..3ed562aea 100644 --- a/tests/harness/coverage.test.ts +++ b/tests/harness/coverage.test.ts @@ -194,17 +194,21 @@ test("the body of a signature-blocked function reports its diagnostics", () => { "SC2020 'WeakRef' is part of the standard library types but has no scriptc lowering yet", "SC2020 'new WeakRef' is part of the standard library types but has no scriptc lowering yet", ]); - // Nothing calls unused: its signature and body report as unreached. + // Nothing calls unused or described: their signatures and bodies report + // as unreached. described's declared `this` is its receiver, so + // `this.label` reports nothing. expect(sites(coverage.unreached?.diagnostics ?? [])).toEqual([ "SC2004 uses of 'extra' inherit the blocker on its declaration", + "SC2004 uses of 'target' inherit the blocker on its declaration", + "SC2020 'WeakRef' is part of the standard library types but has no scriptc lowering yet", "SC2020 'WeakRef' is part of the standard library types but has no scriptc lowering yet", "SC2020 'new WeakRef' is part of the standard library types but has no scriptc lowering yet", ]); // The bodies' statements still count once each, as not static. const un = coverage.unreached?.stats; - expect(coverage.stats.functionsSkipped + (un?.functionsSkipped ?? 0)).toBe(2); - expect(coverage.stats.statementsTotal + (un?.statementsTotal ?? 0)).toBe(6); - expect(coverage.stats.statementsFailed + (un?.statementsFailed ?? 0)).toBe(6); + expect(coverage.stats.functionsSkipped + (un?.functionsSkipped ?? 0)).toBe(3); + expect(coverage.stats.statementsTotal + (un?.statementsTotal ?? 0)).toBe(7); + expect(coverage.stats.statementsFailed + (un?.statementsFailed ?? 0)).toBe(7); }); test("deferred fences in a module's top-level code report as running at startup", () => { From 0078dc2c46c93e6443fae1e3296456129ba190e0 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 10:10:59 -0500 Subject: [PATCH 041/197] Check the elements a spread push stores into an evolving empty array The evolving-array check made the binding checked-dynamic for any spread write, so `xs.push(...values)` of an array whose elements match the binding's agreed element type turned a typed `number[]` into a checked-dynamic array, and the spread call through it was refused with SC1090 "spread arguments in calls through 'unknown' values". A spread now stores the source array's element type, or each of a tuple's element types, and the binding stays typed when they match. A spread of another iterable still makes it checked-dynamic. --- .../src/frontend/lowering/lower-stmts.ts | 27 ++++++++++++++----- tests/corpus/7006-evolving-array-aliases.ts | 9 +++++++ 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/packages/compiler/src/frontend/lowering/lower-stmts.ts b/packages/compiler/src/frontend/lowering/lower-stmts.ts index 29086d326..cc01d7401 100644 --- a/packages/compiler/src/frontend/lowering/lower-stmts.ts +++ b/packages/compiler/src/frontend/lowering/lower-stmts.ts @@ -4800,9 +4800,9 @@ export function lowerVarDeclList( * it, and reads inside closures as the unsettled array. When every read * with a settled element type maps to the same array type, the binding has * that type. When reads disagree, a write (`xs.push(v)`, `xs.unshift(v)`, - * `xs[i] = v`) stores a value that type's element does not hold, or the - * binding is reassigned, it is checked-dynamic, as the same binding is at - * module scope. Null for other bindings. */ + * `xs.push(...values)`, `xs[i] = v`) stores a value that type's element does + * not hold, or the binding is reassigned, it is checked-dynamic, as the same + * binding is at module scope. Null for other bindings. */ function evolvingArrayType(lowerer: Lowerer, decl: ts.VariableDeclaration): IrType | null { if (!ts.isIdentifier(decl.name) || decl.initializer === undefined) return null; let initializer: ts.Expression = decl.initializer; @@ -4870,11 +4870,24 @@ function evolvingArrayType(lowerer: Lowerer, decl: ts.VariableDeclaration): IrTy result = mapped; } if (result === null || result.kind !== "array") return DYN; + // The element types a write stores: its value's, or for a spread + // (`xs.push(...values)`) the source array's element or tuple's elements. + // Another spread source stores elements this check cannot see. + const storedTypes = (value: ts.Expression): ts.Type[] | null => { + if (!ts.isSpreadElement(value)) return [lowerer.typeOf(value)]; + const source = lowerer.typeOf(value.expression); + if (lowerer.checker.isArrayType(source) || lowerer.checker.isTupleType(source)) + return [...lowerer.checker.getTypeArguments(source as ts.TypeReference)]; + return null; + }; for (const value of written) { - if (ts.isSpreadElement(value)) return DYN; - const t = lowerer.mapTypeOf(lowerer.checker.getBaseTypeOfLiteralType(lowerer.typeOf(value))); - if (!t || (!typeEquals(t, result.elem) && lowerer.widthLiftPlan(t, result.elem) === null)) - return DYN; + const stored = storedTypes(value); + if (stored === null) return DYN; + for (const type of stored) { + const t = lowerer.mapTypeOf(lowerer.checker.getBaseTypeOfLiteralType(type)); + if (!t || (!typeEquals(t, result.elem) && lowerer.widthLiftPlan(t, result.elem) === null)) + return DYN; + } } return result; } diff --git a/tests/corpus/7006-evolving-array-aliases.ts b/tests/corpus/7006-evolving-array-aliases.ts index 4810c440e..215b3f3c5 100644 --- a/tests/corpus/7006-evolving-array-aliases.ts +++ b/tests/corpus/7006-evolving-array-aliases.ts @@ -68,3 +68,12 @@ console.log(chain()); console.log(viaAlias(), "|", letAlias()); console.log(nested(true), "|", nested(false), "|", elementWrite()); console.log(agreeing(), "|", passed()); + +function spreadSame(): number { + const xs = []; + xs.push(1); + const more = [2, 3]; + xs.push(...more); + return xs.reduce((sum, v) => sum + v, 0); +} +console.log(spreadSame()); From de6ef6590f6e9678797d3eacd8453ff37a780236 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 10:12:31 -0500 Subject: [PATCH 042/197] Read optional elements at a proven integer index without a round trip An element read that may be undefined, lowered inline from `%arr.idxOr` (`a[i]` as `T | undefined`, and the read in `a[x] = a[y]!`), took an integer index only from a loop's integer slot. Any other index, such as a double counter with a proven integer range or `k | 0`, was converted and tested for an exact integer on every read. The read now takes the same exact integer as a borrowed read does (`exactInteger`), on 64-bit targets. The loop-induction treatment above the hole-free prefix still applies only to a loop's integer slot. --- .../backend/llvm/local-array-reads.test.ts | 14 ++++ .../src/backend/llvm/local-array-reads.ts | 13 +++- .../7143-optional-read-integer-index.ts.json | 6 ++ .../7143-optional-read-integer-index.ts | 70 +++++++++++++++++++ .../7143-optional-read-integer-index.ts.json | 1 + 5 files changed, 101 insertions(+), 3 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7143-optional-read-integer-index.ts.json create mode 100644 tests/corpus/7143-optional-read-integer-index.ts create mode 100644 tests/harness/corpus-costs/7143-optional-read-integer-index.ts.json diff --git a/packages/compiler/src/backend/llvm/local-array-reads.test.ts b/packages/compiler/src/backend/llvm/local-array-reads.test.ts index b09029865..7af495d42 100644 --- a/packages/compiler/src/backend/llvm/local-array-reads.test.ts +++ b/packages/compiler/src/backend/llvm/local-array-reads.test.ts @@ -940,3 +940,17 @@ test("a dense read tests the hole-free prefix before the capacity", () => { expect(ir.lastIndexOf("local.array.range", firstCap)).toBeGreaterThan(firstFilled); } }); + +test("an optional read at a proven integer index skips the double round trip", () => { + const module = fixture(); + const call = module.functions[2]!.body[0]!; + if (call.kind !== "varDecl" || call.init?.kind !== "call") throw new Error("missing call"); + call.init.args[1] = { kind: "bin", op: "|", left: ref("i", F64), right: num(0), type: F64, loc }; + expect(candidates(module).get("value")?.borrow).toBe(true); + expect(validateModule(module)).toEqual([]); + // The int32 index is the dense offset on 64-bit targets; a 32-bit target + // converts the double, as a borrowed read does. + expect(workBody(module, 64)).not.toContain("local.array.exact"); + expect(workBody(module, 64)).toContain("local.array.value"); + expect(workBody(module, 32)).toContain("local.array.exact"); +}); diff --git a/packages/compiler/src/backend/llvm/local-array-reads.ts b/packages/compiler/src/backend/llvm/local-array-reads.ts index cdd00c4e7..7a593e865 100644 --- a/packages/compiler/src/backend/llvm/local-array-reads.ts +++ b/packages/compiler/src/backend/llvm/local-array-reads.ts @@ -644,8 +644,15 @@ export function emitLocalArrayRead( ): { slot: string; type: IrType } | null { const B = host.B; const array = host.emitStableReceiver(read.array, [read.index]); - const integerIndex = inline ? host.emitIntegerLoopIndex(read.index) : null; + const loopIndex = inline ? host.emitIntegerLoopIndex(read.index) : null; const index = host.emitExpr(read.index); + // A proven integer index (a counter with a known range, say) needs no + // double round trip, as in a borrowed read. + const exact = + inline && loopIndex === null && host.sizeType === "i64" + ? exactInteger(host, index, read.index) + : null; + const integerIndex = loopIndex ?? (exact ? widenInteger(host, exact) : null); const nullable = read.type.kind === "union" ? host.nullableUnions.get(read.type.unionId) : null; if (nullable) { // A nullable union is the element pointer itself: borrowed from an @@ -665,7 +672,7 @@ export function emitLocalArrayRead( array, index, integerIndex, - integerIndex !== null, + loopIndex !== null, store, no, slow, @@ -726,7 +733,7 @@ export function emitLocalArrayRead( array, index, integerIndex, - integerIndex !== null, + loopIndex !== null, storeValue, no, slow, diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7143-optional-read-integer-index.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7143-optional-read-integer-index.ts.json new file mode 100644 index 000000000..ae0d1f610 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7143-optional-read-integer-index.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7143-optional-read-integer-index.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7143-optional-read-integer-index.ts b/tests/corpus/7143-optional-read-integer-index.ts new file mode 100644 index 000000000..43fd59a6e --- /dev/null +++ b/tests/corpus/7143-optional-read-integer-index.ts @@ -0,0 +1,70 @@ +// Element reads that may be undefined, at indices with a proven integer +// range: loop counters with an unknown limit, `i | 0`, `x >>> 0`, negative +// counters, -0, indices past the length and the capacity, and copies +// `a[x] = b[y]!` that meet holes. +function copy(src: number[], n: number): number[] { + const out: number[] = new Array(n); + for (let i = 0; i < n; i++) out[i] = src[i]!; + return out; +} +const src: number[] = [1, 2, 3, 4]; +const copied = copy(src, 6); +console.log(copied.length, copied.join(","), 4 in copied, copied[5] === undefined); + +const holes: number[] = new Array(5); +holes[0] = 10; +holes[3] = 13; +const fromHoles = copy(holes, 5); +console.log(fromHoles.join(","), 1 in fromHoles, 3 in fromHoles); + +function show(a: (string | undefined)[], from: number, to: number): string { + const out: string[] = []; + for (let i = from; i < to; i++) { + const v = a[i]; + out.push(v === undefined ? "_" : v); + } + return out.join(","); +} +const words: (string | undefined)[] = ["a", undefined, "c"]; +words[5] = "f"; +console.log(show(words, -3, 8)); +console.log(show(words, 0, 0), show(words, 2, 1)); + +function masked(a: number[], keys: number[]): string { + const out: string[] = []; + for (const k of keys) { + const v = a[k | 0]; + const w = a[k >>> 0]; + out.push(`${v === undefined ? "-" : v}/${w === undefined ? "-" : w}`); + } + return out.join(" "); +} +const values: number[] = [5, 6, 7]; +console.log(masked(values, [0, 1.9, 2, 3, -1, -0, 4294967297, 2 ** 31])); + +const negativeZero = -0; +console.log(values[negativeZero], values[0 * -1], Object.is(negativeZero, -0)); + +// A counter that starts below zero and one that passes the capacity. +let total = 0; +for (let i = -2; i < values.length + 3; i++) total += values[i] ?? 100; +console.log(total); +const big: number[] = []; +for (let k = 0; k < 20; k++) big.push(k); +big.length = 10; +let seen = 0; +for (let i = 0; i < 40; i++) if (big[i] !== undefined) seen++; +console.log(seen, big.length); + +// Rotations through the read, with a hole that becomes undefined. +function rotate(p: number[], r: number): void { + const first = p[0]!; + for (let i = 0; i < r; i++) p[i] = p[i + 1]!; + p[r] = first; +} +const ring: number[] = new Array(4); +ring[0] = 0; +ring[1] = 1; +ring[3] = 3; +rotate(ring, 3); +console.log(ring.join(","), 1 in ring, ring[1] === undefined); diff --git a/tests/harness/corpus-costs/7143-optional-read-integer-index.ts.json b/tests/harness/corpus-costs/7143-optional-read-integer-index.ts.json new file mode 100644 index 000000000..16148d8ce --- /dev/null +++ b/tests/harness/corpus-costs/7143-optional-read-integer-index.ts.json @@ -0,0 +1 @@ +{ "plain": 3500, "san": 5400 } From 3a3610d12ebfaedce85f3e877338500da9e7f8dc Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 10:15:10 -0500 Subject: [PATCH 043/197] Look up spawn and execFileSync commands on the replacement environment's PATH spawn and execFileSync with an `env` option searched the parent's PATH for a command without a '/'. They now resolve it the way spawnSync does, as Node (libuv) does: on the replacement environment's PATH, or the default path when it has none, with relative entries read from the cwd option's directory. --- .../corpus/7036-child-env-path-lookup.ts.json | 6 ++ packages/runtime/src/scr_child.c | 62 +++++++++++++------ tests/corpus/7036-child-env-path-lookup.ts | 40 ++++++++++++ .../7036-child-env-path-lookup.ts.json | 1 + 4 files changed, 89 insertions(+), 20 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7036-child-env-path-lookup.ts.json create mode 100644 tests/corpus/7036-child-env-path-lookup.ts create mode 100644 tests/harness/corpus-costs/7036-child-env-path-lookup.ts.json diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7036-child-env-path-lookup.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7036-child-env-path-lookup.ts.json new file mode 100644 index 000000000..171ab060a --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7036-child-env-path-lookup.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7036-child-env-path-lookup.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_child.c b/packages/runtime/src/scr_child.c index df60fcb26..c1d98b35b 100644 --- a/packages/runtime/src/scr_child.c +++ b/packages/runtime/src/scr_child.c @@ -3116,9 +3116,12 @@ static void scr_exec_envp_free(char **envp); * environment, found as libuv finds it: a name with a '/' is used as * given; otherwise each entry of the replacement's PATH (an empty entry * is the current directory), or of the default path when the replacement - * has no PATH, is tried in order. Returns a malloc'd path, or NULL with - * *err set: EACCES when a candidate exists but cannot run, else ENOENT. */ -static char *scr_spawn_path_lookup(const char *file, char *const *envp, int *err) { + * has no PATH, is tried in order. Relative candidates are checked from + * `cwd`, the directory the child starts in (NULL: this process's). Returns + * a malloc'd path, or NULL with *err set: EACCES when a candidate exists + * but cannot run, else ENOENT. */ +static char *scr_spawn_path_lookup(const char *file, char *const *envp, const char *cwd, + int *err) { if (strchr(file, '/')) { char *copy = strdup(file); if (!copy) scr_child_oom(); @@ -3132,6 +3135,7 @@ static char *scr_spawn_path_lookup(const char *file, char *const *envp, int *err } } size_t file_len = strlen(file); + size_t cwd_len = cwd ? strlen(cwd) : 0; bool denied = false; for (const char *dir = path;;) { const char *end = strchr(dir, ':'); @@ -3143,13 +3147,24 @@ static char *scr_spawn_path_lookup(const char *file, char *const *envp, int *err memcpy(candidate, prefix, prefix_len); candidate[prefix_len] = '/'; memcpy(candidate + prefix_len + 1, file, file_len + 1); + char *checked = candidate; + if (cwd_len > 0 && candidate[0] != '/') { + checked = malloc(cwd_len + 1 + strlen(candidate) + 1); + if (!checked) scr_child_oom(); + memcpy(checked, cwd, cwd_len); + checked[cwd_len] = '/'; + strcpy(checked + cwd_len + 1, candidate); + } struct stat st; - if (stat(candidate, &st) == 0) { - if (S_ISREG(st.st_mode) && access(candidate, X_OK) == 0) return candidate; - denied = true; + bool runs = false; + if (stat(checked, &st) == 0) { + runs = S_ISREG(st.st_mode) && access(checked, X_OK) == 0; + if (!runs) denied = true; } else if (errno == EACCES) { denied = true; } + if (checked != candidate) free(checked); + if (runs) return candidate; free(candidate); if (!end) break; dir = end + 1; @@ -3158,6 +3173,22 @@ static char *scr_spawn_path_lookup(const char *file, char *const *envp, int *err return NULL; } +/* posix_spawnp for the child_process spawns: with `envp` (a replacement + * environment) the command resolves on that environment's PATH + * (scr_spawn_path_lookup) as Node's does; without it, on this process's + * PATH over `environ`. Returns posix_spawn's error number. */ +static int scr_spawn_command(pid_t *pid, const char *file, const posix_spawn_file_actions_t *fa, + const posix_spawnattr_t *attr, char *const *argv, + char *const *envp, const char *cwd) { + if (envp == NULL) return posix_spawnp(pid, file, fa, attr, argv, environ); + int err = 0; + char *resolved = scr_spawn_path_lookup(file, envp, cwd, &err); + if (!resolved) return err; + err = posix_spawn(pid, resolved, fa, attr, argv, envp); + free(resolved); + return err; +} + /* env_pairs: NULL inherits the parent's environment; else [k,v,...] * replaces it, as Node's `env` option does, and the command is looked up * on the replacement's PATH (scr_spawn_path_lookup). */ @@ -3200,16 +3231,7 @@ static ScrSpawnRes *scr_spawn_sync_core(ScrStr *cmd, ScrArr *args, double timeou char **argv = scr_child_argv(cmd, args); char **envp = env_pairs != NULL ? scr_exec_envp(env_pairs) : NULL; pid_t pid = -1; - int spawn_err; - if (envp != NULL) { - char *resolved = scr_spawn_path_lookup(cmd->data, envp, &spawn_err); - if (resolved) { - spawn_err = posix_spawn(&pid, resolved, &fa, NULL, argv, envp); - free(resolved); - } - } else { - spawn_err = posix_spawnp(&pid, cmd->data, &fa, NULL, argv, environ); - } + int spawn_err = scr_spawn_command(&pid, cmd->data, &fa, NULL, argv, envp, NULL); posix_spawn_file_actions_destroy(&fa); free(argv); scr_exec_envp_free(envp); @@ -3556,8 +3578,8 @@ ScrStr *scr_exec_sync_core(ScrStr *cmd, ScrArr *args, const ScrExecOpts *o) { char **argv = scr_child_argv(cmd, args); char **envp = o->env_pairs != NULL ? scr_exec_envp(o->env_pairs) : NULL; pid_t pid = -1; - int spawn_err = posix_spawnp(&pid, cmd->data, &fa, NULL, argv, - envp != NULL ? envp : environ); + int spawn_err = scr_spawn_command(&pid, cmd->data, &fa, NULL, argv, envp, + o->cwd != NULL ? o->cwd->data : NULL); posix_spawn_file_actions_destroy(&fa); free(argv); scr_exec_envp_free(envp); @@ -5115,8 +5137,8 @@ ScrChild *scr_spawn_opts(ScrStr *cmd, ScrArr *args, double in_mode, char **envp = has_env && env_pairs != NULL ? scr_exec_envp(env_pairs) : NULL; pid_t pid = -1; const char *file = shell ? "/bin/sh" : cmd->data; - int spawn_err = posix_spawnp(&pid, file, &fa, &at, argv, - envp != NULL ? envp : environ); + int spawn_err = scr_spawn_command(&pid, file, &fa, &at, argv, envp, + cwd != NULL && cwd->len > 0 ? cwd->data : NULL); posix_spawnattr_destroy(&at); posix_spawn_file_actions_destroy(&fa); if (!shell) free(argv); diff --git a/tests/corpus/7036-child-env-path-lookup.ts b/tests/corpus/7036-child-env-path-lookup.ts new file mode 100644 index 000000000..cac0670e9 --- /dev/null +++ b/tests/corpus/7036-child-env-path-lookup.ts @@ -0,0 +1,40 @@ +// spawn and execFileSync with an env option find the command on the +// replacement environment's PATH, or on the default path when it has +// none, as Node does; a relative PATH entry is read from the cwd option. +import { execFileSync, spawn } from "node:child_process"; +import { chmodSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const dir = mkdtempSync(join(tmpdir(), "scr-child-path-")); +mkdirSync(join(dir, "tools")); +writeFileSync(join(dir, "tools", "child-tool"), "#!/bin/sh\necho child-tool $1\n"); +chmodSync(join(dir, "tools", "child-tool"), 0o755); + +const show = (e: unknown): string => { + const err = e as NodeJS.ErrnoException; + return `${err.code} ${err.message.replace(dir, "")}`; +}; + +console.log(execFileSync("child-tool", ["sync"], { env: { PATH: join(dir, "tools") }, encoding: "utf8" }).trim()); +console.log(execFileSync("sh", ["-c", "echo default"], { env: { A: "1" }, encoding: "utf8" }).trim()); +try { + execFileSync("sh", ["-c", "echo never"], { env: { PATH: "/nonexistent" }, encoding: "utf8" }); +} catch (e) { + console.log(show(e)); +} +console.log( + execFileSync("child-tool", ["relative"], { env: { PATH: "tools" }, cwd: dir, encoding: "utf8" }).trim(), +); + +const child = spawn("child-tool", ["async"], { env: { PATH: join(dir, "tools") }, stdio: ["ignore", "pipe", "ignore"] }); +let out = ""; +child.stdout!.on("data", (chunk: Buffer) => (out += chunk.toString())); +child.on("close", (code) => { + console.log(out.trim(), code); + const missing = spawn("sh", ["-c", "true"], { env: { PATH: "/nonexistent" }, stdio: "ignore" }); + missing.on("error", (e) => { + console.log("error", show(e)); + rmSync(dir, { recursive: true }); + }); +}); diff --git a/tests/harness/corpus-costs/7036-child-env-path-lookup.ts.json b/tests/harness/corpus-costs/7036-child-env-path-lookup.ts.json new file mode 100644 index 000000000..f9bfce7d9 --- /dev/null +++ b/tests/harness/corpus-costs/7036-child-env-path-lookup.ts.json @@ -0,0 +1 @@ +{ "plain": 2570, "san": 4110 } From 6e79566c255877f002203ac0eeeac0cd1928e9a4 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 10:18:57 -0500 Subject: [PATCH 044/197] Keep key order and absent optional fields in filter predicate record copies A record conversion behind a written type predicate built the copy in the target shape's field order and wrote every optional field, so a source object without an optional key came out with that key present and holding undefined, and Object.keys listed the target's order. The copy now keeps the source's key order, as a width copy does, and an optional field the source lacks stays absent. --- .../lowering/containers/filter-predicates.ts | 34 +++++++++++++++++-- .../7007-filter-written-type-predicates.ts | 21 ++++++++++++ 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/packages/compiler/src/frontend/lowering/containers/filter-predicates.ts b/packages/compiler/src/frontend/lowering/containers/filter-predicates.ts index df57813fe..3934edbf0 100644 --- a/packages/compiler/src/frontend/lowering/containers/filter-predicates.ts +++ b/packages/compiler/src/frontend/lowering/containers/filter-predicates.ts @@ -176,13 +176,43 @@ function conversionBody( }; const value = converted(lowerer, read, own.type, field.type, loc); if (value === null) return null; - fields.push({ name: field.name, value }); + fields.push({ + name: field.name, + value: keepAbsence(lowerer, input, from.shapeId, field, value, loc), + }); } - return [{ kind: "return", value: { kind: "recordLit", fields, type: to, loc }, loc }]; + // The copy keeps the source's own keys in the source's order, as a width + // copy does (key-order-witness.ts). + const copy: IrExpr = { kind: "recordLit", fields, type: to, loc }; + return [{ kind: "return", value: lowerer.keyOrder.copy(copy, from.shapeId), loc }]; } return null; } +/** A converted field value that keeps the source field's absence: an + * optional field the source object lacks stays absent in the copy rather + * than becoming a property that holds undefined. */ +function keepAbsence( + lowerer: Lowerer, + input: IrExpr, + shapeId: string, + field: { name: string; type: IrType }, + value: IrExpr, + loc: SrcLoc, +): IrExpr { + if (field.type.kind !== "union") return value; + if (!lowerer.unions.get(field.type.unionId)?.arms.some((arm) => arm.kind === "undefinedT")) + return value; + return { + kind: "ternary", + cond: { kind: "recordHas", obj: input, shapeId, field: field.name, type: BOOL, loc }, + then: value, + else_: { kind: "fieldAbsent", unionId: field.type.unionId, type: field.type, loc }, + type: field.type, + loc, + }; +} + /** A union value to one of its arms, or to a union of some of its arms: * each arm the target holds returns, any other arm throws. */ function unionConversion( diff --git a/tests/corpus/7007-filter-written-type-predicates.ts b/tests/corpus/7007-filter-written-type-predicates.ts index c2449b469..b75b5727d 100644 --- a/tests/corpus/7007-filter-written-type-predicates.ts +++ b/tests/corpus/7007-filter-written-type-predicates.ts @@ -85,3 +85,24 @@ function isText(v: string | undefined): v is string { } const maybe: (string | undefined)[] = ["a", undefined, "b"]; console.log(maybe.filter(isText).join("-")); + +// Narrowing a record field keeps each object's own keys, in its own order, +// and an optional field the object lacks stays absent. +interface Row { + id: unknown; + note?: string; + label: string; +} +interface NumericRow { + id: number; + note?: string; + label: string; +} +const tableRows: Row[] = [ + { label: "a", id: 1 }, + { label: "b", id: "s" }, + { label: "c", id: 3, note: "n" }, +]; +for (const row of tableRows.filter((r): r is NumericRow => typeof r.id === "number")) { + console.log(Object.keys(row).join(","), JSON.stringify(row), "note" in row, row.id + 1); +} From 8e8abbe1535e6cf808ad334ce13a49d7e7ab0308 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 10:21:53 -0500 Subject: [PATCH 045/197] Rebuild after edits to source text the output carries --- .../src/frontend/input-tracker.test.ts | 22 +++++++++++++ .../compiler/src/frontend/input-tracker.ts | 31 ++++++++++++++++--- .../src/frontend/lowering/divergences.ts | 5 ++- .../src/frontend/token-equivalence.test.ts | 12 +++++++ .../src/frontend/token-equivalence.ts | 28 +++++++++++------ .../compiler/src/frontend/ts7/ast-node.ts | 23 +++++++++++++- packages/compiler/src/native/prepare.test.ts | 30 ++++++++++++++++++ packages/compiler/src/native/prepare.ts | 5 ++- 8 files changed, 140 insertions(+), 16 deletions(-) diff --git a/packages/compiler/src/frontend/input-tracker.test.ts b/packages/compiler/src/frontend/input-tracker.test.ts index 4b6ef6d90..b4f3957f5 100644 --- a/packages/compiler/src/frontend/input-tracker.test.ts +++ b/packages/compiler/src/frontend/input-tracker.test.ts @@ -60,6 +60,28 @@ test("semantic validation reads non-source dependencies once and still invalidat expect(reads.mock.calls.filter(([path]) => path === dependency)).toHaveLength(1); }); +test("an equivalent edit of a file whose raw text the frontend read does not match", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-inputs-")); + scratch.push(dir); + const source = join(dir, "entry.ts"); + const previous = "export const answer = 1;\n"; + await writeFile(source, previous); + const tracker = new FrontendInputTracker(); + tracker.run(() => { + trackedReadFile(source); + tracker.record({ op: "text", path: source }); + }); + const snapshot = tracker.snapshot(); + expect(validFrontendInputSnapshot(snapshot)).toBe(true); + expect(frontendInputsStillMatch(snapshot)).toBe(true); + await writeFile(source, `// comment\n${previous}`); + const equivalent = vi.fn(() => true); + expect( + frontendInputsSemanticallyMatch(snapshot, new Map([[source, previous]]), equivalent), + ).toBeNull(); + expect(equivalent).not.toHaveBeenCalled(); +}); + test("tracked frontend reads invalidate on byte edits", async () => { const dir = await mkdtemp(join(tmpdir(), "scriptc-inputs-")); scratch.push(dir); diff --git a/packages/compiler/src/frontend/input-tracker.ts b/packages/compiler/src/frontend/input-tracker.ts index ed2f7b176..d27ab6285 100644 --- a/packages/compiler/src/frontend/input-tracker.ts +++ b/packages/compiler/src/frontend/input-tracker.ts @@ -2,6 +2,7 @@ import { createHash } from "node:crypto"; import { AsyncLocalStorage } from "node:async_hooks"; import { readFileSync, readdirSync, realpathSync, statSync } from "node:fs"; import { basename, dirname, resolve } from "node:path"; +import { observeSourceText } from "./ts7/ast-node.js"; /** * Filesystem observations made while constructing and lowering one frontend @@ -16,7 +17,10 @@ export type FrontendInputProbe = | { op: "kind"; path: string; kind: "file" | "directory" | "other" | "missing" } | { op: "entries"; path: string; files: string[]; directories: string[] } | { op: "entries-error"; path: string } - | { op: "realpath"; path: string; target: string | null }; + | { op: "realpath"; path: string; target: string | null } + /** The frontend's output depends on the file's raw text or line + * positions, not only on its tokens (observeSourceText). */ + | { op: "text"; path: string }; export interface FrontendInputSnapshot { version: 1; @@ -105,6 +109,7 @@ export class FrontendInputTracker { record(probe: FrontendInputProbe): void { const key = `${probe.op}\0${probe.path}`; + if (probe.op === "text" && this.probes.has(key)) return; const previous = this.probes.get(key); if (previous !== undefined && JSON.stringify(previous) !== JSON.stringify(probe)) { // An input changed while this frontend was running. The build may @@ -158,6 +163,13 @@ function record(probe: FrontendInputProbe): void { (synchronousTracker ?? activeTracker.getStore())?.record(probe); } +// Source text the frontend reads raw (call-site text in messages, comptime +// callbacks, line numbers) changes with edits that keep the tokens. +observeSourceText((fileName) => { + const tracker = synchronousTracker ?? activeTracker.getStore(); + if (tracker !== undefined) tracker.record({ op: "text", path: resolve(fileName) }); +}); + /** Mark the active frontend as unsafe to persist because an exact host query * bypassed the tracked filesystem wrappers. No-op outside a frontend run. */ export function markFrontendInputsUnstable(): void { @@ -394,6 +406,9 @@ export function frontendInputsStillMatch( return probe.target === null; } } + case "text": + // The file probe of the same path checks its content. + return true; } }); } @@ -411,7 +426,11 @@ export function frontendInputsSemanticallyMatch( ): FrontendSemanticMatch | null { if (!validFrontendInputSnapshot(snapshot)) return null; const sourceProbes = new Map>(); - for (const probe of snapshot.probes) if (probe.op === "file") sourceProbes.set(probe.path, probe); + const rawText = new Set(); + for (const probe of snapshot.probes) { + if (probe.op === "file") sourceProbes.set(probe.path, probe); + else if (probe.op === "text") rawText.add(probe.path); + } for (const [path, previous] of previousSources) { if (sourceProbes.get(path)?.digest !== frontendSourceDigest(previous)) return null; } @@ -436,8 +455,10 @@ export function frontendInputsSemanticallyMatch( probes.push(probe); continue; } - // One edit that is not equivalent already rules out the match. - if (!isSemanticEquivalent(probe.path, previous, current)) return null; + // One edit that is not equivalent already rules out the match, and so + // does any edit of a file whose raw text the frontend read. + if (rawText.has(probe.path) || !isSemanticEquivalent(probe.path, previous, current)) + return null; changed.push({ path: probe.path, previous, current }); probes.push({ ...probe, digest: currentDigest }); } @@ -488,6 +509,8 @@ export function validFrontendInputSnapshot(snapshot: unknown): snapshot is Front return true; case "realpath": return value.target === null || typeof value.target === "string"; + case "text": + return true; default: return false; } diff --git a/packages/compiler/src/frontend/lowering/divergences.ts b/packages/compiler/src/frontend/lowering/divergences.ts index 1c362cb09..c971c672d 100644 --- a/packages/compiler/src/frontend/lowering/divergences.ts +++ b/packages/compiler/src/frontend/lowering/divergences.ts @@ -464,8 +464,11 @@ function identityUse(root: ts.Node, name: string, from = -1): IdentityUse | null cur = cur.expression; return cur; }; + // The text appears only in warnings. A build that reported warnings is + // not reused after a source edit, so the read is not recorded as a source + // text dependency (ts.getTextOfNode would record it). const textOf = (node: ts.Node): string => { - const text = ts.getTextOfNode(node); + const text = ts.getSourceFileOfNode(node).text.slice(ts.tokenStartOfNode(node), node.end); return text.length > 60 ? `${text.slice(0, 57)}...` : text; }; const visit = (node: ts.Node): "skip" | "stop" | undefined => { diff --git a/packages/compiler/src/frontend/token-equivalence.test.ts b/packages/compiler/src/frontend/token-equivalence.test.ts index 2c7b8ea0a..afd81d9b6 100644 --- a/packages/compiler/src/frontend/token-equivalence.test.ts +++ b/packages/compiler/src/frontend/token-equivalence.test.ts @@ -28,6 +28,18 @@ test("comment-like text inside regular expressions is code", () => { ); expect(equivalent("const q = a / b; // ratio\n", "const q = a / b; // the ratio\n")).toBe(true); expect(equivalent("return /a/; // one\n", "return /a/; // two\n")).toBe(true); + // A `for await` header closes like a `for` header. + expect( + equivalent( + "for await (const x of xs) /[//]/.test(x); c = 1;\n", + "for await (const x of xs) /[//]/.test(x); c = 2;\n", + ), + ).toBe(false); +}); + +test("U+2028 and U+2029 end a line comment, so their files are never equivalent", () => { + expect(equivalent("// a\u2028c = 1;\n", "// a\u2028c = 2;\n")).toBe(false); + expect(equivalent("/* a\u2029*/ c = 1;\n", "/* b\u2029*/ c = 1;\n")).toBe(false); }); test("comments that can affect compilation are kept", () => { diff --git a/packages/compiler/src/frontend/token-equivalence.ts b/packages/compiler/src/frontend/token-equivalence.ts index 6387a0e4b..94c60d836 100644 --- a/packages/compiler/src/frontend/token-equivalence.ts +++ b/packages/compiler/src/frontend/token-equivalence.ts @@ -15,12 +15,14 @@ * annotations lowering reads). * - A `/` starts a regular expression or divides according to the token * before it. After `)` it divides unless the parenthesis closes an `if`, - * `while`, `for`, or `with` header. After `}`, `++` or `--` the scanner + * `while`, `for`, `for await`, or `with` header. After `}`, `++` or `--` the scanner * cannot tell which, so it gives up, as it does for escaped identifiers, * which can spell keywords. * - Template literals track their `${` substitutions on a brace stack. - * - JSX files, unterminated strings, comments, templates and regular - * expressions, and any line terminator outside ASCII give up. + * - JSX files, files that contain U+2028 or U+2029 anywhere (they end a + * line comment and separate lines for automatic semicolon insertion), + * and unterminated strings, comments, templates and regular expressions + * give up. * * Plain character scans keep the scanner free of Unicode property escapes: * any non-ASCII code unit outside strings, templates, regular expressions @@ -118,6 +120,7 @@ function isSpace(code: number): boolean { /** The tokens of `source`, or null when the scanner cannot classify it. */ function scan(path: string, source: string): Token[] | null { if (path.endsWith(".tsx") || path.endsWith(".jsx")) return null; + if (source.includes("\u2028") || source.includes("\u2029")) return null; const typeScript = isTypeScript(path); const tokens: Token[] = []; // Brace kinds: true for a template substitution, false otherwise. @@ -125,13 +128,17 @@ function scan(path: string, source: string): Token[] | null { // Paren kinds: true for a statement header. const parens: boolean[] = []; let lineBreakBefore = false; - // The previous token, and whether a `)` closed a statement header. - const last: { token: string | null } = { token: null }; + // The previous token and the one before it, and whether a `)` closed a + // statement header. + const last: { token: string | null; before: string | null } = { token: null, before: null }; let closedHeader = false; const push = (text: string, comment = false): void => { tokens.push({ text, comment, lineBreakBefore }); lineBreakBefore = false; - if (!comment) last.token = text; + if (!comment) { + last.before = last.token; + last.token = text; + } }; let index = 0; const length = source.length; @@ -162,6 +169,7 @@ function scan(path: string, source: string): Token[] | null { index = position; push(source.slice(start, position)); braces.push(true); + last.before = last.token; last.token = "${"; return true; } @@ -176,8 +184,7 @@ function scan(path: string, source: string): Token[] | null { index++; continue; } - if (code > 127 && (code === 0x2028 || code === 0x2029 || code === 0xa0 || code === 0xfeff)) - return null; + if (code > 127 && (code === 0xa0 || code === 0xfeff)) return null; const next = source.charCodeAt(index + 1); // Comments. if (code === 47 && (next === 47 || next === 42)) { @@ -305,7 +312,10 @@ function scan(path: string, source: string): Token[] | null { } // Brackets. if (code === 40) { - parens.push(last.token !== null && HEADER_KEYWORDS.has(last.token)); + parens.push( + last.token !== null && + (HEADER_KEYWORDS.has(last.token) || (last.token === "await" && last.before === "for")), + ); index++; push("("); closedHeader = false; diff --git a/packages/compiler/src/frontend/ts7/ast-node.ts b/packages/compiler/src/frontend/ts7/ast-node.ts index be2745616..40e77e6f1 100644 --- a/packages/compiler/src/frontend/ts7/ast-node.ts +++ b/packages/compiler/src/frontend/ts7/ast-node.ts @@ -17,6 +17,16 @@ import { import type { SourceFile } from "./ast-types.js"; import { SyntaxKind, NodeFlags } from "./enums.js"; +/** Receives the file name of a source file whose raw text or line + * positions a reader takes: the text of a node other than a token + * (getText), a node's text with its trivia (getFullText), or line starts. */ +let sourceTextObserver: ((fileName: string) => void) | undefined; + +/** Installs the observer of raw source text reads (sourceTextObserver). */ +export function observeSourceText(observer: ((fileName: string) => void) | undefined): void { + sourceTextObserver = observer; +} + /** A list of child nodes with the list's own position, the shape of tsc-ts's * NodeList. The file keeps the nodes array, not this object: each read of a * list field returns a new AstNodeList over the same array, so `nodes` keeps @@ -202,7 +212,14 @@ export class AstFile { return result; } + /** Reports a read of this file's raw text or line positions. */ + noteSourceText(): void { + if (sourceTextObserver !== undefined && this.root.kind === SyntaxKind.SourceFile) + sourceTextObserver(this.root.fileName); + } + lineStarts(): number[] { + this.noteSourceText(); if (this.lines === undefined) this.lines = astLineStarts(this.root.text ?? ""); return this.lines; } @@ -462,10 +479,14 @@ export class AstNode { return this.getStart(sourceFile) - this.pos; } getFullText(sourceFile?: AstNode): string { - return ((sourceFile ?? this.file.root).text ?? "").substring(this.pos, this.end); + const source = sourceFile ?? this.file.root; + source.file.noteSourceText(); + return (source.text ?? "").substring(this.pos, this.end); } getText(sourceFile?: AstNode): string { const source = sourceFile ?? this.file.root; + // A token's text is the same in every token-equivalent version. + if (this.kind > SyntaxKind.LastToken) source.file.noteSourceText(); return (source.text ?? "").substring(this.getStart(source), this.end); } diff --git a/packages/compiler/src/native/prepare.test.ts b/packages/compiler/src/native/prepare.test.ts index 35e866d6a..78d451f46 100644 --- a/packages/compiler/src/native/prepare.test.ts +++ b/packages/compiler/src/native/prepare.test.ts @@ -113,6 +113,36 @@ test("comment and whitespace edits reuse the frontend output a fresh build repro expect(edited.ok && edited.llvm).not.toBe(first.llvm); }); +test("a comment edit inside source text the output carries is rebuilt", () => { + const f = fixture(); + // A failing assert(value) reports its call's source text. + const source = (call: string) => + `import assert from "node:assert";\nexport const value = "first";\n${call};\n`; + writeFileSync(f.dependency, source("assert(value.length > 0)")); + const first = f.prepare(); + expect(first.ok, JSON.stringify(first)).toBe(true); + if (!first.ok) return; + writeFileSync(f.dependency, source("assert(value.length > 0 /* checked */)")); + const edited = f.prepare(); + expect(f.loads()).toBe(2); + expect(edited.ok && edited.llvm).not.toBe(first.llvm); +}); + +test("a whitespace edit that moves a reported source line is rebuilt", () => { + const f = fixture(); + // node:test reports the file, line and column of each test. + const source = (gap: string) => + `import { test } from "node:test";\nexport const value = "first";\n${gap}test("value", () => {});\n`; + writeFileSync(f.dependency, source("")); + const first = f.prepare(); + expect(first.ok, JSON.stringify(first)).toBe(true); + if (!first.ok) return; + writeFileSync(f.dependency, source("\n")); + const edited = f.prepare(); + expect(f.loads()).toBe(2); + expect(edited.ok && edited.llvm).not.toBe(first.llvm); +}); + test("structurally invalid cached features cause a rebuild even with a valid digest", () => { const f = fixture(); expect(f.prepare().ok).toBe(true); diff --git a/packages/compiler/src/native/prepare.ts b/packages/compiler/src/native/prepare.ts index 53b22cfaa..261879194 100644 --- a/packages/compiler/src/native/prepare.ts +++ b/packages/compiler/src/native/prepare.ts @@ -114,7 +114,10 @@ function readFrontendCache( * that cannot affect compilation; see token-equivalence.ts). Equivalent * sources lower to the same IR apart from source locations, which only dev * builds' debug information and the serialized IR carry. Diagnostics carry - * locations too, so an entry that reported warnings is rebuilt. */ + * locations too, so an entry that reported warnings is rebuilt. A changed + * source whose raw text or line positions the frontend read (call-site text + * in messages, comptime callbacks, test locations) is rebuilt as well: + * frontendInputsSemanticallyMatch refuses its `text` probe. */ function equivalentFrontendInput( stale: FrontendCacheEntry, options: CompileRequestOptions, From ccfca67aecc819a66e38baafe01e213f99f1337f Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 10:30:56 -0500 Subject: [PATCH 046/197] Fix chalk builder field calls and hot-shots send failures and tcp completion --- packages/compiler/shims/hot-shots/client.ts | 23 ++++++++++--- .../src/frontend/lowering/lower-calls.ts | 4 +++ .../npm/cases/chalk5-builder/main.ts.json | 1 + .../fixtures/npm/cases/chalk5/main.ts.json | 1 + .../npm/cases/hot-shots-buffered/main.ts.json | 2 ++ .../cases/hot-shots-send-error/main.ts.json | 8 +++++ .../npm/cases/hot-shots-tcp/main.ts.json | 8 +++++ .../npm/cases/hot-shots-udp/main.ts.json | 2 ++ packages/runtime/src/scr_dgram.h | 7 ++++ .../fixtures/npm/cases/chalk5-builder/main.ts | 5 ++- .../npm/cases/hot-shots-send-error/main.ts | 27 +++++++++++++++ .../cases/hot-shots-send-error/package.json | 1 + .../fixtures/npm/cases/hot-shots-tcp/main.ts | 33 +++++++++++++++++++ .../npm/cases/hot-shots-tcp/package.json | 1 + 14 files changed, 118 insertions(+), 5 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/hot-shots-send-error/main.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/hot-shots-tcp/main.ts.json create mode 100644 tests/fixtures/npm/cases/hot-shots-send-error/main.ts create mode 100644 tests/fixtures/npm/cases/hot-shots-send-error/package.json create mode 100644 tests/fixtures/npm/cases/hot-shots-tcp/main.ts create mode 100644 tests/fixtures/npm/cases/hot-shots-tcp/package.json diff --git a/packages/compiler/shims/hot-shots/client.ts b/packages/compiler/shims/hot-shots/client.ts index b407132f4..354744e7b 100644 --- a/packages/compiler/shims/hot-shots/client.ts +++ b/packages/compiler/shims/hot-shots/client.ts @@ -179,6 +179,8 @@ interface Transport { close(): void; onError(listener: (err: Error) => void): void; offError(listener: (err: Error) => void): void; + /** Emits 'error' on the socket, as hot-shots' `socket.emit('error')`. */ + emitError(err: Error): void; onClose(listener: () => void): void; } @@ -261,6 +263,9 @@ class UdpTransport implements Transport { offError(listener: (err: Error) => void): void { this.errors.remove(listener); } + emitError(err: Error): void { + this.errors.emit(err); + } onClose(listener: () => void): void { this.socket.on("close", listener); } @@ -281,10 +286,10 @@ class TcpTransport implements Transport { this.socket.on("error", (err) => errors.emit(err)); } send(buf: Buffer, callback: SendCallback): void { - // The typed socket's write takes the data alone; the completion runs - // on the next tick. - this.socket.write(addEol(buf)); - process.nextTick(() => callback(null)); + // hot-shots writes the line with the 'ascii' encoding, which encodes + // strings as 'latin1' does. The callback runs once the data is written, + // so a message held back by a full socket buffer stays in flight. + this.socket.write(Buffer.from(addEol(buf), "latin1"), (err) => callback(err ?? null)); } close(): void { this.socket.destroy(); @@ -295,6 +300,9 @@ class TcpTransport implements Transport { offError(listener: (err: Error) => void): void { this.errors.remove(listener); } + emitError(err: Error): void { + this.errors.emit(err); + } onClose(listener: () => void): void { this.socket.on("close", () => listener()); } @@ -376,6 +384,9 @@ class UdsTransport implements Transport { offError(listener: (err: Error) => void): void { this.errors.remove(listener); } + emitError(err: Error): void { + this.errors.emit(err); + } onClose(listener: () => void): void { this.closeListeners.push(listener); } @@ -999,6 +1010,10 @@ export class StatsD { handler(errFormatted); } else { console.error(String(errFormatted)); + // hot-shots also emits the error on the current socket, where the + // uds reconnection handler listens; with no listener it throws. + const current = this.transport; + if (current !== null) current.emitError(errFormatted); } } }; diff --git a/packages/compiler/src/frontend/lowering/lower-calls.ts b/packages/compiler/src/frontend/lowering/lower-calls.ts index b650ca567..301670c3a 100644 --- a/packages/compiler/src/frontend/lowering/lower-calls.ts +++ b/packages/compiler/src/frontend/lowering/lower-calls.ts @@ -9250,6 +9250,10 @@ export function lowerRecordFieldCall( if (callee.type.kind === "object" && !init) { const direct = callableInstanceStringCall(lowerer, call, callee, loc); if (direct) return direct; + // Other argument counts call the %call closure through its value, as + // a call on the builder itself does (`theme.error("a", "b")`). + const closure = lowerer.hybridCallUnwrap(callee); + if (closure !== callee) return lowerCallThroughValue(lowerer, call, closure, loc); } if (callee.type.kind === "record" || callee.type.kind === "object") callee = lowerer.hybridCallUnwrap(callee); diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/chalk5-builder/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/chalk5-builder/main.ts.json index 545c4d701..8f7268daf 100644 --- a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/chalk5-builder/main.ts.json +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/chalk5-builder/main.ts.json @@ -1,5 +1,6 @@ { "order": [ + "/packages/compiler/shims/chalk/v5.ts", "/tests/fixtures/npm/cases/chalk5-builder/main.ts" ], "diags": [] diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/chalk5/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/chalk5/main.ts.json index 065c96347..8365d4db9 100644 --- a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/chalk5/main.ts.json +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/chalk5/main.ts.json @@ -1,5 +1,6 @@ { "order": [ + "/packages/compiler/shims/chalk/v5.ts", "/tests/fixtures/npm/cases/chalk5/main.ts" ], "diags": [] diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/hot-shots-buffered/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/hot-shots-buffered/main.ts.json index e60660c09..a48e1af78 100644 --- a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/hot-shots-buffered/main.ts.json +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/hot-shots-buffered/main.ts.json @@ -1,5 +1,7 @@ { "order": [ + "/packages/compiler/shims/hot-shots/client.ts", + "/packages/compiler/shims/hot-shots/v8.ts", "/tests/fixtures/npm/cases/hot-shots-buffered/main.ts" ], "diags": [] diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/hot-shots-send-error/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/hot-shots-send-error/main.ts.json new file mode 100644 index 000000000..716cfe02f --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/hot-shots-send-error/main.ts.json @@ -0,0 +1,8 @@ +{ + "order": [ + "/packages/compiler/shims/hot-shots/client.ts", + "/packages/compiler/shims/hot-shots/v8.ts", + "/tests/fixtures/npm/cases/hot-shots-send-error/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/hot-shots-tcp/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/hot-shots-tcp/main.ts.json new file mode 100644 index 000000000..f131361e8 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/hot-shots-tcp/main.ts.json @@ -0,0 +1,8 @@ +{ + "order": [ + "/packages/compiler/shims/hot-shots/client.ts", + "/packages/compiler/shims/hot-shots/v8.ts", + "/tests/fixtures/npm/cases/hot-shots-tcp/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/hot-shots-udp/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/hot-shots-udp/main.ts.json index 54baff890..a3014fb9f 100644 --- a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/hot-shots-udp/main.ts.json +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/hot-shots-udp/main.ts.json @@ -1,5 +1,7 @@ { "order": [ + "/packages/compiler/shims/hot-shots/client.ts", + "/packages/compiler/shims/hot-shots/v8.ts", "/tests/fixtures/npm/cases/hot-shots-udp/main.ts" ], "diags": [] diff --git a/packages/runtime/src/scr_dgram.h b/packages/runtime/src/scr_dgram.h index 8396f0693..08db077df 100644 --- a/packages/runtime/src/scr_dgram.h +++ b/packages/runtime/src/scr_dgram.h @@ -55,6 +55,13 @@ double scr_dgram_addr_port(ScrDgramSocket *s); void scr_dgram_close(ScrDgramSocket *s, ScrClosure *cb /*moves, nullable*/); /* throws on closed */ void scr_dgram_unref(ScrDgramSocket *s); void scr_dgram_ref(ScrDgramSocket *s); +/* Unix datagram sockets for the hot-shots package shim (the unix-dgram + * addon's surface): each answers its result (fd, 0) or a negative errno; + * send answers 1 when the socket would block. */ +double scr_dgram_unix_socket(void); +double scr_dgram_unix_connect(double fd, ScrStr *path /*borrowed*/); +double scr_dgram_unix_send(double fd, ScrBytes *data /*borrowed*/); +double scr_dgram_unix_close(double fd); /* Socket options: Node's checks and errors (ErrnoException " * ", ERR_SOCKET_BAD_BUFFER_SIZE, ERR_SOCKET_BUFFER_SIZE, the * TypeError after close()). The TTL setters and setMulticastLoopback diff --git a/tests/fixtures/npm/cases/chalk5-builder/main.ts b/tests/fixtures/npm/cases/chalk5-builder/main.ts index 8f2623717..2a4fff61b 100644 --- a/tests/fixtures/npm/cases/chalk5-builder/main.ts +++ b/tests/fixtures/npm/cases/chalk5-builder/main.ts @@ -1,6 +1,7 @@ // chalk 5 builders used as values (the package compiles from scriptc's // built-in shim): builders in function slots and arrays, styles picked at -// run time, visible at level 0, chalkStderr, and the root's own call. +// run time, builders in record fields called with any number of arguments, +// visible at level 0, chalkStderr, and the root's own call. import chalk, { Chalk, chalkStderr, type ChalkInstance, type ForegroundColorName } from "chalk"; const show = (label: string, value: string): void => { @@ -36,6 +37,8 @@ const off = new Chalk({ level: 0 }); show("visible off", off.visible("v") + "|" + off.red.visible("v") + "|" + off.red("plain")); show("root call", c("a", 1, true) + "|" + c() + "|" + c(undefined) + "|" + c(null, "n")); show("builder args", c.red(undefined) + "|" + c.red(null) + "|" + c.red(1, 2) + "|" + c.red()); +const theme = { error: c.red, note: c.bold.blue }; +show("field args", theme.error("a", "b") + "|" + theme.note("x", 1, true) + "|" + theme.error("one") + "|" + theme.error()); show("multiline", c.cyan("one\r\ntwo\nthree")); show("stderr level", String(chalkStderr.level >= 0)); show("rgb levels", [1, 2, 3].map((level) => new Chalk({ level: level as 1 | 2 | 3 }).rgb(255, 136, 0)("o")).join("|")); diff --git a/tests/fixtures/npm/cases/hot-shots-send-error/main.ts b/tests/fixtures/npm/cases/hot-shots-send-error/main.ts new file mode 100644 index 000000000..e829c5e33 --- /dev/null +++ b/tests/fixtures/npm/cases/hot-shots-send-error/main.ts @@ -0,0 +1,27 @@ +// The hot-shots StatsD client (compiled from scriptc's built-in shim) when a +// send fails: with a callback or an errorHandler the error goes there; with +// neither, the client logs it and emits it on the socket, where an 'error' +// event nobody listens for ends the process with status 1. +import { StatsD } from "hot-shots"; + +// A datagram larger than udp allows fails with EMSGSIZE before it leaves +// the host; port 9 keeps the message the same on every run. +const big = "x".repeat(70000); + +const withCallback = new StatsD({ host: "127.0.0.1", port: 9 }); +withCallback.increment(big, 1, 1, [], (err: Error | null | undefined) => { + console.log("callback:", err ? err.message : "no error"); + withCallback.close(); + + const withHandler = new StatsD({ + host: "127.0.0.1", + port: 9, + errorHandler: (error: Error) => { + console.log("errorHandler:", error.message); + withHandler.close(); + console.log("without a callback or errorHandler"); + new StatsD({ host: "127.0.0.1", port: 9 }).increment(big); + }, + }); + withHandler.increment(big); +}); diff --git a/tests/fixtures/npm/cases/hot-shots-send-error/package.json b/tests/fixtures/npm/cases/hot-shots-send-error/package.json new file mode 100644 index 000000000..5ffd9800b --- /dev/null +++ b/tests/fixtures/npm/cases/hot-shots-send-error/package.json @@ -0,0 +1 @@ +{ "type": "module" } diff --git a/tests/fixtures/npm/cases/hot-shots-tcp/main.ts b/tests/fixtures/npm/cases/hot-shots-tcp/main.ts new file mode 100644 index 000000000..0812c90f1 --- /dev/null +++ b/tests/fixtures/npm/cases/hot-shots-tcp/main.ts @@ -0,0 +1,33 @@ +// The hot-shots StatsD client over tcp (compiled from scriptc's built-in +// shim): a send completes once the socket has written its line, so close() +// waits for lines that a full socket buffer holds back, and the receiver +// gets every line before the connection ends. +import * as net from "node:net"; +import { StatsD } from "hot-shots"; + +const LINES = 20000; +const name = "m".repeat(1000); +let bytes = 0; +let ended: () => void = () => {}; +const serverDone = new Promise((resolve) => { + ended = resolve; +}); +const server = net.createServer((socket) => { + socket.pause(); + setTimeout(() => socket.resume(), 150); + socket.on("data", (chunk: Buffer) => { + bytes += chunk.length; + }); + socket.on("end", () => ended()); +}); + +server.listen(0, "127.0.0.1", () => { + const port = (server.address() as net.AddressInfo).port; + const client = new StatsD({ host: "127.0.0.1", port, protocol: "tcp" }); + for (let i = 0; i < LINES; i++) client.increment(name); + const closed = new Promise((resolve) => client.close(() => resolve())); + void Promise.all([closed, serverDone]).then(() => { + console.log("closed; received", bytes, "of", LINES * (name.length + 5), "bytes"); + server.close(() => console.log("server closed")); + }); +}); diff --git a/tests/fixtures/npm/cases/hot-shots-tcp/package.json b/tests/fixtures/npm/cases/hot-shots-tcp/package.json new file mode 100644 index 000000000..5ffd9800b --- /dev/null +++ b/tests/fixtures/npm/cases/hot-shots-tcp/package.json @@ -0,0 +1 @@ +{ "type": "module" } From a8a545a81f9fd6aa5132977d8e26715d4c7df462 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 10:36:22 -0500 Subject: [PATCH 047/197] Read the characters of ASCII strings in versioned scan loops as bytes `for (let i = 0; i < s.length; i++) if (s.charCodeAt(i) === 32) n++` tested the string's ASCII bit and the index against the length on every character, and its fallback is an opaque runtime call, so LLVM could not hoist either test or vectorize the loop. A typed-array loop version now also covers `s.charCodeAt(index)` of a string binding the loop does not write, at a counter index plus a constant or another index the version proves. A string is immutable, so its bytes, its length and its ASCII bit are fixed when it is built. The guard requires the ASCII bit, so each UTF-16 unit is one byte and the length is the byte length, and the fast copy reads each unit with one byte load. Other strings and indices run the checked loop. --- .../backend/llvm/bytes-loop-versions.test.ts | 69 ++++++++++ .../src/backend/llvm/bytes-loop-versions.ts | 41 ++++-- packages/compiler/src/backend/llvm/emitter.ts | 1 + .../src/backend/llvm/expr-containers.ts | 7 +- .../compiler/src/backend/llvm/string-reads.ts | 38 ++++++ .../7302-string-scan-loop-versions.ts.json | 6 + .../corpus/7302-string-scan-loop-versions.ts | 120 ++++++++++++++++++ .../7302-string-scan-loop-versions.ts.json | 1 + 8 files changed, 273 insertions(+), 10 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7302-string-scan-loop-versions.ts.json create mode 100644 tests/corpus/7302-string-scan-loop-versions.ts create mode 100644 tests/harness/corpus-costs/7302-string-scan-loop-versions.ts.json diff --git a/packages/compiler/src/backend/llvm/bytes-loop-versions.test.ts b/packages/compiler/src/backend/llvm/bytes-loop-versions.test.ts index ace8f294c..2e22832a1 100644 --- a/packages/compiler/src/backend/llvm/bytes-loop-versions.test.ts +++ b/packages/compiler/src/backend/llvm/bytes-loop-versions.test.ts @@ -99,6 +99,7 @@ function match( f: IrFunction, loop: IrStmt & { kind: "for" }, captured: string[] = [], + scalarStrings: string[] = [], ): BytesLoopVersion | null { const ranges = analyzeIntegerRanges(f); const locals = new Map(f.locals.map((l) => [l.id, l])); @@ -111,6 +112,7 @@ function match( proven: new Set(), workers: false, owned: new Set(), + scalarStrings: new Set(scalarStrings), }); } @@ -425,3 +427,70 @@ test("bodies compiled without optimization keep one checked loop", () => { expect(body).not.toContain("bytes.version"); expect(body).toContain("bytes.index.invalid"); }); + +const STR: IrType = { kind: "string" }; +const charCode = (receiver: IrExpr, index: IrExpr): IrExpr => ({ + kind: "strIntrinsic", + method: "charCodeAt", + receiver, + args: [index], + type: F64, + loc, +}); + +/** The kernel with a string parameter `text`. */ +function textKernel(body: IrStmt[], mutable = false): IrFunction { + const kernel = fn(body, [local("text", STR, mutable)]); + kernel.params.push({ localId: "text", name: "text", type: STR }); + return kernel; +} + +test("charCodeAt reads of a string binding version with the typed-array reads", () => { + const read = charCode(ref("text", STR), bin("+", ref("i"), num(1))); + const loop = counted([set(ref("out", f64s), ref("i"), read)]); + const version = match(textKernel([loop]), loop)!; + expect(version.bounds.has(read)).toBe(true); + expect( + version.receivers.map((r) => [r.ref.kind === "varRef" && r.ref.localId, r.string]), + ).toEqual([ + ["out", false], + ["text", true], + ]); + expect(version.affine.map((c) => [c.receiver, c.minOffset, c.maxEnd])).toEqual([ + [0, 0, 0], + [1, 1, 1], + ]); + // A string the loop rebinds keeps its checked reads. + const rebinding = counted([ + set(ref("out", f64s), ref("i"), charCode(ref("text", STR), ref("i"))), + { kind: "assign", localId: "text", value: ref("text", STR), loc }, + ]); + const rebound = match(textKernel([rebinding], true), rebinding)!; + expect(rebound.receivers.map((r) => r.string)).toEqual([false]); + // A string held as a slice of another string has no header to test. + const sliced = match(textKernel([loop]), loop, [], ["text"])!; + expect(sliced.receivers.map((r) => r.string)).toEqual([false]); + expect(sliced.bounds.has(read)).toBe(false); +}); + +test("the guard tests a string's ASCII bit and the fast copy reads bytes", () => { + const loop = counted([set(ref("out", f64s), ref("i"), charCode(ref("text", STR), ref("i")))]); + const kernel = textKernel([loop]); + const mod: IrModule = { + irVersion: 17, + sourceFile: loc.file, + entry: "__main", + functions: [ + { name: "__main", params: [], returnType: VOID, locals: [], body: [], loc }, + kernel, + ], + }; + const body = /^define internal [^\n]*@sc_bf_kernel[^]*?^}/m.exec(emitLlvmModule(mod))![0]; + const guard = body.slice(0, body.search(/^bytes\.version\.slow\d*:/m)); + expect(guard).toMatch(/load i32, ptr %\S+\n\s+%\S+ = icmp slt i32 %\S+, 0/); + const fast = body.slice(body.lastIndexOf("bytes.version.fast")); + expect(fast).toMatch(/load i8, ptr %\S+\n\s+%\S+ = uitofp i8/); + expect(fast).not.toContain("@sc_str_char_code_at_int"); + // The checked copy keeps the inline helper and its fallback. + expect(body).toContain("@sc_str_char_code_at_int"); +}); diff --git a/packages/compiler/src/backend/llvm/bytes-loop-versions.ts b/packages/compiler/src/backend/llvm/bytes-loop-versions.ts index c3cee3feb..ad64260a2 100644 --- a/packages/compiler/src/backend/llvm/bytes-loop-versions.ts +++ b/packages/compiler/src/backend/llvm/bytes-loop-versions.ts @@ -8,6 +8,7 @@ import { exactInteger, widenInteger } from "./integer-values.js"; import { emitCountedLoopLimit, emitCountedLoopSafety } from "./counted-loops.js"; import { emitBytesHeaderLength, emitBytesSharedStorage } from "./expr-bytes.js"; import { f64Lit } from "./common.js"; +import { emitStringAsciiFlag, emitStringByteLength } from "./string-reads.js"; type Node = IrExpr | IrStmt; type VarRef = IrExpr & { kind: "varRef" }; @@ -45,8 +46,9 @@ export interface BytesLoopVersion { /** Worker programs: typed-array accesses whose views the guard proves * thread-owned (no shared storage). */ owned: ReadonlySet; - /** Root receiver bindings, read once before the loop. */ - receivers: { ref: VarRef; global: boolean }[]; + /** Root receiver bindings, read once before the loop; `string` marks a + * string whose `charCodeAt` reads the version covers. */ + receivers: { ref: VarRef; global: boolean; string: boolean }[]; /** Receivers the guard requires to have no shared storage. */ ownedReceivers: number[]; fixed: FixedCheck[]; @@ -80,6 +82,9 @@ export interface BytesLoopFacts { workers: boolean; /** Accesses already proved thread-owned by an enclosing version. */ owned: ReadonlySet; + /** String bindings held as a slice of another string (string-slices.ts): + * they have no string header for the guard to read. */ + scalarStrings: ReadonlySet; } const MAX_COST = 400; @@ -106,6 +111,12 @@ const INVARIANT_OPS = new Set(["+", "-", "*", "&", "|", "^", "<<", ">>", ">>>"]) * such as `base` for `const base = r * cols`, must also be an exact * integer between -2^32 and 2^32, which the guard tests. * + * `s.charCodeAt(index)` of a string binding qualifies the same way. A + * string is immutable: its bytes, its length and the capacity-word bit that + * proves it ASCII (string-reads.ts) are fixed when it is built. The guard + * also requires that bit, so each UTF-16 unit is one byte and the length + * is the byte length, and the fast copy reads each unit with one byte load. + * * In worker programs, every access of a view that may be shared tests its * shared storage. A view's storage is also fixed at construction, so the * same guard tests each invariant receiver once, and the fast copy reads @@ -196,7 +207,8 @@ export function matchBytesLoopVersion( }; const resolving = new Set(); const root = (e: IrExpr): VarRef | null => { - if (e.kind !== "varRef" || e.type.kind !== "bytes") return null; + if (e.kind !== "varRef" || (e.type.kind !== "bytes" && e.type.kind !== "string")) return null; + if (facts.scalarStrings.has(e.localId)) return null; const init = temp(e.localId); if (init) { if (resolving.has(e.localId)) return null; @@ -337,7 +349,7 @@ export function matchBytesLoopVersion( return sum + Math.max(Math.abs(r.min), Math.abs(r.max)); }, 0); - const receivers: { ref: VarRef; global: boolean }[] = []; + const receivers: { ref: VarRef; global: boolean; string: boolean }[] = []; const receiverIndex = new Map(); const fixed = new Map(); const affineChecks = new Map< @@ -349,7 +361,11 @@ export function matchBytesLoopVersion( if (index === undefined) { index = receivers.length; receiverIndex.set(ref.localId, index); - receivers.push({ ref, global: !facts.locals.has(ref.localId) }); + receivers.push({ + ref, + global: !facts.locals.has(ref.localId), + string: ref.type.kind === "string", + }); } return index; }; @@ -359,7 +375,7 @@ export function matchBytesLoopVersion( const access = (node: Node, array: IrExpr, index: IrExpr, width: number): void => { const ref = root(array); if (!ref) return; - if (facts.workers && !facts.owned.has(node)) { + if (facts.workers && ref.type.kind === "bytes" && !facts.owned.has(node)) { const receiver = receiverOf(ref); const nodes = owned.get(receiver); if (nodes) nodes.push(node); @@ -408,7 +424,8 @@ export function matchBytesLoopVersion( const numeric = byteNumberAccess(e); if (numeric) access(e, e.receiver, e.args[numeric.offsetArg]!, numeric.width); else if (e.method === "get" && e.args[0]) access(e, e.receiver, e.args[0], 1); - } + } else if (e.kind === "strIntrinsic" && e.method === "charCodeAt" && e.args[0]) + access(e, e.receiver, e.args[0], 1); return everyExprChild(e, visitExpr, visitStmt); }; const visitStmt = (s: IrStmt): boolean => { @@ -535,14 +552,20 @@ export function emitBytesLoopVersionGuard( } return value; }); - const lengths = receivers.map((receiver) => { - const len = emitBytesHeaderLength(host, receiver); + const lengths = receivers.map((receiver, index) => { + const len = version.receivers[index]!.string + ? emitStringByteLength(host, receiver) + : emitBytesHeaderLength(host, receiver); if (host.sizeType === "i64") return len; const wide = B.tmp(); B.line(`${wide} = zext ${host.sizeType} ${len} to i64`); return wide; }); const conditions: string[] = []; + // A string's byte length is its UTF-16 length when it is ASCII. + version.receivers.forEach(({ string }, index) => { + if (string) conditions.push(emitStringAsciiFlag(host, receivers[index]!)); + }); for (const index of version.ownedReceivers) { const owned = B.tmp(); B.line(`${owned} = icmp eq ptr ${emitBytesSharedStorage(host, receivers[index]!)}, null`); diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index bfde59a30..4933ba8b7 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -3869,6 +3869,7 @@ export class LlEmitter { proven: this.bytesBounds, workers: this.mod.workers === true, owned: this.bytesOwned, + scalarStrings: new Set(this.stringSlices.keys()), }); } diff --git a/packages/compiler/src/backend/llvm/expr-containers.ts b/packages/compiler/src/backend/llvm/expr-containers.ts index 8970aa3be..70e526685 100644 --- a/packages/compiler/src/backend/llvm/expr-containers.ts +++ b/packages/compiler/src/backend/llvm/expr-containers.ts @@ -49,6 +49,7 @@ import { STRING_CHAR_AT_INT_MARKER, STRING_CHAR_CODE_AT_DECL, STRING_CHAR_CODE_AT_INT_MARKER, + emitAsciiCharCode, STRING_UTF16_LEN_DECL, } from "./string-reads.js"; import { exactInteger, widenInteger } from "./integer-values.js"; @@ -319,8 +320,12 @@ export function emitStrIntrinsic( const t = B.tmp(); const index = e.args[0] ? exactInteger(host, args[0]!, e.args[0]) : null; if (index) { - host.declare(STRING_CHAR_CODE_AT_INT_MARKER); const wide = widenInteger(host, index); + // An enclosing loop version proved the string ASCII and the index + // below its length (bytes-loop-versions.ts). + if (host.bytesBounds.has(e)) + return { name: emitAsciiCharCode(host, r.name, wide), type: e.type }; + host.declare(STRING_CHAR_CODE_AT_INT_MARKER); B.line(`${t} = call double @sc_str_char_code_at_int(ptr ${r.name}, i64 ${wide})`); return { name: t, type: e.type }; } diff --git a/packages/compiler/src/backend/llvm/string-reads.ts b/packages/compiler/src/backend/llvm/string-reads.ts index 03a9fa282..c5841006c 100644 --- a/packages/compiler/src/backend/llvm/string-reads.ts +++ b/packages/compiler/src/backend/llvm/string-reads.ts @@ -17,6 +17,7 @@ * is thread-local. */ +import type { LlvmEmitterContext } from "./expr-context.js"; import { exactIndexLines, sizeToDouble } from "./common.js"; import { wtf8Bytes } from "../literals.js"; import { literalCapWord, literalCapWord32 } from "./string-key-hash.js"; @@ -37,6 +38,43 @@ export const STRING_CHAR_AT_DECL = "declare ptr @scr_str_char_at(ptr, double)"; /** Requests the integer-index charAt helper, as the marker above does. */ export const STRING_CHAR_AT_INT_MARKER = "; uses @sc_str_char_at_int"; +/** The proven-ASCII flag of string `s`, as an i1. */ +export function emitStringAsciiFlag(host: LlvmEmitterContext, s: string): string { + const B = host.B; + const wordPointer = B.tmp(), + word = B.tmp(), + ascii = B.tmp(); + B.line(`${wordPointer} = getelementptr inbounds %ScrStr, ptr ${s}, i32 0, i32 2`); + B.line(`${word} = load i32, ptr ${wordPointer}`); + B.line(`${ascii} = icmp slt i32 ${word}, 0`); + return ascii; +} + +/** The byte length of string `s`, as the target's size type. */ +export function emitStringByteLength(host: LlvmEmitterContext, s: string): string { + const B = host.B; + const lengthPointer = B.tmp(), + length = B.tmp(); + B.line(`${lengthPointer} = getelementptr inbounds %ScrStr, ptr ${s}, i32 0, i32 1`); + B.line(`${length} = load ${host.sizeType}, ptr ${lengthPointer}`); + return length; +} + +/** Unit `index` (an i64) of a proven-ASCII string `s` whose length covers + * it: one byte load, as a double. */ +export function emitAsciiCharCode(host: LlvmEmitterContext, s: string, index: string): string { + const B = host.B; + const data = B.tmp(), + pointer = B.tmp(), + byte = B.tmp(), + code = B.tmp(); + B.line(`${data} = getelementptr inbounds %ScrStr, ptr ${s}, i32 1`); + B.line(`${pointer} = getelementptr inbounds i8, ptr ${data}, i64 ${index}`); + B.line(`${byte} = load i8, ptr ${pointer}`); + B.line(`${code} = uitofp i8 ${byte} to double`); + return code; +} + /** Load the proven-ASCII flag of `%s` into `%ascii`. */ function asciiTest(): string[] { return [ diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7302-string-scan-loop-versions.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7302-string-scan-loop-versions.ts.json new file mode 100644 index 000000000..feda0ecf7 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7302-string-scan-loop-versions.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7302-string-scan-loop-versions.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7302-string-scan-loop-versions.ts b/tests/corpus/7302-string-scan-loop-versions.ts new file mode 100644 index 000000000..a8a975d86 --- /dev/null +++ b/tests/corpus/7302-string-scan-loop-versions.ts @@ -0,0 +1,120 @@ +// charCodeAt scans in counted for loops over ASCII strings, Latin-1 and BMP +// text, surrogate pairs and lone surrogates, empty strings, neighbors at +// i - 1 and i + 1, loops that stop early or read past the end, strings built +// by concatenation and slicing, a scan of a slice local, two strings read at +// one index, a string rebound inside or after the loop, nested scans and a +// module-level scan. + +function spaces(s: string): number { + let n = 0; + for (let i = 0; i < s.length; i++) if (s.charCodeAt(i) === 32) n++; + return n; +} + +function hash(s: string): number { + let h = 0; + for (let i = 0; i < s.length; i++) h = (h * 31 + s.charCodeAt(i)) | 0; + return h; +} + +function rises(s: string): number { + let n = 0; + for (let i = 1; i < s.length; i++) if (s.charCodeAt(i) > s.charCodeAt(i - 1)) n++; + return n; +} + +function peaks(s: string): number { + let n = 0; + for (let i = 1; i < s.length - 1; i++) { + const c = s.charCodeAt(i); + if (c > s.charCodeAt(i - 1) && c > s.charCodeAt(i + 1)) n++; + } + return n; +} + +function firstDigit(s: string): number { + for (let i = 0; i < s.length; i++) { + const c = s.charCodeAt(i); + if (c >= 48 && c <= 57) return i; + } + return -1; +} + +function pastEnd(s: string): string { + let sum = 0; + for (let i = 0; i <= s.length; i++) sum += s.charCodeAt(i); + return String(sum); +} + +function sameAt(a: string, b: string): number { + let n = 0; + const end = Math.min(a.length, b.length); + for (let i = 0; i < end; i++) if (a.charCodeAt(i) === b.charCodeAt(i)) n++; + return n; +} + +function stepped(s: string): number { + let h = 7; + for (let i = s.length - 1; i >= 0; i -= 2) h = (h * 33) ^ s.charCodeAt(i); + return h; +} + +function slicedScan(s: string, from: number, to: number): number { + const piece = s.slice(from, to); + let n = piece.length; + for (let i = 0; i < piece.length; i++) n += piece.charCodeAt(i); + return n; +} + +function rebound(s: string): number { + let n = 0; + for (let i = 0; i < s.length; i++) { + n += s.charCodeAt(i); + if (i === 2) s = s.toUpperCase(); + } + return n; +} + +const samples = [ + "the quick brown fox jumps over the lazy dog", + "", + " ", + "abc123 def", + "café au lait", + "日本語のテキスト と 空白", + "emoji 😀 and 🎉 here", + "lone \uD800 surrogate \uDFFF end", + "zigzag azbycxdw", +]; +for (const s of samples) { + console.log( + JSON.stringify(s), + spaces(s), + hash(s), + rises(s), + peaks(s), + firstDigit(s), + pastEnd(s), + stepped(s), + rebound(s), + slicedScan(s, 2, 9), + ); +} + +let built = ""; +for (let k = 0; k < 40; k++) built += String.fromCharCode(65 + (k % 26)) + (k % 3 === 0 ? " " : ""); +const sliced = built.slice(5, 30); +console.log("built", spaces(built), hash(built), rises(sliced), peaks(sliced), sameAt(built, sliced)); +console.log("pairs", sameAt("hello world", "help, world!"), sameAt("abc", ""), sameAt("é", "e")); + +let total = 0; +for (const s of samples) { + for (let i = 0; i < s.length; i++) total = (total + s.charCodeAt(i) * (i + 1)) | 0; +} +console.log("nested", total); + +let text = "module level scan, with commas, and spaces"; +let commas = 0; +for (let i = 0; i < text.length; i++) if (text.charCodeAt(i) === 44) commas++; +text = "rebound after the loop"; +console.log("module", commas, text.length, spaces(text)); diff --git a/tests/harness/corpus-costs/7302-string-scan-loop-versions.ts.json b/tests/harness/corpus-costs/7302-string-scan-loop-versions.ts.json new file mode 100644 index 000000000..793171eba --- /dev/null +++ b/tests/harness/corpus-costs/7302-string-scan-loop-versions.ts.json @@ -0,0 +1 @@ +{ "plain": 4000, "san": 6300 } From 057cc02326c9f276e32b4e0aadc21427a29c499f Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 10:43:08 -0500 Subject: [PATCH 048/197] Order a spread source's keys after key-order witnesses settle The merge helper for a fixed-shape spread into an index-signature record wrote the source's keys in the shape's declared order when it was lowered. Key-order witnesses can replace that order afterwards with the order the program builds the source's values in, so a source declared `{ start, end }` but built as `{ end, start }` merged as `start,end` where Node gives `end,start`. The source's writes now go into one block that a shape-order finalizer fills in the settled order, as the other order-snapshotting helpers do. Corpus program 6916 covers sources built in another order than declared, with an absent optional field and with two spreads. --- .../lowering/containers/indexed-objects.ts | 28 ++++++++++++------- .../6916-index-record-spread-overrides.ts | 17 +++++++++++ 2 files changed, 35 insertions(+), 10 deletions(-) diff --git a/packages/compiler/src/frontend/lowering/containers/indexed-objects.ts b/packages/compiler/src/frontend/lowering/containers/indexed-objects.ts index 75d584e4b..668603d47 100644 --- a/packages/compiler/src/frontend/lowering/containers/indexed-objects.ts +++ b/packages/compiler/src/frontend/lowering/containers/indexed-objects.ts @@ -1589,16 +1589,12 @@ export function lowerIndexMergeHelper( // own planned lift (absent optionals skip — the unset convention); // no overflow exists to walk. if (plan.fields !== null) { - // Keys enter in DECLARATION order, as the object's own keys do - // (declaredOrder omits internal '%'-fields, which never copy). + // Keys enter in the source's enumeration order, as its own keys do. + // That order settles only after lowering (key-order witnesses), so + // the writes go into one block that a finalizer orders. lowerer.keyOrder.observed.add(from.id); - const orderedFields = from.declaredOrder - ? from.declaredOrder.flatMap((n) => { - const f = from.fields.find((x) => x.name === n); - return f ? [f] : []; - }) - : from.fields; - for (const ff of orderedFields) { + const fieldStmts = new Map(); + for (const ff of from.fields) { const lift = plan.fields.get(ff.name)!; const raw: IrExpr = { kind: "recordGet", @@ -1623,7 +1619,8 @@ export function lowerIndexMergeHelper( overflowOnly: true, loc, }; - body.push( + fieldStmts.set( + ff.name, utag >= 0 && ff.type.kind === "union" ? { kind: "if", @@ -1635,6 +1632,17 @@ export function lowerIndexMergeHelper( : write, ); } + // declaredOrder omits internal '%'-fields, which never copy. + const ordered = (shape: IrRecordShape): IrStmt[] => + (shape.declaredOrder ?? shape.fields.map((f) => f.name)).flatMap((field) => { + const stmt = fieldStmts.get(field); + return stmt ? [stmt] : []; + }); + const group: IrStmt & { kind: "block" } = { kind: "block", body: ordered(from), loc }; + lowerer.shapeOrderHelperFinalizers.push(() => { + group.body = ordered(lowerer.shapes.get(plan.shapeId) ?? from); + }); + body.push(group); return; } const fIv = from.indexValue!; diff --git a/tests/corpus/6916-index-record-spread-overrides.ts b/tests/corpus/6916-index-record-spread-overrides.ts index 094b17f02..0a33b0e1b 100644 --- a/tests/corpus/6916-index-record-spread-overrides.ts +++ b/tests/corpus/6916-index-record-spread-overrides.ts @@ -41,3 +41,20 @@ const labels: Record = { first: "a", ...{ zeta: "z", alpha: "a" console.log(Object.entries(labels).map(([k, v]) => k + "=" + v).join(" ")); const flags: { [key: string]: boolean } = { ...counts, mid: true, zeta: false, alpha: true }; console.log(JSON.stringify(flags)); + +// A source whose values are built in another order than its interface +// declares: the spread follows the order the values are built in. +interface Span { + start: number; + end: number; + label?: string; +} +function span(start: number, size: number, label?: string): Span { + return label === undefined ? { end: start + size, start } : { label, end: start + size, start }; +} +const spans: Record = { ...span(1, 4), step: 2 }; +console.log(Object.keys(spans).join(","), JSON.stringify(spans)); +const named: Record = { first: 0, ...span(5, 2, "w"), last: 9 }; +console.log(Object.keys(named).join(","), JSON.stringify(named)); +const both: Record = { ...counts, ...{ end: 3, start: 1 }, mid: 7 }; +console.log(Object.keys(both).join(","), JSON.stringify(both)); From a6a1eb2b95c7b2c4d1ea436526151aba44a110c1 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 10:57:53 -0500 Subject: [PATCH 049/197] Support Duplex.from, compose stages, readable.compose and pipeline function destinations --- .../builtin-apis/streams-and-readline.mdx | 15 +- packages/cli/test/native-link-info.test.ts | 4 +- .../ambient/scriptc-node-fallback.d.ts | 61 ++- .../compiler/src/backend/llvm/lib-dispatch.ts | 2 +- .../compiler/src/backend/llvm/lib-shared.ts | 6 + .../compiler/src/backend/llvm/lib-stream.ts | 26 +- packages/compiler/src/backend/runtime-abi.ts | 4 +- .../src/backend/runtime-pack-targets.test.ts | 2 +- .../compiler/src/backend/runtime-pack.test.ts | 2 +- .../src/frontend/lowering/lower-calls.ts | 4 +- .../src/frontend/lowering/lower-stmts.ts | 23 +- .../src/frontend/lowering/lower-stream.ts | 444 +++++++++++++++--- packages/compiler/src/ir/builtin-effects.ts | 4 + .../compiler/src/ir/builtin-signatures.ts | 19 +- packages/compiler/src/ir/runtime-features.ts | 6 + packages/compiler/src/ir/validate.ts | 46 +- .../test/native-codegen-integration.test.ts | 4 +- .../corpus/7252-duplex-from-generator.ts.json | 6 + .../tests/corpus/7253-duplex-from.mjs.json | 6 + .../corpus/7254-compose-functions.ts.json | 6 + .../tests/corpus/7255-compose-stages.mjs.json | 6 + .../corpus/7256-readable-compose.mjs.json | 6 + ...57-pipeline-generator-destination.mjs.json | 6 + ...258-pipeline-generator-destination.ts.json | 6 + ...9-promises-pipeline-stage-options.cjs.json | 6 + .../runtime-darwin-arm64/scripts/build.mjs | 2 +- .../runtime-pack-common/scripts/build.mjs | 2 +- packages/runtime/src/scr_console.c | 2 +- packages/runtime/src/scr_core.h | 2 +- packages/runtime/src/scr_stream.c | 229 +++++++-- packages/runtime/src/scr_stream.h | 18 + packages/runtime/src/scr_stream_ops.c | 232 +++++++++ tests/corpus/7252-duplex-from-generator.ts | 69 +++ tests/corpus/7253-duplex-from.mjs | 48 ++ tests/corpus/7254-compose-functions.ts | 56 +++ tests/corpus/7255-compose-stages.mjs | 55 +++ tests/corpus/7256-readable-compose.mjs | 53 +++ .../7257-pipeline-generator-destination.mjs | 61 +++ .../7258-pipeline-generator-destination.ts | 47 ++ .../7259-promises-pipeline-stage-options.cjs | 65 +++ .../7252-duplex-from-generator.ts.json | 1 + .../corpus-costs/7253-duplex-from.mjs.json | 1 + .../7254-compose-functions.ts.json | 1 + .../corpus-costs/7255-compose-stages.mjs.json | 1 + .../7256-readable-compose.mjs.json | 1 + ...57-pipeline-generator-destination.mjs.json | 1 + ...258-pipeline-generator-destination.ts.json | 1 + ...9-promises-pipeline-stage-options.cjs.json | 1 + .../self-hosting-native-toolchain.test.ts | 2 +- 49 files changed, 1510 insertions(+), 161 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7252-duplex-from-generator.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7253-duplex-from.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7254-compose-functions.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7255-compose-stages.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7256-readable-compose.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7257-pipeline-generator-destination.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7258-pipeline-generator-destination.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7259-promises-pipeline-stage-options.cjs.json create mode 100644 tests/corpus/7252-duplex-from-generator.ts create mode 100644 tests/corpus/7253-duplex-from.mjs create mode 100644 tests/corpus/7254-compose-functions.ts create mode 100644 tests/corpus/7255-compose-stages.mjs create mode 100644 tests/corpus/7256-readable-compose.mjs create mode 100644 tests/corpus/7257-pipeline-generator-destination.mjs create mode 100644 tests/corpus/7258-pipeline-generator-destination.ts create mode 100644 tests/corpus/7259-promises-pipeline-stage-options.cjs create mode 100644 tests/harness/corpus-costs/7252-duplex-from-generator.ts.json create mode 100644 tests/harness/corpus-costs/7253-duplex-from.mjs.json create mode 100644 tests/harness/corpus-costs/7254-compose-functions.ts.json create mode 100644 tests/harness/corpus-costs/7255-compose-stages.mjs.json create mode 100644 tests/harness/corpus-costs/7256-readable-compose.mjs.json create mode 100644 tests/harness/corpus-costs/7257-pipeline-generator-destination.mjs.json create mode 100644 tests/harness/corpus-costs/7258-pipeline-generator-destination.ts.json create mode 100644 tests/harness/corpus-costs/7259-promises-pipeline-stage-options.cjs.json diff --git a/docs/content/limitations/builtin-apis/streams-and-readline.mdx b/docs/content/limitations/builtin-apis/streams-and-readline.mdx index 2a514a5a0..060a0bf95 100644 --- a/docs/content/limitations/builtin-apis/streams-and-readline.mdx +++ b/docs/content/limitations/builtin-apis/streams-and-readline.mdx @@ -12,16 +12,21 @@ An object-mode write passes its encoding argument to `write` and `transform` as Node does (`undefined` when absent, `null` when a callback took its place); a parameter declared `string` receives `""` for `undefined` and `null`, and one declared `string | null | undefined` receives Node's value. `readable.read()` on an object-mode stream answers its Buffers; another value throws a not-supported error (read such values with `'data'` listeners, `for await`, or `pipe`). `node:stream/consumers` and readline interfaces read object-mode streams of strings and Buffers; other values throw a not-supported error. - `pipeline` accepts function stages anywhere among its streams: async generator functions (`async function* (source)`, iterating `source` with `for await`), an async generator source function (`async function* ({ signal })`), and an async function destination whose value resolves the promise form and reaches the callback's second argument. + `pipeline` accepts function stages anywhere among its streams: async generator functions (`async function* (source)`, iterating `source` with `for await`), an async generator source function (`async function* ({ signal })`), an async generator function as the destination, and an async function destination whose value resolves the promise form and reaches the callback's second argument. A stage is written inline, or named by a function declaration or a `const` bound to a function expression that nothing reassigns. The source may also be an array, a string, a synchronous generator object, or a synchronous generator function without parameters; their elements reach the next stage on the same ticks as in Node. Stages may take the `{ signal }` argument. - An async generator function as the destination, using the PassThrough that `pipeline()` returns for a function destination, and function stages or iterable sources together with pipeline options are unsupported. + With a function destination, `pipeline()` returns an object-mode PassThrough that receives an async generator destination's values, or an async function destination's value unless it is `null` or `undefined`. + `@types/node` types that result as `NodeJS.WritableStream`, so `for await` over it needs the `PassThrough` type (`pipeline(...) as PassThrough`, or a JSDoc `@type`). + In typed code, the callback of a pipeline with an async function destination receives `null` instead of `undefined` when there is no error. `Readable.from` accepts `objectMode` and `highWaterMark` options. The `Readable` operators (`map`, `filter`, `forEach`, `toArray`, `some`, `find`, `every`, `flatMap`, `drop`, `take`, and `reduce`) accept `concurrency`, `highWaterMark`, and `signal` options; their callbacks take only the chunk, `map` returns strings or Buffers, and `flatMap` returns strings or arrays of them. - `readable.compose` is unsupported. - `stream.compose` accepts two or more streams, or one `Duplex`; functions, iterables, and web streams as stages are unsupported, and the `ERR_INVALID_ARG_VALUE` message for a stream that is no longer readable or writable names the stream's class instead of printing the stream. - `pipeline` from `node:stream/promises` accepts a trailing `{ signal, end }` object, and `finished` accepts `readable`, `writable`, `error`, `cleanup`, and `signal` options, with boolean literals for the flags. + `Duplex.from` accepts an async generator function, which reads what is written to the Duplex, a `Duplex`, `Transform` or `PassThrough`, which it returns, and the values `Readable.from` accepts, which give a Duplex that is not writable. + A lone `Readable` or `Writable`, a promise, a `{ readable, writable }` pair, a web stream, and a body function that declares the options argument are unsupported in `Duplex.from`. + `stream.compose` accepts streams and async generator functions in any position, and an array, a string, or a generator as the first stage; one stage is `Duplex.from` of it. + Web streams as compose stages are unsupported, and the `ERR_INVALID_ARG_VALUE` message for a stream that is no longer readable or writable names the stream's class instead of printing the stream. + `readable.compose(stage, { signal })` is `stream.compose(readable, stage)` with the signal added; the options are an object literal whose `signal` is a variable or a property read. + `pipeline` from `node:stream/promises` accepts a trailing `{ signal, end }` object, with or without function stages, and `finished` accepts `readable`, `writable`, `error`, `cleanup`, and `signal` options, with boolean literals for the flags. Stream constructors accept a `signal` option in an options object literal, and `stream.addAbortSignal` accepts any stream. The `construct` option and `_construct` method, `decodeStrings`, and `defaultEncoding` work on every stream class, and the `writev` option and `_writev` method on Writable and Duplex; `writev` on a Transform or PassThrough, assigning `_construct` after construction, and a non-literal `decodeStrings` or `defaultEncoding` in an options object literal are unsupported. `setDefaultEncoding()` takes any encoding Node knows. diff --git a/packages/cli/test/native-link-info.test.ts b/packages/cli/test/native-link-info.test.ts index a7a5e718a..a1b673630 100644 --- a/packages/cli/test/native-link-info.test.ts +++ b/packages/cli/test/native-link-info.test.ts @@ -72,7 +72,7 @@ describe.runIf(supported)("macOS arm64 native link info", () => { minimum_os: "14.0", }, program: { object, entry_symbol: "main" }, - runtime_abi: { version: 27, marker: "scr_runtime_abi_v27" }, + runtime_abi: { version: 28, marker: "scr_runtime_abi_v28" }, runtime_pack: { kind: "precompiled", package: "@scriptc/runtime-darwin-arm64", @@ -165,6 +165,6 @@ describe.runIf(supported)("macOS arm64 native link info", () => { () => null, (failure: { stderr?: string }) => failure, ); - expect(error?.stderr).toContain("scr_runtime_abi_v27"); + expect(error?.stderr).toContain("scr_runtime_abi_v28"); }); }); diff --git a/packages/compiler/ambient/scriptc-node-fallback.d.ts b/packages/compiler/ambient/scriptc-node-fallback.d.ts index abbd023d4..380f5d814 100644 --- a/packages/compiler/ambient/scriptc-node-fallback.d.ts +++ b/packages/compiler/ambient/scriptc-node-fallback.d.ts @@ -3780,9 +3780,8 @@ declare module "util" { export type TextDecoder = globalThis.TextDecoder; export var TextEncoder: typeof globalThis.TextEncoder; export var TextDecoder: typeof globalThis.TextDecoder; - /* stream.pipeline's (...streams: unknown[]) => unknown is assignable to - * every other target's parameter type, so its overload comes first; no - * other target is assignable to it. */ + /* stream.pipeline's overload comes first: its __promisify__ member keeps + * every other target out of it. */ export function promisify( fn: typeof import("node:stream").pipeline, ): typeof import("node:stream/promises").pipeline; @@ -7266,6 +7265,8 @@ declare module "stream" { _destroy(error: Error | null, callback: (error?: Error | null) => void): void; [Symbol.asyncIterator](): AsyncIterableIterator; iterator(options?: { destroyOnReturn?: boolean }): AsyncIterableIterator; + /* compose(this, stage), destroyed when options.signal aborts. */ + compose(stage: ComposeStage, options?: { signal: AbortSignal }): Duplex; push(chunk: Buffer | string | null, encoding?: string): boolean; /* An object-mode stream's values. */ push(chunk: any, encoding?: string): boolean; @@ -7456,6 +7457,9 @@ declare module "stream" { * mirrors @types/node: extends Readable, the writable half declared * again. The lowering treats both halves as first-class. */ class Duplex extends Readable { + /* Duplex.from: an async generator function, an iterable, a string or + * a stream (duplexify.js). */ + static from(body: unknown): Duplex; constructor(opts: ObjectDuplexOptions); constructor(opts: StringDuplexOptions); constructor(opts?: DuplexOptions); @@ -7576,11 +7580,36 @@ declare module "stream" { /* addAbortSignal(signal, stream): the stream, destroyed with an * AbortError (cause: signal.reason) when the signal aborts. */ function addAbortSignal(signal: AbortSignal, stream: T): T; - /* Declared so requires typecheck; each fences at its use site. */ - function pipeline(...streams: unknown[]): unknown; - /* compose(...streams): a Duplex that writes to the first stream and - * reads from the last, with the streams running as a pipeline. */ - function compose(...streams: Array): Duplex; + /* pipeline(...stages, callback): the last stream, or the object-mode + * PassThrough that a function destination feeds. */ + type PipelineResult = T extends [ + ...unknown[], + infer D, + (...args: any[]) => unknown, + ] + ? D extends (...args: any[]) => unknown + ? PassThrough + : D + : unknown; + function pipeline(...streams: T): PipelineResult; + /* util.promisify's pipeline overload matches only pipeline itself: a + * generic signature alone admits every function. */ + namespace pipeline { + const __promisify__: typeof promises.pipeline; + } + /* A compose stage: a stream, an async generator function over the + * previous stage, or (first) an iterable, async iterable or web + * ReadableStream. */ + type ComposeStage = + | Readable + | Writable + | Iterable + | AsyncIterable + | globalThis.ReadableStream + | ((source: any, options?: { signal: AbortSignal }) => unknown); + /* compose(...streams): a Duplex that writes to the first stage and + * reads from the last, with the stages running as a pipeline. */ + function compose(...streams: ComposeStage[]): Duplex; /* finished's options: readable/writable choose which ends count, * error: false ignores 'error' events, signal aborts the wait. */ interface FinishedOptions { @@ -7599,13 +7628,20 @@ declare module "stream" { * else 65536 — Node's own state.js split); literal argument only. */ function getDefaultHighWaterMark(objectMode: boolean): number; /* The promise pipeline's result: a last stage that is an async - * function resolves the promise with its value. */ + * function resolves the promise with its value (before the options + * argument, when one follows). */ type PipelinePromise = T extends [ ...unknown[], (...args: any[]) => Promise, ] ? Promise

- : Promise; + : T extends [ + ...unknown[], + (...args: any[]) => Promise, + { signal?: AbortSignal; end?: boolean }, + ] + ? Promise

+ : Promise; namespace promises { function pipeline(...streams: T): PipelinePromise; function finished( @@ -7635,6 +7671,7 @@ declare module "stream" { DuplexOptions, TransformOptions, FinishedOptions, + PipelinePromise, }; } declare module "node:stream" { @@ -7649,9 +7686,7 @@ declare module "stream/promises" { * pipeline's trailing { signal, end } object and finished's options * (readable, writable, error, cleanup, signal) are supported. A last * stage that is an async function resolves the promise with its value. */ - function pipeline( - ...streams: T - ): T extends [...unknown[], (...args: any[]) => Promise] ? Promise

: Promise; + function pipeline(...streams: T): import("stream").PipelinePromise; function finished( stream: unknown, options?: { diff --git a/packages/compiler/src/backend/llvm/lib-dispatch.ts b/packages/compiler/src/backend/llvm/lib-dispatch.ts index 24f2bf019..fd5b5d681 100644 --- a/packages/compiler/src/backend/llvm/lib-dispatch.ts +++ b/packages/compiler/src/backend/llvm/lib-dispatch.ts @@ -734,7 +734,7 @@ export function emitLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlValue { if (e.fn === "process.nextTick") return host.emitAsyncContextLibCall(e); if ( e.fn === "sp.pipeline" || - e.fn === "sp.pipelineStages" || + e.fn.startsWith("sp.pipelineStages") || e.fn.startsWith("sp.pipelineOpts") || e.fn.startsWith("sp.finishedOpts") ) diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index 46bfc279e..94318c3aa 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -1127,6 +1127,8 @@ export const LIB_FN_SYMS: Record = { "readable.pushEncoding": "scr_stream_set_push_encoding", "stream.objectMode": "scr_stream_object_mode", "stream.decodeStrings": "scr_stream_decode_strings", + "stream.duplexFromGen": "scr_stream_duplex_from_gen", + "stream.duplexFromDyn": "scr_stream_duplex_from_dyn", "writable.writeEncoding": "scr_stream_write_encoding", "writable.setDefaultEncoding": "scr_stream_set_default_encoding", "stream.chunkDone": "scr_stream_chunk_done", @@ -1733,6 +1735,8 @@ export const USES_TIMERS_LIB_FNS = new Set([ "stream.setTransform", "stream.setFlush", "stream.setConstruct", + "stream.duplexFromGen", + "stream.duplexFromDyn", "stream.setWritev", "stream.setWritevDefault", "process.activeResources", @@ -1744,8 +1748,10 @@ export const USES_TIMERS_LIB_FNS = new Set([ "sp.finished", "sp.pipeline", "sp.pipelineStages", + "sp.pipelineStagesOpts", "stream.pipelineStages", "stream.pipelineStagesDyn", + "stream.composeStages", "sp.pipelineOpts", "sp.pipelineOptsSignal", "sp.finishedOpts", diff --git a/packages/compiler/src/backend/llvm/lib-stream.ts b/packages/compiler/src/backend/llvm/lib-stream.ts index 4611b3bb8..5fd4af1a9 100644 --- a/packages/compiler/src/backend/llvm/lib-stream.ts +++ b/packages/compiler/src/backend/llvm/lib-stream.ts @@ -701,12 +701,15 @@ export function emitStreamLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlV } if ( e.fn === "sp.pipelineStages" || + e.fn === "sp.pipelineStagesOpts" || e.fn === "stream.pipelineStages" || - e.fn === "stream.pipelineStagesDyn" + e.fn === "stream.pipelineStagesDyn" || + e.fn === "stream.composeStages" ) { // (n, k, positions, flags, mask, s1..sn, stage1..stagek, helpers..., - // box?, (signal, abort)?[, cb]): the streams, the stage closures and - // the helpers ride stack arrays; absent groups pass null. + // box?, (signal, abort)?[, cb | end, signal?]): the streams, the + // stage closures and the helpers ride stack arrays; absent groups + // pass null. const [nArg, kArg, positionsArg, flagsArg, maskArg] = e.args; if ( nArg?.kind !== "numLit" || @@ -719,7 +722,7 @@ export function emitStreamLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlV const n = nArg.value; const k = kArg.value; const flags = flagsArg.value; - const helperCount = 3 * pipelinePumpedRuns(n + k, positionsArg.value); + const helperCount = 3 * pipelinePumpedRuns(n + k, positionsArg.value, flags); const args = e.args.map((a) => host.emitExpr(a)); const packed = (from: number, count: number): string => { if (count === 0) return "null"; @@ -749,6 +752,21 @@ export function emitStreamLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlV if (e.fn === "sp.pipelineStages") { host.declare(`declare ptr @scr_sp_pipeline_stages(${params})`); B.line(`${t} = call ptr @scr_sp_pipeline_stages(${head})`); + } else if (e.fn === "sp.pipelineStagesOpts") { + const end = args[at++]!.name; + const outer = take((flags & 64) !== 0); + let add = "null"; + if (outer !== "null") { + host.declare(`declare ptr @scr_events_add_abort_listener(ptr, ptr)`); + add = "@scr_events_add_abort_listener"; + } + host.declare(`declare ptr @scr_sp_pipeline_stages_opts(${params}, i1, ptr, ptr)`); + B.line( + `${t} = call ptr @scr_sp_pipeline_stages_opts(${head}, i1 ${end}, ptr ${outer}, ptr ${add})`, + ); + } else if (e.fn === "stream.composeStages") { + host.declare(`declare ptr @scr_stream_compose_stages(${params})`); + B.line(`${t} = call ptr @scr_stream_compose_stages(${head})`); } else if (e.fn === "stream.pipelineStagesDyn") { host.declare(`declare ptr @scr_stream_pipeline_stages_dyn(${params}, ptr)`); B.line( diff --git a/packages/compiler/src/backend/runtime-abi.ts b/packages/compiler/src/backend/runtime-abi.ts index 3a41d9176..a616dbec2 100644 --- a/packages/compiler/src/backend/runtime-abi.ts +++ b/packages/compiler/src/backend/runtime-abi.ts @@ -2,8 +2,8 @@ * of the link contract: a program object keeps this symbol undefined and the * matching runtime defines it, so incompatible manual links fail before the * program can start. */ -export const RUNTIME_ABI_VERSION = 27 as const; -export const RUNTIME_ABI_MARKER = "scr_runtime_abi_v27" as const; +export const RUNTIME_ABI_VERSION = 28 as const; +export const RUNTIME_ABI_MARKER = "scr_runtime_abi_v28" as const; /** Object consumption is useful today, but the complete scr_* surface may * still change before 1.0. The versioned marker prevents accidental mixing; diff --git a/packages/compiler/src/backend/runtime-pack-targets.test.ts b/packages/compiler/src/backend/runtime-pack-targets.test.ts index af61e5229..c2536ede8 100644 --- a/packages/compiler/src/backend/runtime-pack-targets.test.ts +++ b/packages/compiler/src/backend/runtime-pack-targets.test.ts @@ -52,7 +52,7 @@ async function fixture(target: NativeTargetSpec, workers = false) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 27, marker: "scr_runtime_abi_v27" }, + runtime_abi: { version: 28, marker: "scr_runtime_abi_v28" }, compiler: { command: "fixture", identity: "fixture", target: target.llvmTriple }, macros: { executable: [], excluded: ["SCR_LIB"], sanitizer: "external-toolchain-required" }, flavors: { diff --git a/packages/compiler/src/backend/runtime-pack.test.ts b/packages/compiler/src/backend/runtime-pack.test.ts index d09509911..33b1ca38e 100644 --- a/packages/compiler/src/backend/runtime-pack.test.ts +++ b/packages/compiler/src/backend/runtime-pack.test.ts @@ -127,7 +127,7 @@ async function fixture(target: NativeTargetSpec = MACOS_ARM64_TARGET) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 27, marker: "scr_runtime_abi_v27" }, + runtime_abi: { version: 28, marker: "scr_runtime_abi_v28" }, compiler: { command: "clang", identity: "fixture clang", diff --git a/packages/compiler/src/frontend/lowering/lower-calls.ts b/packages/compiler/src/frontend/lowering/lower-calls.ts index 39b7207c0..875fb9e7a 100644 --- a/packages/compiler/src/frontend/lowering/lower-calls.ts +++ b/packages/compiler/src/frontend/lowering/lower-calls.ts @@ -9843,7 +9843,9 @@ export function lowerObjectMethodCall( if ( info && !found && - (STREAM_API_MEMBERS.has(access.name.text) || STREAM_OPERATORS.has(access.name.text)) && + (STREAM_API_MEMBERS.has(access.name.text) || + STREAM_OPERATORS.has(access.name.text) || + access.name.text === "compose") && streamSidesOf(lowerer, info) !== null ) { const stream = lowerStreamMethodCall(lowerer, call, access, info); diff --git a/packages/compiler/src/frontend/lowering/lower-stmts.ts b/packages/compiler/src/frontend/lowering/lower-stmts.ts index 213abfec6..305cbd9b4 100644 --- a/packages/compiler/src/frontend/lowering/lower-stmts.ts +++ b/packages/compiler/src/frontend/lowering/lower-stmts.ts @@ -2,7 +2,7 @@ import { lowerTupleAliasDecl, stripTypeWrappers } from "./typed-arguments.js"; import { declaredMemberOf, isPromisePrototypeRead } from "./expressions/property-access.js"; import { hostsAll } from "./expressions/hosted-narrowing.js"; import { requestGeneralStreamSurface } from "./general-streams.js"; -import { lowerForAwaitDynStream } from "./dyn-streams.js"; +import { lowerForAwaitDynStream, nodeStreamInterfaceSide } from "./dyn-streams.js"; import { lowerForAwaitValueStream } from "./builtins/hash-stream.js"; import { lowerChildPropertyWrite } from "./handle-properties.js"; import { @@ -163,7 +163,11 @@ import { objectFactorySignature } from "./object-factory-new.js"; import { isClassCallback, lowerClassCallbackAssign } from "./class-callbacks.js"; import { classForInHasKey, classPropertiesHelper } from "./class-dynamic-dispatch.js"; import { genericIfaceBindingKeepsClass, superConditionLead } from "./lower-classes.js"; -import { lowerStreamUnderscoreAssign, streamClassAliasDecl } from "./lower-stream.js"; +import { + lowerStreamUnderscoreAssign, + streamClassAliasDecl, + streamSidesOf, +} from "./lower-stream.js"; import { staticAssertInstanceDecl } from "./assert-instances.js"; import { lowerHttpResPropertyAssignment, @@ -9451,6 +9455,21 @@ export function lowerForOf(lowerer: Lowerer, stmt: ts.ForOfStatement): IrStmt { ) { return lowerForAwaitStdin(lowerer, stmt); } + // NodeJS.WritableStream is what @types/node answers for pipeline() + // with a function destination, whose value is a PassThrough. + if (nodeStreamInterfaceSide(lowerer, lowerer.typeOf(stmt.expression)) === "w") { + const mapped = lowerer.mapTypeOf(lowerer.typeOf(stmt.expression)); + const sides = + mapped?.kind === "object" + ? streamSidesOf(lowerer, lowerer.classes.get(mapped.className)) + : null; + if (sides !== "r" && sides !== "rw") + lowerer.noLowering( + "for await over a NodeJS.WritableStream", + stmt.expression, + "type the stream as the readable class it is: pipeline() answers a PassThrough for a function destination (pipeline(...) as PassThrough, or a JSDoc @type)", + ); + } // A checked-dynamic NodeJS.ReadableStream iterates its native stream, // and a Hash, Hmac, Cipheriv or Decipheriv iterates its Transform. const dynStream = diff --git a/packages/compiler/src/frontend/lowering/lower-stream.ts b/packages/compiler/src/frontend/lowering/lower-stream.ts index 20f156124..214a2402e 100644 --- a/packages/compiler/src/frontend/lowering/lower-stream.ts +++ b/packages/compiler/src/frontend/lowering/lower-stream.ts @@ -56,6 +56,7 @@ import { bindingNeverReassigned } from "./binding-analysis.js"; import { extractIteratorValue, genStep, genValue, stepDrivable } from "./lower-generators.js"; import { LiftedFrame, driveGenerator } from "./containers/iterator-helpers.js"; import { lowerArrayIteratorValue, lowerStringIteratorValue } from "./iterator-adapters.js"; +import { staticReadableStreamElementType } from "./lower-stdlib.js"; import { BOOL, DYN, @@ -1816,8 +1817,23 @@ export function lowerStreamStaticCall( if (member !== "from") { lowerer.noLowering(`${cls.slice(1)}.${member}`, call); } + if (cls === "%Duplex") { + const fromArgs = call.arguments.nodes; + if (fromArgs.length !== 1) { + lowerer.noLowering( + `Duplex.from with ${fromArgs.length} arguments`, + call, + "Duplex.from takes one body", + ); + } + return duplexFromBody(lowerer, fromArgs[0]!, locOf(call)); + } if (cls !== "%Readable") { - lowerer.noLowering(`${cls.slice(1)}.from`, call, "Readable.from is the lowered form"); + lowerer.noLowering( + `${cls.slice(1)}.from`, + call, + "Readable.from and Duplex.from are the lowered forms", + ); } const args = call.arguments.nodes; if (args.length > 2) { @@ -1827,16 +1843,34 @@ export function lowerStreamStaticCall( "Readable.from(iterable, options) is the supported form", ); } - const type: IrType = { kind: "object", className: "%Readable" }; + return lowerFromSource(lowerer, call, args[0], args[1], false); +} + +/** Readable.from(source, options), and for Duplex.from over an iterable + * (duplex) duplexify.js's from(Duplexify, body, { objectMode: true, + * writable: false }): the same builders with SCR_FROM_DUPLEX, answering a + * Duplexify without a writable side. */ +function lowerFromSource( + lowerer: Lowerer, + at: ts.Node, + srcNode: ts.Expression | undefined, + optsNode: ts.Expression | undefined, + duplex: boolean, +): IrExpr { + const loc = locOf(at); + const call = at; + const type: IrType = { kind: "object", className: duplex ? "%Duplex" : "%Readable" }; + const what = duplex ? "Duplex.from" : "Readable.from"; + // SCR_FROM_DUPLEX + const duplexFlag = duplex ? 32 : 0; const js = isJsSourceFile(ts.getSourceFileOfNode(call)); - const srcNode = args[0]; - const optsNode = args[1]; const srcT = srcNode ? lowerer.mapTypeOf(lowerer.typeOf(srcNode)) : null; const typedElem = (t: IrType | null | undefined): "string" | "bytes" | null => t?.kind === "string" ? "string" : t?.kind === "bytes" && t.elem === "u8" ? "bytes" : null; const single = typedElem(srcT); const elem = srcT?.kind === "array" ? typedElem(srcT.elem) : null; - const opts = optsNode ? fromOptions(lowerer, optsNode) : { flags: 0, hwm: null }; + const opts0 = optsNode ? fromOptions(lowerer, optsNode) : { flags: 0, hwm: null }; + const opts = opts0 ? { ...opts0, flags: opts0.flags | duplexFlag } : opts0; // A class whose [Symbol.iterator]() or [Symbol.asyncIterator]() is a // generator method: from.js calls the method when the stream is built, // then reads the generator like any other. @@ -1934,6 +1968,7 @@ export function lowerStreamStaticCall( // The checked-dynamic form: the runtime takes from.js's ladder // (ERR_INVALID_ARG_TYPE for a non-iterable, the options, null values). const src = srcNode ? lowerer.lowerExprExpecting(srcNode, DYN) : dynUndefinedExpr(loc); + if (duplex) return { kind: "libCall", fn: "stream.duplexFromDyn", args: [src], type, loc }; const o = lowerOperatorValue(lowerer, optsNode, "Readable.from options", loc, true); return { kind: "libCall", fn: "readable.fromDyn", args: [src, o], type, loc }; } @@ -1945,7 +1980,7 @@ export function lowerStreamStaticCall( ); } lowerer.noLowering( - `Readable.from over a '${srcT ? lowerer.fmt(srcT) : "?"}' source`, + `${what} over a '${srcT ? lowerer.fmt(srcT) : "?"}' source`, srcNode ?? call, "the supported sources are a string or Buffer, and arrays, Sets, Maps, generators and class iterables of values that convert to checked-dynamic values", ); @@ -2762,22 +2797,28 @@ export function lowerStreamModuleCall( : null; const stageArgs = opts ? args.slice(0, -1) : args; if (hasPipelineFunctionStage(lowerer, stageArgs)) { - if (opts) { - lowerer.noLowering( - "pipeline with function stages or iterable sources and options", - lastArg, - "pipeline(source, async function* (source) { ... }, destination) without options is supported", - ); - } const staged = lowerPipelineStages(lowerer, stageArgs, loc); + const type = staged.destPromise ?? { kind: "promise", inner: VOID }; // A promise destination's value settles the result (same type). - return staged.wrap({ - kind: "libCall", - fn: "sp.pipelineStages", - args: staged.args, - type: staged.destPromise ?? { kind: "promise", inner: VOID }, - loc, - }); + if (!opts) { + return staged.wrap({ + kind: "libCall", + fn: "sp.pipelineStages", + args: staged.args, + type, + loc, + }); + } + // the options: end, and the signal (flags bit 6) + const flagsArg = staged.args[3]!; + if (flagsArg.kind !== "numLit") + throw new InternalCompilerError("pipeline flags not a literal"); + const args = [...staged.args, boolLit(opts.end, loc)]; + if (opts.signal) { + args[3] = numLit(flagsArg.value | 64, loc); + args.push(opts.signal); + } + return staged.wrap({ kind: "libCall", fn: "sp.pipelineStagesOpts", args, type, loc }); } let streams: IrExpr[]; try { @@ -2958,22 +2999,27 @@ export function lowerStreamModuleCall( } if (hasPipelineFunctionStage(lowerer, args.slice(0, -1))) { const staged = lowerPipelineStages(lowerer, args.slice(0, -1), loc); - if (staged.dest !== 0) { - // callback(err, value): the destination's value reaches the - // callback through the checked-dynamic path. Node answers an - // object-mode PassThrough here; reading it has no lowering. - if (!ts.isExpressionStatement(call.parent)) { - lowerer.noLowering( - "using the PassThrough that pipeline() answers for a function destination", - call, - "read the destination's value in the callback", - ); - } + const cbNode = args[args.length - 1]!; + const cbParams = + ts.isArrowFunction(cbNode) || ts.isFunctionExpression(cbNode) + ? cbNode.parameters.nodes.length + : 0; + if (staged.dest === 2 || (staged.dest === 1 && cbParams > 1)) { + // callback(err, value): the destination's value (undefined for + // an async generator) reaches the callback through the + // checked-dynamic path. const cb = lowerer.lowerExprExpecting(args[args.length - 1]!, DYN); + // SCR_PIPE_TYPED_CB: typed code's callback takes null for no error + const stagedArgs = [...staged.args]; + const flagsArg = stagedArgs[3]!; + if (flagsArg.kind !== "numLit") + throw new InternalCompilerError("pipeline flags not a literal"); + if (!isJsSourceFile(ts.getSourceFileOfNode(call))) + stagedArgs[3] = numLit(flagsArg.value | 128, loc); return staged.wrap({ kind: "libCall", fn: "stream.pipelineStagesDyn", - args: [...staged.args, cb], + args: [...stagedArgs, cb], type: staged.last, loc, }); @@ -3010,7 +3056,12 @@ export function lowerStreamModuleCall( }; } - if (bi.member === "compose") return lowerStreamCompose(lowerer, call, loc); + if (bi.member === "compose") { + if (call.arguments.nodes.some((a) => ts.isSpreadElement(a))) { + lowerer.noLowering("compose with spread arguments", call, "pass the streams directly"); + } + return lowerStreamCompose(lowerer, call.arguments.nodes, loc); + } return null; } @@ -3227,7 +3278,7 @@ function lowerIterableSource( interface PipelineStagesLowering { args: IrExpr[]; last: IrType; - dest: 0 | 2; + dest: 0 | 1 | 2; destPromise: (IrType & { kind: "promise" }) | null; wrap: (call: IrExpr) => IrExpr; } @@ -3235,9 +3286,10 @@ interface PipelineStagesLowering { /** pipeline(...) with function stages (pipeline.js). A function may * start the pipeline (a source function, called with `{ signal }`), may * end it (an async function whose promise's value reaches the callback or - * the promise), and runs of async generator functions may sit between - * streams; the last generator of a run that a stream follows is pumped - * into that stream. Each stage's `source` parameter is typed as what the + * the promise, or an async generator function), and runs of async + * generator functions may sit between streams; the last generator of a + * run that a stream follows is pumped into that stream. A function + * destination feeds the object-mode PassThrough that pipeline() answers. Each stage's `source` parameter is typed as what the * pipeline passes it: makeAsyncIterable answers the previous stream * itself (a Readable is async iterable) or the previous stage's * generator. */ @@ -3245,6 +3297,7 @@ function lowerPipelineStages( lowerer: Lowerer, stageArgs: readonly ts.Expression[], loc: SrcLoc, + preLowered?: ReadonlyMap, ): PipelineStagesLowering { if (stageArgs.length > 52) { lowerer.noLowering( @@ -3258,7 +3311,7 @@ function lowerPipelineStages( const helperArgs: IrExpr[] = []; let positions = 0; let mask = 0; - let dest: 0 | 2 = 0; + let dest: 0 | 1 | 2 = 0; let destPromise: (IrType & { kind: "promise" }) | null = null; // What the next function stage receives: null before the first stage, // else the previous stream's type or the previous stage's generator. @@ -3269,14 +3322,19 @@ function lowerPipelineStages( const argOrder: { list: IrExpr[]; index: number }[] = []; for (let j = 0; j < stageArgs.length; j++) { const arg = stageArgs[j]!; - const fn = pipelineStageFunction(lowerer, arg); + // compose's head and tail functions arrive as Duplex.from streams + const fn = preLowered?.has(j) ? null : pipelineStageFunction(lowerer, arg); const syncSourceFn = j === 0 && fn !== null && !ts.isArrowFunction(fn) && fn.asteriskToken !== undefined && !(fn.modifiers?.nodes ?? []).some((m) => m.kind === ts.SyntaxKind.AsyncKeyword); - if (j === 0 && (syncSourceFn || (fn === null && iterableSourceType(lowerer, arg) !== null))) { + if ( + j === 0 && + !preLowered?.has(j) && + (syncSourceFn || (fn === null && iterableSourceType(lowerer, arg) !== null)) + ) { const source = lowerIterableSource(lowerer, arg, syncSourceFn ? fn : null, loc); if (source === null) lowerer.noLowering("pipeline over this source", arg); positions += 1; @@ -3302,7 +3360,7 @@ function lowerPipelineStages( } let stream: IrExpr; try { - stream = lowerStreamArg(lowerer, arg, "pipeline"); + stream = preLowered?.get(j) ?? lowerStreamArg(lowerer, arg, "pipeline"); } catch (e) { if (e instanceof StreamArgTypeThrow) lowerer.noLowering("pipeline over this value", arg); throw e; @@ -3315,15 +3373,6 @@ function lowerPipelineStages( const isLast = j === stageArgs.length - 1; const isAsync = (fn.modifiers?.nodes ?? []).some((m) => m.kind === ts.SyntaxKind.AsyncKeyword); const isGen = !ts.isArrowFunction(fn) && fn.asteriskToken !== undefined; - if (isLast && isGen) { - // pipeline() pumps it into an object-mode PassThrough, whose - // buffering decides when the pipeline finishes - lowerer.noLowering( - "a pipeline destination that is an async generator function", - arg, - "end the pipeline with a stream or an async function (source) { for await (const chunk of source) ... }", - ); - } if (!isAsync || (!isGen && !isLast)) { lowerer.noLowering( isLast @@ -3362,6 +3411,14 @@ function lowerPipelineStages( dest = 2; destPromise = lowered.out; srcT = null; + } else if (isLast) { + // an async generator destination: pumped into the object-mode + // PassThrough that pipeline() answers + const helpers = fromGenHelpers(lowerer, lowered.out, loc); + if (!helpers) lowerer.noLowering("a pipeline destination with this generator type", arg); + helperArgs.push(helpers![0]!, helpers![1]!, helpers![2]!); + dest = 1; + srcT = null; } else { srcT = lowered.out; } @@ -3397,7 +3454,8 @@ function lowerPipelineStages( argDecls.length === 0 ? call : { kind: "seqExpr", stmts: argDecls, result: call, type: call.type, loc }; - // SCR_PIPE_* flags: bits 1-2 the destination, bit 5 the signal and abort. + // SCR_PIPE_* flags: bits 1-2 the destination, bit 5 the signal and + // abort (bit 6, the promise form's options signal, is set by its caller). const flags = (dest << 1) | (mask !== 0 ? 32 : 0); const head: IrExpr[] = [ numLit(streams.length, loc), @@ -3410,7 +3468,7 @@ function lowerPipelineStages( ...helperArgs, ]; const last: IrType = - dest === 2 ? { kind: "object", className: "%PassThrough" } : streams[streams.length - 1]!.type; + dest !== 0 ? { kind: "object", className: "%PassThrough" } : streams[streams.length - 1]!.type; if (mask === 0) return { args: head, last, dest, destPromise, wrap: bindArgs }; // const ac = new AbortController(): its signal goes to the stages, and // finishImpl's ac.abort() runs through the abort closure. @@ -3622,6 +3680,166 @@ function lowerStageFunction( } } +/** The async generator a Duplex.from body iterates (duplexify.js's + * fromAsyncGen source), as a closure the runtime calls with next() and + * ack(): each pass awaits next() (the deferred a write or the final + * settles), calls ack() (process.nextTick(cb) and a fresh deferred), + * returns at the done marker, and yields the chunk. */ +function duplexSourceHelper(lowerer: Lowerer, genT: SyncGenType, loc: SrcLoc): IrExpr | null { + const nextT = funcOf([], { kind: "promise", inner: DYN }); + const ackT = funcOf([], VOID); + const type = funcOf([nextT, ackT], genT); + const key = "duplexSource"; + let name = lowerer.valueHelpers.get(key); + if (!name) { + const resultType = genResultRecord(DYN, VOID, lowerer.shapes, lowerer.unions); + if (!resultType) return null; + name = `%stream.duplexSource.${lowerer.valueHelpers.size}`; + const frame = new LiftedFrame(loc); + const nextId = frame.param("next", nextT); + const ackId = frame.param("ack", ackT); + const valueId = frame.local("value", DYN, false); + const value = varRef(valueId, DYN, loc); + const body: IrStmt[] = [ + { + kind: "while", + cond: { kind: "boolLit", value: true, type: BOOL, loc }, + body: [ + { + kind: "varDecl", + localId: valueId, + init: { + kind: "awaitExpr", + value: { + kind: "callValue", + callee: varRef(nextId, nextT, loc), + args: [], + type: { kind: "promise", inner: DYN }, + loc, + }, + type: DYN, + loc, + }, + loc, + }, + { + kind: "exprStmt", + expr: { + kind: "callValue", + callee: varRef(ackId, ackT, loc), + args: [], + type: VOID, + loc, + }, + loc, + }, + { + kind: "if", + cond: { kind: "libCall", fn: "stream.chunkDone", args: [value], type: BOOL, loc }, + then: [{ kind: "return", value: null, loc }], + else_: null, + loc, + }, + { kind: "exprStmt", expr: { kind: "yieldExpr", value, type: VOID, loc }, loc }, + ], + loc, + }, + { kind: "return", value: null, loc }, + ]; + lowerer.valueHelpers.set(key, name); + lowerer.liftedFns.push({ + name, + params: frame.params, + returnType: VOID, + async: true, + locals: frame.locals, + body, + generator: { yieldT: DYN, nextT: UNDEFINED_T, resultType }, + loc, + }); + } + return { kind: "closure", fnName: name, captures: [], type, loc }; +} + +/** Duplex.from(body) (duplexify.js) for the bodies with a lowering: an + * async generator function, whose source generator the Duplex's writes + * feed (stream.duplexFromGen); a duplex stream, which answers itself; and + * the bodies Readable.from reads (iterables, strings, generators), as a + * Duplexify that is not writable. */ +function duplexFromBody(lowerer: Lowerer, node: ts.Expression, loc: SrcLoc): IrExpr { + const duplex: IrType = { kind: "object", className: "%Duplex" }; + const fn = pipelineStageFunction(lowerer, node); + if (fn !== null) { + const isAsync = (fn.modifiers?.nodes ?? []).some((m) => m.kind === ts.SyntaxKind.AsyncKeyword); + const isGen = !ts.isArrowFunction(fn) && fn.asteriskToken !== undefined; + if (!isAsync || !isGen) { + lowerer.noLowering( + "Duplex.from over a function that is not an async generator function", + node, + "write the body as async function* (source) { for await (const chunk of source) ... }", + ); + } + if (fn.parameters.nodes.length > 1) { + lowerer.noLowering( + "a Duplex.from body that declares the options argument", + fn.parameters.nodes[1]!, + "the body takes only its source", + ); + } + const sourceT: SyncGenType = { + kind: "generator", + async: true, + yieldT: DYN, + retT: VOID, + nextT: UNDEFINED_T, + }; + const lowered = lowerStageFunction(lowerer, fn, sourceT); + const helpers = + lowered.out.kind === "generator" ? fromGenHelpers(lowerer, lowered.out, loc) : null; + const source = duplexSourceHelper(lowerer, sourceT, loc); + if (!helpers || !source) + lowerer.noLowering("Duplex.from over a function with this generator type", node); + return { + kind: "libCall", + fn: "stream.duplexFromGen", + args: [lowered.closure, source!, ...helpers!], + type: duplex, + loc, + }; + } + // duplexify.js: a stream answers itself (a Duplex) or is wrapped; any + // other body reads as from(Duplexify, body, { objectMode: true, + // writable: false }) + let bodyT: IrType | null; + try { + bodyT = lowerer.mapTypeOf(lowerer.typeOf(node)); + } catch { + bodyT = null; + } + const bodySides = + bodyT?.kind === "object" ? streamSidesOf(lowerer, lowerer.classes.get(bodyT.className)) : null; + if (bodySides === null) return lowerFromSource(lowerer, node, node, undefined, true); + let stream: IrExpr; + try { + stream = lowerStreamArg(lowerer, node, "compose"); + } catch (e) { + if (e instanceof StreamArgTypeThrow) return e.expr; + throw e; + } + const sides = + stream.type.kind === "object" + ? streamSidesOf(lowerer, lowerer.classes.get(stream.type.className)) + : null; + if (sides !== "rw") { + lowerer.noLowering( + `Duplex.from over a ${sides === "r" ? "Readable" : sides === "w" ? "Writable" : "value of this type"}`, + node, + "an async generator function, an array, a string, a generator, an async generator, or a Duplex, Transform or PassThrough is the lowered body", + ); + } + return stream; +} + /** The checker type of a stage's string or Buffer chunks, or null for * other chunk types. */ function stageChunkCheckerType(lowerer: Lowerer, chunk: IrType, at: ts.Node): ts.Type | null { @@ -3636,12 +3854,8 @@ function stageChunkCheckerType(lowerer: Lowerer, chunk: IrType, at: ts.Node): ts * classes fix which positions can be readable and writable; a stream in * the wrong position, a single Readable or Writable (Duplex.from wraps * those), and functions, iterables or web streams have no lowering. */ -function lowerStreamCompose(lowerer: Lowerer, call: ts.CallExpression, loc: SrcLoc): IrExpr { - const args = call.arguments.nodes; +function lowerStreamCompose(lowerer: Lowerer, args: readonly ts.Expression[], loc: SrcLoc): IrExpr { const duplex: IrType = { kind: "object", className: "%Duplex" }; - if (args.some((a) => ts.isSpreadElement(a))) { - lowerer.noLowering("compose with spread arguments", call, "pass the streams directly"); - } if (args.length === 0) { return nodeThrowExpr( 1, @@ -3651,9 +3865,43 @@ function lowerStreamCompose(lowerer: Lowerer, call: ts.CallExpression, loc: SrcL loc, ); } + for (const a of args) { + // (typeOf on a stage function's name would report what collecting its + // declared signature deferred) + if (pipelineStageFunction(lowerer, a) !== null) continue; + if (staticReadableStreamElementType(lowerer, a) !== null) { + lowerer.noLowering( + "compose with a web ReadableStream", + a, + "web streams do not convert to Node streams yet; compose Node streams, iterables or async generator functions", + ); + } + } + // Duplex.from(stage) for one stage (duplexify.js), and for a function at + // either end; functions between, an iterable first stage or a source + // function run as the pipeline's function stages (compose.js) + if (args.length === 1) return duplexFromBody(lowerer, args[0]!, loc); + const last = args.length - 1; + const isFn = (i: number): boolean => pipelineStageFunction(lowerer, args[i]!) !== null; + const pre = new Map(); + if (isFn(0)) pre.set(0, duplexFromBody(lowerer, args[0]!, loc)); + if (isFn(last)) pre.set(last, duplexFromBody(lowerer, args[last]!, loc)); + const staged = + args.some((_, i) => !pre.has(i) && isFn(i)) || + (!pre.has(0) && iterableSourceType(lowerer, args[0]!) !== null); + if (staged) { + const lowered = lowerPipelineStages(lowerer, args, loc, pre); + return lowered.wrap({ + kind: "libCall", + fn: "stream.composeStages", + args: lowered.args, + type: duplex, + loc, + }); + } let streams: IrExpr[]; try { - streams = args.map((a) => lowerStreamArg(lowerer, a, "compose")); + streams = args.map((a, i) => pre.get(i) ?? lowerStreamArg(lowerer, a, "compose")); } catch (e) { if (e instanceof StreamArgTypeThrow) return e.expr; throw e; @@ -4179,6 +4427,80 @@ function pipeIntoDyn( }; } +/** readable.compose(stage, options): compose(this, stage), with + * addAbortSignal(options.signal, composed) when the signal is set + * (operators.js). */ +function lowerReadableCompose( + lowerer: Lowerer, + call: ts.CallExpression, + access: ts.PropertyAccessExpression, +): IrExpr { + const loc = locOf(call); + const args = call.arguments.nodes; + if (args.length === 0 || args.length > 2 || args.some((a) => ts.isSpreadElement(a))) { + lowerer.noLowering( + `readable.compose with ${args.length === 0 ? "no stage" : "these arguments"}`, + call, + "the supported form is readable.compose(stage) or readable.compose(stage, { signal })", + ); + } + let signalNode: ts.Expression | null = null; + const options = args[1]; + if ( + options !== undefined && + options.kind !== ts.SyntaxKind.NullKeyword && + !(ts.isIdentifier(options) && options.text === "undefined") + ) { + if (!ts.isObjectLiteralExpression(options)) { + lowerer.noLowering( + "readable.compose options that are not an object literal", + options, + "pass { signal } as an object literal", + ); + } + for (const p of options.properties.nodes) { + const key = + (ts.isPropertyAssignment(p) || ts.isShorthandPropertyAssignment(p)) && + ts.isIdentifier(p.name) + ? p.name.text + : null; + if (key !== "signal") { + lowerer.noLowering( + key === null + ? "readable.compose options with this member" + : `the readable.compose option '${key}'`, + p, + "signal is the readable.compose option", + ); + } + const value = ts.isShorthandPropertyAssignment(p) + ? (p.name as ts.Identifier) + : (p as ts.PropertyAssignment).initializer; + if (value.kind === ts.SyntaxKind.NullKeyword) continue; + if (ts.isIdentifier(value) && value.text === "undefined") continue; + // The signal is read after compose runs, so only values whose + // evaluation has no effects. + if (!sideEffectFreeOptionValue(value) && !pureRead(value)) { + lowerer.noLowering( + "the readable.compose option 'signal' with this value", + p, + "pass a variable or property read (const signal = controller.signal)", + ); + } + signalNode = value; + } + } + const composed = lowerStreamCompose(lowerer, [access.expression, args[0]!], loc); + if (signalNode === null) return composed; + return { + kind: "libCall", + fn: "stream.addAbortSignal", + args: [composed, lowerer.lowerExprExpecting(signalNode, DYN)], + type: composed.type, + loc, + }; +} + export function lowerStreamMethodCall( lowerer: Lowerer, call: ts.CallExpression, @@ -4191,6 +4513,12 @@ export function lowerStreamMethodCall( if (opSides === "r" || opSides === "rw") return lowerStreamOperator(lowerer, call, access, member); } + if (member === "compose") { + const composeSides = streamSidesOf(lowerer, info); + return composeSides === "r" || composeSides === "rw" + ? lowerReadableCompose(lowerer, call, access) + : null; + } if (!STREAM_API_MEMBERS.has(member)) return null; const sides = streamSidesOf(lowerer, info); if (!sides) return null; diff --git a/packages/compiler/src/ir/builtin-effects.ts b/packages/compiler/src/ir/builtin-effects.ts index 6ddee1597..bab07c458 100644 --- a/packages/compiler/src/ir/builtin-effects.ts +++ b/packages/compiler/src/ir/builtin-effects.ts @@ -340,7 +340,9 @@ export const MAY_THROW_LIB_FNS: ReadonlySet = new Set([ "sp.finished", "sp.pipeline", "sp.pipelineStages", + "sp.pipelineStagesOpts", "stream.pipelineStages", + "stream.composeStages", "stream.pipelineStagesDyn", "sc.text", "sc.json", @@ -380,6 +382,8 @@ export const MAY_THROW_LIB_FNS: ReadonlySet = new Set([ "writable.end", "writable.uncork", "writable.setDefaultEncoding", + "stream.duplexFromGen", + "stream.duplexFromDyn", "stream.destroy", "stream.destroyErr", "stream.iteratorClose", diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index e561c0dbf..1708ad01a 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -3959,6 +3959,7 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ "stream.pipeline": { argTypes: [F64], result: VOID }, "stream.pipelineDyn": { argTypes: [F64], result: VOID }, "stream.pipelineStages": { argTypes: [F64, F64, F64, F64, F64], result: VOID }, + "stream.composeStages": { argTypes: [F64, F64, F64, F64, F64], result: VOID }, "stream.pipelineStagesDyn": { argTypes: [F64, F64, F64, F64, F64], result: VOID }, /** stream.compose(count, s1..sn): the Duplex over a pipeline of the * streams (+1). MAY THROW (ERR_INVALID_ARG_VALUE for a stream that is @@ -4003,6 +4004,12 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ argTypes: [F64, F64, F64, F64, F64], result: { kind: "promise", inner: VOID }, }, + /** sp.pipelineStages with the options argument: the same head, then + * end (a bool) and, with flags bit 6, the options' signal (dyn). */ + "sp.pipelineStagesOpts": { + argTypes: [F64, F64, F64, F64, F64], + result: { kind: "promise", inner: VOID }, + }, /** node:stream/consumers — the promise consumers over the readable * machinery: (s) → a pending promise settled at the terminal point * with the accumulated result (sc.text: the utf8 decode, sc.json: the @@ -4060,6 +4067,8 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ * after construction; answers the receiver (+1, the chaining shape). */ "stream.objectMode": { argTypes: [null, BOOL, BOOL], result: VOID }, "stream.decodeStrings": { argTypes: [null, BOOL], result: VOID }, + "stream.duplexFromGen": { argTypes: [null, null, null, null, null, null], result: VOID }, + "stream.duplexFromDyn": { argTypes: [DYN], result: VOID }, "writable.writeEncoding": { argTypes: [null, STRING], result: VOID }, "writable.setDefaultEncoding": { argTypes: [null, DYN], result: VOID }, /** Whether a readable.nextChunkDyn step answered the end (the done @@ -4815,10 +4824,12 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ export type IrLibFn = keyof typeof LIB_FN_SIGS; /** The function-stage runs of a pipeline of `total` stages that a stream - * follows (positions bit j: stage j is a function): each pumps its last - * generator into that stream through one (next, read, return) triple. */ -export function pipelinePumpedRuns(total: number, positions: number): number { - let runs = 0; + * follows (positions bit j: stage j is a function), and an async + * generator destination's run (flags bits 1-2 = 1, pumped into the + * PassThrough pipeline() answers): each pumps its last generator through + * one (next, read, return) triple. */ +export function pipelinePumpedRuns(total: number, positions: number, flags: number): number { + let runs = ((flags >> 1) & 3) === 1 ? 1 : 0; for (let j = 0; j + 1 < total; j++) { if (Math.floor(positions / 2 ** j) % 2 === 1 && Math.floor(positions / 2 ** (j + 1)) % 2 === 0) runs++; diff --git a/packages/compiler/src/ir/runtime-features.ts b/packages/compiler/src/ir/runtime-features.ts index 555f5d306..da9cab1c3 100644 --- a/packages/compiler/src/ir/runtime-features.ts +++ b/packages/compiler/src/ir/runtime-features.ts @@ -219,6 +219,8 @@ const STREAM_LIB_FNS: ReadonlySet = new Set([ "stream.chunkDone", "stream.objectMode", "stream.decodeStrings", + "stream.duplexFromGen", + "stream.duplexFromDyn", "console.bind", "crypto.hashStream", "crypto.hashStreamPiped", @@ -231,6 +233,7 @@ const STREAM_LIB_FNS: ReadonlySet = new Set([ "sp.finished", "sp.pipeline", "sp.pipelineStages", + "sp.pipelineStagesOpts", "stream.addAbortSignal", "stream.addAbortSignalFn", "stream.destroy", @@ -258,6 +261,7 @@ const STREAM_LIB_PREFIXES: readonly string[] = [ "sp.pipelineOpts", "stream.finishedOpts", "stream.pipelineStages", + "stream.composeStages", "stream.set", "transform.", "writable.", @@ -271,6 +275,8 @@ const READABLE_DYN_ASYNC_LIB_FNS: ReadonlySet = new Set([ "readable.fromDone", "readable.fromDyn", "readable.fromIter", + "stream.duplexFromGen", + "stream.duplexFromDyn", ]); /** libCall name prefixes of the Readable entry points that await through the dyn-async unit. */ diff --git a/packages/compiler/src/ir/validate.ts b/packages/compiler/src/ir/validate.ts index 68bc61b21..a76784083 100644 --- a/packages/compiler/src/ir/validate.ts +++ b/packages/compiler/src/ir/validate.ts @@ -4101,8 +4101,10 @@ function validateFunction( e.fn === "stream.pipelineDyn" || e.fn === "stream.compose" || e.fn === "stream.pipelineStages" || + e.fn === "stream.composeStages" || e.fn === "stream.pipelineStagesDyn" || e.fn === "sp.pipelineStages" || + e.fn === "sp.pipelineStagesOpts" || e.fn === "sp.pipeline" || e.fn === "sp.pipelineOpts" || e.fn === "sp.pipelineOptsSignal" || @@ -5495,6 +5497,8 @@ function validateFunction( e.fn === "stream.destroyErr" || e.fn === "stream.objectMode" || e.fn === "stream.decodeStrings" || + e.fn === "stream.duplexFromGen" || + e.fn === "stream.duplexFromDyn" || e.fn === "stream.setConstruct" || e.fn === "stream.setWritev" || e.fn === "stream.setWritevDefault" || @@ -5507,8 +5511,10 @@ function validateFunction( e.fn === "stream.pipelineDyn" || e.fn === "stream.compose" || e.fn === "stream.pipelineStages" || + e.fn === "stream.composeStages" || e.fn === "stream.pipelineStagesDyn" || e.fn === "sp.pipelineStages" || + e.fn === "sp.pipelineStagesOpts" || e.fn === "stream.addAbortSignal" || e.fn === "stream.addAbortSignalFn" || e.fn === "stream.finishedOpts" || @@ -5548,6 +5554,13 @@ function validateFunction( } return true; } + if (e.fn === "stream.duplexFromGen" || e.fn === "stream.duplexFromDyn") { + // Duplex.from: (stage, source, helpers...) or (body) → a Duplex. + if (!isStreamObject(e.type)) { + err(`libCall ${e.fn} must return a stream class, got ${e.type.kind}`, e.loc); + } + return true; + } if (e.fn.endsWith(".newDyn")) { // The dyn-options constructor: (optsDyn) → the stream class. if (!isStreamObject(e.type)) { @@ -5618,13 +5631,15 @@ function validateFunction( if ( e.fn === "stream.pipelineStages" || e.fn === "stream.pipelineStagesDyn" || - e.fn === "sp.pipelineStages" + e.fn === "sp.pipelineStages" || + e.fn === "sp.pipelineStagesOpts" || + e.fn === "stream.composeStages" ) { // (n, k, positions, flags, mask, s1..sn, stage1..stagek, - // helpers..., box?, (signal, abort)?[, cb]): positions bit j marks - // stage j a function, flags bits 1-2 the destination, bit 5 the - // signal group; helpers are three per function run a stream - // follows. + // helpers..., box?, (signal, abort)?[, cb | end, signal?]): + // positions bit j marks stage j a function, flags bits 1-2 the + // destination, bit 5 the signal group, bit 6 the options' signal; + // helpers are three per pumped function run. const [n, k, positions, flags] = e.args .slice(0, 4) .map((a) => (a?.kind === "numLit" ? a.value : -1)); @@ -5635,9 +5650,14 @@ function validateFunction( if (Math.floor(positions! / 2 ** j) % 2 === 1) functions++; } const lastIsFunction = Math.floor(positions! / 2 ** (total - 1)) % 2 === 1; - const helperCount = 3 * pipelinePumpedRuns(total, positions!); + const helperCount = 3 * pipelinePumpedRuns(total, positions!, flags!); const groups = helperCount + (destKind === 2 ? 1 : 0) + (flags! & 32 ? 2 : 0); - const cbArgs = e.fn === "sp.pipelineStages" ? 0 : 1; + const cbArgs = + e.fn === "sp.pipelineStagesOpts" + ? 1 + (flags! & 64 ? 1 : 0) + : e.fn === "sp.pipelineStages" || e.fn === "stream.composeStages" + ? 0 + : 1; if ( flags! < 0 || n! < 0 || @@ -5646,7 +5666,8 @@ function validateFunction( total > 52 || positions! >= 2 ** total || functions !== k || - (destKind === 2) !== lastIsFunction || + destKind === 3 || + (destKind !== 0) !== lastIsFunction || total < 2 || e.args.length !== 5 + n! + k! + groups + cbArgs ) { @@ -5674,12 +5695,19 @@ function validateFunction( if (e.args[at2]?.type.kind !== "dyn" || e.args[at2 + 1]?.type.kind !== "func") err(`libCall ${e.fn} signal and abort must be a dyn and a func`, e.loc); } - if (cbArgs) { + if (e.fn === "sp.pipelineStagesOpts") { + const tail = e.args.slice(e.args.length - cbArgs); + if (tail[0]?.type.kind !== "bool" || (cbArgs > 1 && tail[1]?.type.kind !== "dyn")) + err(`libCall ${e.fn} end and signal must be a bool and a dyn`, e.loc); + if (e.type.kind !== "promise") err(`libCall ${e.fn} must return a promise`, e.loc); + } else if (cbArgs) { const cbK = e.args[e.args.length - 1]?.type.kind; if (e.fn === "stream.pipelineStages" ? cbK !== "func" : cbK !== "dyn") { err(`libCall ${e.fn} callback has the wrong kind`, e.loc); } if (!isStreamObject(e.type)) err(`libCall ${e.fn} must return a stream`, e.loc); + } else if (e.fn === "stream.composeStages") { + if (!isStreamObject(e.type)) err(`libCall ${e.fn} must return a stream`, e.loc); } else if (e.type.kind !== "promise") { err(`libCall ${e.fn} must return a promise`, e.loc); } diff --git a/packages/compiler/test/native-codegen-integration.test.ts b/packages/compiler/test/native-codegen-integration.test.ts index 35a747d60..71b9570b4 100644 --- a/packages/compiler/test/native-codegen-integration.test.ts +++ b/packages/compiler/test/native-codegen-integration.test.ts @@ -378,7 +378,7 @@ done: expect(await symbols(helperObject, ["-u"])).toEqual(await symbols(clangObject, ["-u"])); expect(await symbols(helperObject, ["-gU"])).toEqual(await symbols(clangObject, ["-gU"])); expect(await symbols(helperObject, ["-gU"])).toEqual(["0000000000000000 T _main"]); - expect(await symbols(helperObject, ["-u"])).toContain("_scr_runtime_abi_v27"); + expect(await symbols(helperObject, ["-u"])).toContain("_scr_runtime_abi_v28"); const clangExe = join(dir, "clang-program"); const linkDriver = join(dir, "clang-driver.c"); @@ -712,7 +712,7 @@ done: .split("\n"); expect(undefinedSymbols).toContain("_sf_scale"); expect(undefinedSymbols).toContain("_sf_callback_mix"); - expect(undefinedSymbols).toContain("_scr_runtime_abi_v27"); + expect(undefinedSymbols).toContain("_scr_runtime_abi_v28"); }); test("helper and clang object paths retain the same outbound FFI C ABI", async () => { diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7252-duplex-from-generator.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7252-duplex-from-generator.ts.json new file mode 100644 index 000000000..d29eed123 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7252-duplex-from-generator.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7252-duplex-from-generator.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7253-duplex-from.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7253-duplex-from.mjs.json new file mode 100644 index 000000000..e12ce5a52 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7253-duplex-from.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7253-duplex-from.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7254-compose-functions.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7254-compose-functions.ts.json new file mode 100644 index 000000000..07e468aa2 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7254-compose-functions.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7254-compose-functions.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7255-compose-stages.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7255-compose-stages.mjs.json new file mode 100644 index 000000000..036d9836e --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7255-compose-stages.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7255-compose-stages.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7256-readable-compose.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7256-readable-compose.mjs.json new file mode 100644 index 000000000..0b5993d32 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7256-readable-compose.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7256-readable-compose.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7257-pipeline-generator-destination.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7257-pipeline-generator-destination.mjs.json new file mode 100644 index 000000000..79354d9ce --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7257-pipeline-generator-destination.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7257-pipeline-generator-destination.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7258-pipeline-generator-destination.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7258-pipeline-generator-destination.ts.json new file mode 100644 index 000000000..313c9f19e --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7258-pipeline-generator-destination.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7258-pipeline-generator-destination.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7259-promises-pipeline-stage-options.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7259-promises-pipeline-stage-options.cjs.json new file mode 100644 index 000000000..6fc082ea1 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7259-promises-pipeline-stage-options.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7259-promises-pipeline-stage-options.cjs" + ], + "diags": [] +} diff --git a/packages/runtime-darwin-arm64/scripts/build.mjs b/packages/runtime-darwin-arm64/scripts/build.mjs index 631750cdf..3d22b2360 100644 --- a/packages/runtime-darwin-arm64/scripts/build.mjs +++ b/packages/runtime-darwin-arm64/scripts/build.mjs @@ -340,7 +340,7 @@ async function build() { package: packageManifest.name, version: packageManifest.version, target: RUNTIME_PACK_MATRIX.target, - runtime_abi: { version: 27, marker: "scr_runtime_abi_v27" }, + runtime_abi: { version: 28, marker: "scr_runtime_abi_v28" }, compiler: { command: compiler, identity: compilerVersion, diff --git a/packages/runtime-pack-common/scripts/build.mjs b/packages/runtime-pack-common/scripts/build.mjs index 0e81033ad..909412bc7 100644 --- a/packages/runtime-pack-common/scripts/build.mjs +++ b/packages/runtime-pack-common/scripts/build.mjs @@ -437,7 +437,7 @@ async function build() { package: packageManifest.name, version: packageManifest.version, target: matrix.target, - runtime_abi: { version: 27, marker: "scr_runtime_abi_v27" }, + runtime_abi: { version: 28, marker: "scr_runtime_abi_v28" }, compiler: { command: compiler, identity: compilerVersion, diff --git a/packages/runtime/src/scr_console.c b/packages/runtime/src/scr_console.c index a3ac43cc0..9bd5abf8c 100644 --- a/packages/runtime/src/scr_console.c +++ b/packages/runtime/src/scr_console.c @@ -148,7 +148,7 @@ static void scr_enable_utf8_console_output(void) { } #endif -void scr_runtime_abi_v27(void) {} +void scr_runtime_abi_v28(void) {} void scr_init(void) { #ifdef SCR_WORKERS diff --git a/packages/runtime/src/scr_core.h b/packages/runtime/src/scr_core.h index fea020d16..35332d0ca 100644 --- a/packages/runtime/src/scr_core.h +++ b/packages/runtime/src/scr_core.h @@ -81,7 +81,7 @@ void scr_init(void); /* Program objects emitted by the bundled LLVM helper reference this symbol. * Its versioned spelling makes a mismatched manual runtime link fail before * the program can start. */ -void scr_runtime_abi_v27(void); +void scr_runtime_abi_v28(void); void scr_runtime_workers_v8(void); /* ── the trap funnel (scr_console.c; scr_library.c under -DSCR_LIB) ────── diff --git a/packages/runtime/src/scr_stream.c b/packages/runtime/src/scr_stream.c index 691b6af8e..75debe615 100644 --- a/packages/runtime/src/scr_stream.c +++ b/packages/runtime/src/scr_stream.c @@ -1534,8 +1534,21 @@ static bool scr_from_check_hwm(double hwm) { * highWaterMark 1 for iterables, the mode's default for the string / * Buffer case or a nullish option, or the option's value. +1, or NULL with the error * pending. Moves f. */ +/* from.js's `new Readable(...)`: a Readable, or for Duplex.from (duplexify.js's + * from(Duplexify, body, { objectMode: true, writable: false })) a Duplexify + * whose writable side is off. */ +static ScrStream *scr_from_new(bool duplex, double hwm, ScrClosure *read, ScrStreamReadInv read_inv, + ScrClosure *destroy, ScrStreamErrInv destroy_inv) { + if (!duplex) return scr_stream_new_readable(hwm, true, true, read, read_inv, destroy, destroy_inv); + ScrStream *d = scr_stream_new_duplex(hwm, hwm, true, true, true, true, false, read, read_inv, NULL, + NULL, NULL, NULL, destroy, destroy_inv); + scr_stream_set_object_mode(d, false, true); + d->cls = "Duplexify"; + return d; +} + static ScrStream *scr_from_stream(ScrFromSrc *f, ScrStreamReadInv read_inv, bool object_mode, - int hwm_kind, double hwm, bool strings) { + int hwm_kind, double hwm, bool strings, bool duplex) { if (hwm_kind == SCR_FROM_HWM_SET && !scr_from_check_hwm(hwm)) { scr_from_src_release(f); return NULL; @@ -1547,8 +1560,8 @@ static ScrStream *scr_from_stream(ScrFromSrc *f, ScrStreamReadInv read_inv, bool scr_box_set_ref(read->caps[0], f); /* moves */ ScrClosure *destroy = NULL; if (!f->single) destroy = scr_closure_new((void *)&scr_from_destroy_inv, 0); - ScrStream *s = scr_stream_new_readable(hwm, true, true, read, read_inv, destroy, - destroy ? &scr_from_destroy_inv : NULL); + ScrStream *s = scr_from_new(duplex, hwm, read, read_inv, destroy, + destroy ? &scr_from_destroy_inv : NULL); ScrStreamState *st = s->st; if (object_mode) { st->r.object_entries = true; @@ -1575,7 +1588,7 @@ ScrStream *scr_stream_new_from_readable(ScrClosure *read, ScrStreamReadInv read_ bool object_mode = (bits & SCR_FROM_BYTE_MODE) == 0; if (bits & SCR_FROM_DEFAULT_HWM) hwm = object_mode ? 16 : -1; else if (!(bits & SCR_FROM_HAS_HWM)) hwm = 1; - ScrStream *s = scr_stream_new_readable(hwm, true, true, read, read_inv, destroy, destroy_inv); + ScrStream *s = scr_from_new((bits & SCR_FROM_DUPLEX) != 0, hwm, read, read_inv, destroy, destroy_inv); if (object_mode) s->st->r.object_entries = true; return s; } @@ -1590,7 +1603,7 @@ ScrStream *scr_stream_from_arr(ScrArr *arr, double flags, double hwm) { f->single = (bits & SCR_FROM_SINGLE) != 0; return scr_from_stream(f, &scr_from_read_inv, (bits & SCR_FROM_BYTE_MODE) == 0, (bits & SCR_FROM_HAS_HWM) ? SCR_FROM_HWM_SET : SCR_FROM_HWM_FROM, hwm, - f->strings); + f->strings, (bits & SCR_FROM_DUPLEX) != 0); } /* The JS lane: one checked-dynamic value into the stream. null is @@ -1654,7 +1667,7 @@ static void scr_from_read_dyn_inv(ScrClosure *cb, ScrStream *s, double size) { scr_from_src_release(f); } -ScrStream *scr_stream_from_dyn(ScrDyn *src, ScrDyn *opts) { +static ScrStream *scr_from_dyn_ex(ScrDyn *src, ScrDyn *opts, bool duplex) { bool single = src->kind == SCR_DYN_STR || src->kind == SCR_DYN_BYTES; ScrDyn *iter = NULL; /* from.js's iterator, for sources other than arrays */ int kind = 0; @@ -1666,7 +1679,14 @@ ScrStream *scr_stream_from_dyn(ScrDyn *src, ScrDyn *opts) { if (kind < 0) return NULL; } if (!single && kind == 0) { - scr_dyn_arg_type_fail("iterable", "an instance of Iterable", src); + if (duplex) { + scr_dyn_arg_type_fail("body", + "of type function or an instance of Blob, ReadableStream, WritableStream, " + "Stream, Iterable, AsyncIterable, or Promise or { readable, writable } pair", + src); + } else { + scr_dyn_arg_type_fail("iterable", "an instance of Iterable", src); + } return NULL; } bool object_mode = true; @@ -1713,7 +1733,7 @@ ScrStream *scr_stream_from_dyn(ScrDyn *src, ScrDyn *opts) { } } if (iter) { - double flags = (object_mode ? 0 : SCR_FROM_BYTE_MODE) | + double flags = (object_mode ? 0 : SCR_FROM_BYTE_MODE) | (duplex ? SCR_FROM_DUPLEX : 0) | (hwm_kind == SCR_FROM_HWM_SET ? SCR_FROM_HAS_HWM : hwm_kind == SCR_FROM_HWM_DEFAULT ? SCR_FROM_DEFAULT_HWM : 0); @@ -1724,7 +1744,39 @@ ScrStream *scr_stream_from_dyn(ScrDyn *src, ScrDyn *opts) { f->rc = 1; f->dyn = scr_dyn_retain(src); f->single = single; - return scr_from_stream(f, &scr_from_read_dyn_inv, object_mode, hwm_kind, hwm, false); + return scr_from_stream(f, &scr_from_read_dyn_inv, object_mode, hwm_kind, hwm, false, duplex); +} + +ScrStream *scr_stream_from_dyn(ScrDyn *src, ScrDyn *opts) { + return scr_from_dyn_ex(src, opts, false); +} + +/* Duplex.from(body) over a checked-dynamic body (duplexify.js): a native + * Duplex answers itself, an iterable reads as Duplex.from does, and the + * other bodies Node accepts (a lone Readable or Writable, a function, a + * promise, a { readable, writable } pair) are not supported yet. */ +static ScrStream *scr_stream_boxed(const ScrDyn *value); + +ScrStream *scr_stream_duplex_from_dyn(ScrDyn *body) { + ScrStream *native = scr_stream_of_handle(body); /* +1 */ + if (!native) native = scr_stream_boxed(body); /* +1 */ + if (native && native->st->has_r && native->st->has_w) return native; + bool unsupported = native != NULL; + if (native) scr_stream_release(native); + if (!unsupported && (body->kind == SCR_DYN_FUNC || body->kind == SCR_DYN_PROMISE)) unsupported = true; + if (!unsupported && body->kind == SCR_DYN_OBJ) { + static const char *const keys[] = {"then", "readable", "writable"}; + for (size_t i = 0; i < 3 && !unsupported; i++) { + ScrDyn *v = scr_dyn_obj_get(body, keys[i], strlen(keys[i])); + unsupported = v != NULL && v->kind != SCR_DYN_UNDEF; + } + } + if (unsupported) { + static const char msg[] = "Duplex.from over this body is not supported yet"; + scr_throw_error_msg_code(SCR_ERR_ERROR, msg, sizeof msg - 1, "SC2020"); + return NULL; + } + return scr_from_dyn_ex(body, NULL, true); } /* ── object-mode streams for the stream operators (scr_stream_ops.c) ──── */ @@ -3695,7 +3747,9 @@ typedef ScrPromise *(*ScrPipeNextFn)(ScrClosure *, ScrGen *); typedef ScrDyn *(*ScrPipeReadFn)(ScrClosure *, void *); typedef ScrDyn *(*ScrPipeBoxFn)(ScrClosure *, ScrPromise *); -enum { SCR_PIPE_DEST_STREAM, SCR_PIPE_DEST_PROMISE = 2 }; +enum { SCR_PIPE_DEST_STREAM, SCR_PIPE_DEST_GEN, SCR_PIPE_DEST_PROMISE }; +/* flags bit 7: the callback is typed code's (stream.pipelineStagesDyn) */ +enum { SCR_PIPE_TYPED_CB = 128 }; typedef struct { size_t count; /* the function stages */ @@ -3707,8 +3761,9 @@ typedef struct { int dest; /* the last stage: SCR_PIPE_DEST_* */ unsigned opts_mask; /* bit i: stages[i] takes the { signal } argument */ ScrDyn *signal; /* the pipeline's AbortSignal when opts_mask != 0 */ - /* (next, read, return) for each generator pumped into a stream, in - * pipeline order, or NULL */ + /* (next, read, return) for each generator pumped into a stream (an + * async generator destination's PassThrough included), in pipeline + * order, or NULL */ ScrClosure **helpers; ScrClosure *box; /* SCR_PIPE_DEST_PROMISE: boxes its promise as a dyn promise */ } ScrPipeStages; @@ -4043,7 +4098,7 @@ static void scr_pipe_dest_entry(ScrFiber *fiber, void *arg) { ctx->value = v; if (ctx->value_p) scr_promise_release(ctx->value_p); ctx->value_p = scr_promise_retain(d.typed); - if (pt && v && (v->kind == SCR_DYN_STR || v->kind == SCR_DYN_BYTES)) { + if (pt && v && v->kind != SCR_DYN_NULL && v->kind != SCR_DYN_UNDEF) { scr_stream_write_dyn(pt, v, NULL); if (scr_exc_pending()) scr_exc_clear(); } @@ -4147,17 +4202,22 @@ static ScrStream *scr_stream_pipeline_ex(size_t n, ScrStream **streams, ScrClosu ret_gen = out; continue; } - /* The destination function: pipeline() answers a PassThrough proxy - * that the promise's value feeds. Node's is in object mode; the - * frontend refuses programs that read it, so a byte-mode - * PassThrough plays its part in the pipeline's bookkeeping. */ + /* The destination function: pipeline() answers an object-mode + * PassThrough proxy that the promise's value or the generator's + * values feed. */ ScrDyn *pt_opts = scr_dyn_new_obj(); + scr_dyn_obj_set(pt_opts, "objectMode", 10, scr_dyn_new_bool(true)); ScrStream *pt = scr_stream_new_passthrough_dyn(pt_opts); scr_dyn_release(pt_opts); size_t pi = ctx->n++; ctx->streams[pi] = pt; ctx->finish_count++; - scr_pipe_dest_promise(ctx, out, fs->box, pi, end); + if (fs->dest == SCR_PIPE_DEST_GEN) { + /* pumpToNode(ret, pt, finish, { end }) */ + scr_pump_start(ctx, out, fs->helpers + 3 * h++, pt, end); + } else { + scr_pipe_dest_promise(ctx, out, fs->box, pi, end); + } /* destroyer(pt, false, true), with its cleanup as the last stream's */ ctx->destroyer[pi] = true; scr_pipe_on(pt, "close", scr_pipe_closure((void *)&scr_pipe_destroyer_close_l, ctx, pi), @@ -4245,12 +4305,14 @@ static ScrPipeStages scr_pipe_stages_of(double count, ScrClosure **stages, doubl } /* callback(error, value) through a checked-dynamic callback: Node passes - * both arguments. */ -static void scr_pipe_dyn_value_inv(ScrClosure *cb, ScrError *err, ScrDyn *value, ScrPromise *value_p) { - (void)value_p; + * both arguments. A typed callback (SCR_PIPE_TYPED_CB) takes null for no + * error, as the typed callback forms do. */ +static void scr_pipe_dyn_value_call(ScrClosure *cb, ScrError *err, ScrDyn *value, bool typed) { ScrDyn *fn = scr_stream_dynopt_target(cb); - ScrDyn *args[2] = {err ? scr_dyn_from_error(err) : scr_dyn_retain(scr_dyn_undefined()), + ScrDyn *none = typed ? scr_dyn_new_null() : scr_dyn_retain(scr_dyn_undefined()); + ScrDyn *args[2] = {err ? scr_dyn_from_error(err) : none, value && !err ? scr_dyn_retain(value) : scr_dyn_retain(scr_dyn_undefined())}; + if (err) scr_dyn_release(none); ScrDyn *r = scr_dyn_call(fn, args, 2, "callback"); scr_dyn_release(args[0]); scr_dyn_release(args[1]); @@ -4258,6 +4320,17 @@ static void scr_pipe_dyn_value_inv(ScrClosure *cb, ScrError *err, ScrDyn *value, if (r != NULL) scr_dyn_release(r); } +static void scr_pipe_dyn_value_inv(ScrClosure *cb, ScrError *err, ScrDyn *value, ScrPromise *value_p) { + (void)value_p; + scr_pipe_dyn_value_call(cb, err, value, false); +} + +static void scr_pipe_dyn_value_typed_inv(ScrClosure *cb, ScrError *err, ScrDyn *value, + ScrPromise *value_p) { + (void)value_p; + scr_pipe_dyn_value_call(cb, err, value, true); +} + /* pipeline(...) with function stages (see ScrPipeStages): positions * marks the function stages among the n + count stages, flags bits 1-2 * give the destination kind, and mask names the stages taking @@ -4279,7 +4352,9 @@ ScrStream *scr_stream_pipeline_stages_dyn(double n, ScrStream **streams, double ScrPipeStages fs = scr_pipe_stages_of(count, stages, positions, flags, mask, helpers, box, signal); ScrClosure *c = scr_stream_dynopt_clo(cb, (void *)&scr_stream_finished_dyn_inv); /* a function destination's value reaches the callback */ - ScrPipeValInv vi = fs.dest != SCR_PIPE_DEST_STREAM ? &scr_pipe_dyn_value_inv : NULL; + ScrPipeValInv vi = fs.dest == SCR_PIPE_DEST_STREAM ? NULL + : ((int)flags & SCR_PIPE_TYPED_CB) ? &scr_pipe_dyn_value_typed_inv + : &scr_pipe_dyn_value_inv; return scr_stream_pipeline_ex((size_t)n, streams, c, &scr_stream_finished_dyn_inv, true, NULL, NULL, &fs, abort, vi); } @@ -4333,7 +4408,7 @@ static void *scr_compose_retain_v(void *p) { static void scr_compose_release_v(void *p) { ScrCompose *c = p; if (--c->rc > 0) return; - scr_stream_release(c->head); + if (c->head) scr_stream_release(c->head); scr_stream_release(c->tail); free(c); } @@ -4393,7 +4468,9 @@ static void scr_compose_destroyed(ScrCompose *c, ScrError *err) { /* d._write: head.write(chunk) ? callback() : (ondrain = callback) */ static void scr_compose_write_inv(ScrClosure *cb, ScrStream *s, ScrBytes *chunk) { ScrCompose *c = scr_box_get_ref(cb->caps[0]); - if (scr_stream_write(c->head, chunk, NULL)) { + bool ok = scr_stream_writes_values(s) ? scr_stream_write_dyn(c->head, (ScrDyn *)chunk, NULL) + : scr_stream_write(c->head, chunk, NULL); + if (ok) { if (!scr_exc_pending()) scr_stream_write_done(s, NULL); } else { c->ondrain = true; @@ -4524,43 +4601,54 @@ static void scr_compose_arg_value(size_t i, const char *what, ScrStream *s) { scr_throw_error_msg_code(SCR_ERR_TYPE, m, (size_t)n, "ERR_INVALID_ARG_VALUE"); } -ScrStream *scr_stream_compose(double n_d, ScrStream **streams) { - size_t n = (size_t)n_d; - if (n == 0) { +/* compose(...streams) over n streams and fs's function stages: the stream + * at pipeline position j must be readable unless it is last and writable + * unless it is first. A first stage that is not a stream (an iterable or + * source function) leaves the Duplex without a writable side. */ +static ScrStream *scr_stream_compose_impl(size_t n, ScrStream **streams, const ScrPipeStages *fs, + ScrClosure *abort) { + size_t total = n + (fs ? fs->count : 0); + if (total == 0) { static const char msg[] = "The \"streams\" argument must be specified"; scr_throw_error_msg_code(SCR_ERR_TYPE, msg, sizeof msg - 1, "ERR_MISSING_ARGS"); return NULL; } - if (n == 1) return scr_stream_retain(streams[0]); /* Duplex.from(duplex) */ - for (size_t i = 0; i < n; i++) { - if (i + 1 < n && !scr_stream_prop(streams[i], "readable")) { - scr_compose_arg_value(i, "readable", streams[i]); + if (total == 1 && n == 1) return scr_stream_retain(streams[0]); /* Duplex.from(duplex) */ + for (size_t j = 0, i = 0; j < total; j++) { + if (fs && j < 64 && ((fs->positions >> j) & 1u)) continue; + if (j + 1 < total && !scr_stream_prop(streams[i], "readable")) { + scr_compose_arg_value(j, "readable", streams[i]); return NULL; } - if (i > 0 && !scr_stream_prop(streams[i], "writable")) { - scr_compose_arg_value(i, "writable", streams[i]); + if (j > 0 && !scr_stream_prop(streams[i], "writable")) { + scr_compose_arg_value(j, "writable", streams[i]); return NULL; } + i++; } + bool stage_head = fs && (fs->positions & 1u); ScrCompose *c = calloc(1, sizeof *c); if (!c) scr_stream_oom(); c->rc = 1; - c->head = scr_stream_retain(streams[0]); + c->head = stage_head ? NULL : scr_stream_retain(streams[0]); c->tail = scr_stream_retain(streams[n - 1]); c->onclose = SCR_COMPOSE_RUNNING; ScrStream *tail = scr_stream_pipeline_ex(n, streams, scr_compose_closure((void *)&scr_compose_finished, c), - &scr_compose_finished, true, NULL, NULL, NULL, NULL, + &scr_compose_finished, true, NULL, NULL, fs, abort, NULL); if (!tail) { scr_compose_release_v(c); return NULL; } scr_stream_release(tail); - c->writable = scr_stream_prop(c->head, "writable") != 0; + c->writable = c->head && scr_stream_prop(c->head, "writable") != 0; c->readable = scr_stream_prop(c->tail, "readable") != 0; + /* readableObjectMode: !!tail?.readableObjectMode, writableObjectMode: + * !!head?.writableObjectMode */ bool object_mode = c->tail->st->has_r && c->tail->st->r.object_entries; + bool w_object = c->head && c->head->st->has_w && c->head->st->w.object_mode; ScrStream *d = scr_stream_new_duplex( - object_mode ? 16 : -1, -1, true, true, true, c->readable, c->writable, + object_mode ? 16 : -1, w_object ? 16 : -1, true, true, true, c->readable, c->writable, c->readable ? scr_compose_closure((void *)&scr_compose_read_inv, c) : NULL, c->readable ? &scr_compose_read_inv : NULL, c->writable ? scr_compose_closure((void *)&scr_compose_write_inv, c) : NULL, @@ -4569,6 +4657,7 @@ ScrStream *scr_stream_compose(double n_d, ScrStream **streams) { c->writable ? &scr_compose_final_inv : NULL, scr_compose_closure((void *)&scr_compose_destroy_inv, c), &scr_compose_destroy_inv); if (object_mode) d->st->r.object_entries = true; + if (w_object) scr_stream_set_object_mode(d, false, true); c->d = scr_stream_retain(d); if (c->writable) { scr_compose_on(c->head, "drain", &c->drain_l, (void *)&scr_compose_drain_l, c, scr_ee_inv_fixed0); @@ -4590,6 +4679,19 @@ ScrStream *scr_stream_compose(double n_d, ScrStream **streams) { return d; } +ScrStream *scr_stream_compose(double n_d, ScrStream **streams) { + return scr_stream_compose_impl((size_t)n_d, streams, NULL, NULL); +} + +/* compose(...stages) with function stages among the streams (the + * pipeline stage layout: scr_stream_pipeline_stages). */ +ScrStream *scr_stream_compose_stages(double n, ScrStream **streams, double count, ScrClosure **stages, + double positions, double flags, double mask, ScrClosure **helpers, + ScrClosure *box, ScrDyn *signal, ScrClosure *abort) { + ScrPipeStages fs = scr_pipe_stages_of(count, stages, positions, flags, mask, helpers, box, signal); + return scr_stream_compose_impl((size_t)n, streams, &fs, abort); +} + /* ── node:stream/promises ───────────────────────────────────────────── * The promise forms ride the callback machinery above with a settling * watcher: caps[0] boxes the pending promise; the terminal status @@ -4666,6 +4768,24 @@ ScrPromise *scr_sp_pipeline_stages(double n, ScrStream **streams, double count, return p; } +/* The options' signal: NULL for a nullish one; false (p rejected with + * ERR_INVALID_ARG_TYPE) for a value that is not an AbortSignal. */ +static bool scr_sp_opts_signal(ScrPromise *p, ScrDyn **signal) { + if (*signal && scr_dyn_is_nullish(*signal)) *signal = NULL; + if (!*signal) return true; + ScrDyn *aborted = NULL; + if ((*signal)->kind == SCR_DYN_OBJ || (*signal)->kind == SCR_DYN_HANDLE) { + aborted = scr_stream_signal_prop(*signal, "aborted"); + } + if (!aborted) { + scr_dyn_prop_type_fail("options.signal", "an instance of AbortSignal", *signal); + scr_promise_reject_pending(p); + return false; + } + scr_dyn_release(aborted); + return true; +} + /* pipeline(...streams, { signal, end }): end false leaves the destination * open (the promise settles once the stages before it finish); signal is * NULL or borrowed — a value that is not an AbortSignal rejects with @@ -4673,19 +4793,7 @@ ScrPromise *scr_sp_pipeline_stages(double n, ScrStream **streams, double count, ScrPromise *scr_sp_pipeline_opts(double n, ScrStream **streams, bool end, ScrDyn *signal, ScrAbortAddFn add) { ScrPromise *p = scr_promise_new(); - if (signal && scr_dyn_is_nullish(signal)) signal = NULL; - if (signal) { - ScrDyn *aborted = NULL; - if (signal->kind == SCR_DYN_OBJ || signal->kind == SCR_DYN_HANDLE) { - aborted = scr_stream_signal_prop(signal, "aborted"); - } - if (!aborted) { - scr_dyn_prop_type_fail("options.signal", "an instance of AbortSignal", signal); - scr_promise_reject_pending(p); - return p; - } - scr_dyn_release(aborted); - } + if (!scr_sp_opts_signal(p, &signal)) return p; ScrStream *dst = scr_stream_pipeline_ex((size_t)n, streams, scr_sp_watcher(p), &scr_sp_settle_inv, end, signal, add, NULL, NULL, NULL); if (dst) scr_stream_release(dst); @@ -4693,6 +4801,25 @@ ScrPromise *scr_sp_pipeline_opts(double n, ScrStream **streams, bool end, ScrDyn return p; } +/* scr_sp_pipeline_stages with the options argument: end and the + * options' signal (outer, NULL or borrowed) as scr_sp_pipeline_opts + * takes them. The stages' signal is the pipeline's own. */ +ScrPromise *scr_sp_pipeline_stages_opts(double n, ScrStream **streams, double count, + ScrClosure **stages, double positions, double flags, + double mask, ScrClosure **helpers, ScrClosure *box, + ScrDyn *signal, ScrClosure *abort, bool end, + ScrDyn *outer, ScrAbortAddFn add) { + ScrPromise *p = scr_promise_new(); + if (!scr_sp_opts_signal(p, &outer)) return p; + ScrPipeStages fs = scr_pipe_stages_of(count, stages, positions, flags, mask, helpers, box, signal); + ScrPipeValInv vi = fs.dest != SCR_PIPE_DEST_STREAM ? &scr_sp_value_inv : NULL; + ScrStream *dst = scr_stream_pipeline_ex((size_t)n, streams, scr_sp_watcher(p), &scr_sp_settle_inv, + end, outer, add, &fs, abort, vi); + if (dst) scr_stream_release(dst); + else scr_promise_reject_pending(p); + return p; +} + /* ── node:stream/consumers ──────────────────────────────────────────── * text/json/buffer/arrayBuffer over the readable machinery: a native 'data' listener * accumulates every chunk (Buffer chunks as-is; string chunks — an diff --git a/packages/runtime/src/scr_stream.h b/packages/runtime/src/scr_stream.h index c78a03e1a..0e0416e05 100644 --- a/packages/runtime/src/scr_stream.h +++ b/packages/runtime/src/scr_stream.h @@ -225,6 +225,9 @@ ScrStream *scr_stream_pipeline_stages(double n, ScrStream **streams, double coun double mask, ScrClosure **helpers, ScrClosure *box, struct ScrDyn *signal, ScrClosure *abort, ScrClosure *cb, ScrStreamErrInv inv); +ScrStream *scr_stream_compose_stages(double n, ScrStream **streams, double count, ScrClosure **stages, + double positions, double flags, double mask, ScrClosure **helpers, + ScrClosure *box, ScrDyn *signal, ScrClosure *abort); ScrStream *scr_stream_pipeline_stages_dyn(double n, ScrStream **streams, double count, ScrClosure **stages, double positions, double flags, double mask, ScrClosure **helpers, ScrClosure *box, @@ -331,6 +334,7 @@ enum { SCR_FROM_BYTE_MODE = 4, SCR_FROM_HAS_HWM = 8, SCR_FROM_DEFAULT_HWM = 16, /* a nullish highWaterMark: the mode's default */ + SCR_FROM_DUPLEX = 32, /* Duplex.from: a Duplexify with its writable side off */ }; ScrStream *scr_stream_from_arr(ScrArr *arr, double flags, double hwm); /* Readable.from(iterable, options) over a checked-dynamic source: a @@ -447,6 +451,12 @@ ScrPromise *scr_sp_finished_opts(ScrStream *s, double readable, double writable, * ERR_INVALID_ARG_TYPE. Streams borrowed. */ ScrPromise *scr_sp_pipeline_opts(double n, ScrStream **streams, bool end, ScrDyn *signal, ScrAbortAddFn add); +/* scr_sp_pipeline_stages with the options' end and signal (outer). */ +ScrPromise *scr_sp_pipeline_stages_opts(double n, ScrStream **streams, double count, + ScrClosure **stages, double positions, double flags, + double mask, ScrClosure **helpers, ScrClosure *box, + ScrDyn *signal, ScrClosure *abort, bool end, + ScrDyn *outer, ScrAbortAddFn add); double scr_stream_prop(ScrStream *s, const char *name); ScrError *scr_stream_errored(ScrStream *s); /* +1 or NULL */ @@ -519,6 +529,14 @@ void scr_stream_set_writev_concat(ScrStream *s); void scr_stream_set_construct(ScrStream *s, ScrClosure *cb, ScrStreamPlainInv inv); void scr_stream_set_writev(ScrStream *s, ScrClosure *cb, ScrStreamWritevInv inv); void scr_stream_set_writev_default(ScrStream *s, ScrClosure *cb, ScrStreamWritevInv inv); +/* Duplex.from (duplexify.js): over an async generator function (the + * compiled stage takes the source generator that the compiled source + * makes from next() and ack(); all borrowed, helpers too), and over a + * checked-dynamic body (borrowed). +1 each. The typed iterable bodies take + * the Readable.from builders with SCR_FROM_DUPLEX. */ +ScrStream *scr_stream_duplex_from_gen(ScrClosure *stage, ScrClosure *source, ScrClosure *next, + ScrClosure *read, ScrClosure *ret, ScrClosure *thr); +ScrStream *scr_stream_duplex_from_dyn(struct ScrDyn *body); /* decodeStrings (false keeps string writes as strings), the writable * defaultEncoding option (a known encoding, borrowed), and * setDefaultEncoding (validated; answers the stream +1). The chaining diff --git a/packages/runtime/src/scr_stream_ops.c b/packages/runtime/src/scr_stream_ops.c index 8fd220168..1db90bc5a 100644 --- a/packages/runtime/src/scr_stream_ops.c +++ b/packages/runtime/src/scr_stream_ops.c @@ -2029,6 +2029,238 @@ ScrStream *scr_stream_from_async_iter(ScrDyn *iter, double flags, double hwm) { return scr_stream_from_dyn_iter(scr_dyn_retain(iter), true, flags, hwm); } +/* ── Duplex.from (duplexify.js) ───────────────────────────────────── + * Duplex.from(asyncGeneratorFunction): fromAsyncGen hands the function a + * source generator that the writes and the final feed through a deferred + * promise, and from(Duplexify, value, { objectMode: true, write, final, + * destroy }) reads what the function yields. The source generator is + * compiled (the frontend's Duplex.from source helper): + * + * while (true) { const v = await next(); ack(); if (v is done) return; yield v; } + * + * next() answers the current deferred, a write or the final settles it + * (the chunk, or the done marker), and ack() is fromAsyncGen's + * `process.nextTick(cb)` followed by a fresh deferred. */ + +enum { CHAN_NONE, CHAN_WRITE, CHAN_FINAL, CHAN_ABORT }; + +typedef struct OpChan OpChan; +struct OpChan { + OpObj base; + ScrStream *d; /* +1, dropped at destroy (or exit) */ + ScrPromise *promise; /* +1: the deferred next() answers, NULL once taken */ + ScrPromise *resolve; /* +1: the deferred a write or the final settles, NULL once settled */ + int settled; /* CHAN_*: what settled the deferred, for ack() */ + bool aborted; /* destroy ran (fromAsyncGen's ac.abort()) */ + OpChan *live_prev, *live_next; +}; + +static SCR_TL OpChan *op_chan_live = NULL; + +static void op_chan_drop(OpObj *o) { + OpChan *c = (OpChan *)o; + if (c->d) scr_stream_release(c->d); + if (c->promise) scr_promise_release(c->promise); + if (c->resolve) scr_promise_release(c->resolve); + free(c); +} + +/* createDeferredPromise() */ +static void op_chan_defer(OpChan *c) { + ScrPromise *p = scr_promise_new(); + if (c->promise) scr_promise_release(c->promise); + if (c->resolve) scr_promise_release(c->resolve); + c->promise = p; + c->resolve = scr_promise_retain(p); +} + +/* Unlinks the channel from the live list and drops its stream and its + * deferred (a source still awaiting it stays pending, as in Node). */ +static void op_chan_retire(OpChan *c) { + if (c->live_prev) c->live_prev->live_next = c->live_next; + else if (op_chan_live == c) op_chan_live = c->live_next; + else return; /* already retired */ + if (c->live_next) c->live_next->live_prev = c->live_prev; + c->live_prev = c->live_next = NULL; + ScrStream *d = c->d; + c->d = NULL; + if (c->promise) scr_promise_release(c->promise); + if (c->resolve) scr_promise_release(c->resolve); + c->promise = c->resolve = NULL; + if (d) scr_stream_release(d); + op_unref(c); /* the live list's reference */ +} + +static void op_chan_atexit(void) { + while (op_chan_live) op_chan_retire(op_chan_live); +} + +static ScrClosure *op_chan_closure(void *fn, OpChan *c) { + ScrClosure *cl = scr_closure_new(fn, 1); + cl->caps[0] = scr_box_new_obj(&op_ref, &op_unref, NULL); + scr_box_set_ref(cl->caps[0], op_ref(c)); + return cl; +} + +static OpChan *op_chan_of(ScrClosure *cb) { return scr_box_get_ref(cb->caps[0]); /* +1 */ } + +/* process.nextTick(cb): the write's or the final's completion. */ +static void op_chan_done_tick(ScrClosure *job) { + ScrStream *d = scr_box_get_ref(job->caps[0]); + bool final = scr_box_get_bool(job->caps[1]); + if (final) scr_stream_final_done(d, NULL); + else scr_stream_write_done(d, NULL); + scr_stream_release(d); +} + +/* The source's `await _promise`: the current deferred (+1). */ +static ScrPromise *op_chan_next_inv(ScrClosure *cb) { + OpChan *c = op_chan_of(cb); + ScrPromise *p = c->promise; + c->promise = NULL; + op_unref(c); + return p ? p : scr_promise_new(); +} + +static ScrError *op_chan_abort_error(void) { + ScrStr *message = scr_str_new("The operation was aborted", 25); + ScrError *e = scr_error_new(SCR_ERR_ERROR, message); + scr_str_release(message); + scr_str_release(e->name); + e->name = scr_str_new("AbortError", 10); + scr_error_set_code(e, "ABORT_ERR"); + e->name_present = true; + e->name_enumerable = true; + return e; +} + +/* After the source's await: process.nextTick(cb), then (unless done) the + * aborted check and a fresh deferred. */ +static void op_chan_ack_inv(ScrClosure *cb) { + OpChan *c = op_chan_of(cb); + int settled = c->settled; + c->settled = CHAN_NONE; + if (c->d && (settled == CHAN_WRITE || settled == CHAN_FINAL)) { + ScrClosure *tick = scr_closure_new((void *)&op_chan_done_tick, 2); + tick->caps[0] = scr_box_new_obj(&scr_stream_retain_v, &scr_stream_release_v, &scr_stream_trace); + scr_box_set_ref(tick->caps[0], scr_stream_retain(c->d)); + tick->caps[1] = scr_box_new(SCR_BOX_BOOL); + scr_box_set_bool(tick->caps[1], settled == CHAN_FINAL); + scr_next_tick(tick); + } + if (settled != CHAN_FINAL && settled != CHAN_ABORT) { + if (c->aborted) { + ScrError *e = op_chan_abort_error(); + scr_throw_obj(e, &scr_error_retain_v, &scr_error_release_v, scr_error_trace_arg()); + } else if (c->d) { + op_chan_defer(c); + } + } + op_unref(c); +} + +/* `_resolve(value)`: the write's chunk or the final's done marker. */ +static void op_chan_settle(OpChan *c, ScrDyn *v /*moves*/, int kind) { + ScrPromise *p = c->resolve; + c->resolve = NULL; + if (!p) { + scr_dyn_release(v); + return; + } + c->settled = kind; + scr_promise_fulfill_ref(p, v, scr_dyn_retain_v, scr_dyn_release_v, NULL); + scr_promise_release(p); +} + +/* Duplexify's write(chunk, encoding, cb) */ +static void op_chan_write_inv(ScrClosure *cb, ScrStream *s, ScrBytes *chunk) { + OpChan *c = op_chan_of(cb); + op_chan_settle(c, scr_stream_chunk_value(s, chunk), CHAN_WRITE); + op_unref(c); +} + +/* Duplexify's final(cb) */ +static void op_chan_final_inv(ScrClosure *cb, ScrStream *s) { + (void)s; + OpChan *c = op_chan_of(cb); + op_chan_settle(c, scr_stream_from_done(), CHAN_FINAL); + op_unref(c); +} + +/* destroy: fromAsyncGen's (ac.abort(); a source waiting for a write or + * the final is unblocked with the done marker and a no-op callback; + * cb(err)), then from()'s close of the function's iterator. caps: the + * channel and the from() destroy closure. */ +static void op_dup_destroy_inv(ScrClosure *cb, ScrStream *s, ScrError *err) { + OpChan *c = scr_box_get_ref(cb->caps[0]); + ScrClosure *from = scr_box_get_ref(cb->caps[1]); + c->aborted = true; + if (c->resolve) op_chan_settle(c, scr_stream_from_done(), CHAN_ABORT); + op_chan_retire(c); + op_unref(c); + op_from_destroy_inv(from, s, err); + scr_closure_release(from); +} + +typedef ScrGen *(*OpSourceFn)(ScrClosure *, ScrClosure *, ScrClosure *); +typedef ScrGen *(*OpStageFn)(ScrClosure *, ScrGen *); + +/* Duplex.from(fn) for an async generator function: the compiled stage + * (borrowed) takes the source generator and answers its own; source + * (borrowed) makes the source generator from next() and ack(); the + * compiler's helpers over the stage's generator are borrowed. +1, or + * NULL with the function's throw pending. */ +ScrStream *scr_stream_duplex_from_gen(ScrClosure *stage, ScrClosure *source, ScrClosure *next, + ScrClosure *read, ScrClosure *ret, ScrClosure *thr) { + op_sentinels_init(); + static SCR_TL bool registered = false; + if (!registered) { + registered = true; + scr_atexit(op_chan_atexit); + } + OpChan *c = op_alloc(sizeof *c); + c->base.rc = 1; + c->base.drop = &op_chan_drop; + op_chan_defer(c); + ScrGen *src = ((OpSourceFn)source->fn)(source, op_chan_closure((void *)&op_chan_next_inv, c), + op_chan_closure((void *)&op_chan_ack_inv, c)); + ScrGen *g = scr_exc_pending() ? NULL : ((OpStageFn)stage->fn)(stage, src); /* src moves in */ + if (scr_exc_pending()) { + if (g) scr_gen_release(g); + op_unref(c); + return NULL; + } + OpFrom *f = op_alloc(sizeof *f); + f->base.rc = 1; + f->base.drop = &op_from_drop; + f->kind = FROM_ASYNC; + f->gen = g; + f->next = scr_closure_retain(next); + f->read = scr_closure_retain(read); + f->ret = scr_closure_retain(ret); + f->thr = scr_closure_retain(thr); + ScrClosure *destroy = scr_closure_new((void *)&op_dup_destroy_inv, 2); + destroy->caps[0] = scr_box_new_obj(&op_ref, &op_unref, NULL); + scr_box_set_ref(destroy->caps[0], op_ref(c)); + destroy->caps[1] = scr_box_new_obj(&scr_closure_retain_v, &scr_closure_release_v, &scr_closure_trace_v); + scr_box_set_ref(destroy->caps[1], op_from_closure((void *)&op_from_destroy_inv, f)); + /* Duplexify({ objectMode: true, highWaterMark: 1, write, final, destroy }) */ + ScrStream *d = scr_stream_new_duplex(1, 1, true, true, true, true, true, + op_from_closure((void *)&op_from_read_inv, f), &op_from_read_inv, + op_chan_closure((void *)&op_chan_write_inv, c), &op_chan_write_inv, + op_chan_closure((void *)&op_chan_final_inv, c), &op_chan_final_inv, + destroy, &op_dup_destroy_inv); + scr_stream_set_object_mode(d, true, true); + d->cls = "Duplexify"; + f->out = d; + op_unref(f); + c->d = scr_stream_retain(d); + c->live_next = op_chan_live; + if (op_chan_live) op_chan_live->live_prev = c; + op_chan_live = c; /* the reference c was created with */ + return d; +} + /* ── the promise operators ─────────────────────────────────────────── */ enum { PK_DYN, PK_F64, PK_BOOL, PK_STR, PK_VOID }; diff --git a/tests/corpus/7252-duplex-from-generator.ts b/tests/corpus/7252-duplex-from-generator.ts new file mode 100644 index 000000000..488748021 --- /dev/null +++ b/tests/corpus/7252-duplex-from-generator.ts @@ -0,0 +1,69 @@ +// Duplex.from over an async generator function in typed code: the +// generator reads what is written to the Duplex and its values are the +// readable side. Both sides are in object mode with a highWaterMark of +// 1; each write's callback runs once the generator asks for the next +// value, and end() reaches the generator as the end of its source. +import { Duplex } from "node:stream"; + +const log: string[] = []; + +async function* numbered(source: AsyncIterable): AsyncGenerator { + let n = 0; + for await (const chunk of source) { + n++; + log.push(`gen got ${String(chunk)}`); + yield `${n}:${String(chunk)}`; + } + log.push("gen done"); + yield `total ${n}`; +} + +async function collect(d: Duplex): Promise { + const out: string[] = []; + for await (const chunk of d) out.push(String(chunk)); + return out; +} + +async function basic(): Promise { + const d = Duplex.from(numbered); + log.push(`sides ${d.readable} ${d.writable}`); + log.push(`object mode ${d.readableObjectMode} ${d.writableObjectMode}`); + log.push(`hwm ${d.readableHighWaterMark} ${d.writableHighWaterMark}`); + d.write("a", () => log.push("write a done")); + d.write("b", () => log.push("write b done")); + d.end("c", () => log.push("end done")); + const out = await collect(d); + log.push(`read ${out.join(",")}`); + log.push(`after ${d.readableEnded} ${d.writableFinished} ${d.destroyed}`); +} + +async function values(): Promise { + // object-mode writes keep their values + const d = Duplex.from(async function* (source: AsyncIterable): AsyncGenerator { + for await (const v of source) yield typeof v + " " + JSON.stringify(v); + }); + d.write({ k: 1 }); + d.write([2, 3]); + d.write(4); + d.end(); + log.push(`values ${(await collect(d)).join(" | ")}`); +} + +async function earlyDestroy(): Promise { + const d = Duplex.from(numbered); + d.on("error", (e: Error) => log.push(`error ${e.message}`)); + d.on("close", () => log.push(`close ${d.destroyed}`)); + d.write("x"); + await new Promise((resolve) => setTimeout(resolve, 5)); + d.destroy(new Error("stop")); + await new Promise((resolve) => setTimeout(resolve, 5)); +} + +async function main(): Promise { + await basic(); + await values(); + await earlyDestroy(); + for (const line of log) console.log(line); +} + +void main(); diff --git a/tests/corpus/7253-duplex-from.mjs b/tests/corpus/7253-duplex-from.mjs new file mode 100644 index 000000000..14556348a --- /dev/null +++ b/tests/corpus/7253-duplex-from.mjs @@ -0,0 +1,48 @@ +// Duplex.from in JavaScript: an async generator function, an array, a +// generator object and an async generator object. Over an iterable the +// result is read-only (writable false); over a function it reads what is +// written. The log records the order of the write callbacks, the reads +// and the stream events against each other. +import { Duplex } from "node:stream"; + +const log = []; + +async function* echo(source) { + for await (const chunk of source) { + log.push(`echo ${chunk}`); + yield chunk.toUpperCase(); + } +} + +const fromFn = Duplex.from(echo); +fromFn.on("finish", () => log.push("fn finish")); +fromFn.on("end", () => log.push("fn end")); +fromFn.on("close", () => log.push("fn close")); +fromFn.write("one", () => log.push("fn write one cb")); +fromFn.write("two", () => log.push("fn write two cb")); +fromFn.end(() => log.push("fn end cb")); +for await (const chunk of fromFn) log.push(`fn read ${chunk}`); + +const fromArray = Duplex.from(["p", "q"]); +log.push(`array ${fromArray.readable} ${fromArray.writable} ${fromArray.readableObjectMode}`); +for await (const chunk of fromArray) log.push(`array ${chunk}`); + +function* gen() { + yield "g1"; + yield "g2"; +} +const fromGen = Duplex.from(gen()); +log.push(`gen ${fromGen.writable}`); +for await (const chunk of fromGen) log.push(`gen ${chunk}`); + +async function* agen() { + yield 1; + await null; + yield 2; +} +const fromAgen = Duplex.from(agen()); +fromAgen.on("data", (v) => log.push(`agen data ${v}`)); +await new Promise((resolve) => fromAgen.on("end", resolve)); +log.push(`agen ended ${fromAgen.readableEnded}`); + +console.log(log.join("\n")); diff --git a/tests/corpus/7254-compose-functions.ts b/tests/corpus/7254-compose-functions.ts new file mode 100644 index 000000000..4a2ba7189 --- /dev/null +++ b/tests/corpus/7254-compose-functions.ts @@ -0,0 +1,56 @@ +// stream.compose with async generator function stages in typed code: a +// function at either end becomes Duplex.from of it, functions between +// streams run as pipeline stages, and one stage is Duplex.from of it. +// The composed Duplex writes to the first stage and reads from the last; +// its object modes follow the first stage's writable side and the last +// stage's readable side. +import { Duplex, PassThrough, Transform, compose } from "node:stream"; + +const log: string[] = []; + +async function* upper(source: AsyncIterable): AsyncGenerator { + for await (const chunk of source) yield String(chunk).toUpperCase(); +} + +async function* numbered(source: AsyncIterable): AsyncGenerator { + let n = 0; + for await (const chunk of source) yield `${++n}.${String(chunk)}`; +} + +async function drain(name: string, d: Duplex, chunks: string[]): Promise { + log.push(`${name} r=${d.readable} w=${d.writable} ro=${d.readableObjectMode} wo=${d.writableObjectMode}`); + const out: string[] = []; + d.on("data", (c: unknown) => out.push(String(c))); + const ended = new Promise((resolve) => d.on("end", () => resolve())); + for (const c of chunks) d.write(c); + d.end(); + await ended; + log.push(`${name} ${out.join(" ")}`); +} + +async function main(): Promise { + // functions only: Duplex.from at both ends + await drain("fns", compose(upper, numbered), ["a", "b"]); + // a function between two object-mode streams + await drain( + "middle", + compose(new PassThrough({ objectMode: true }), upper, new PassThrough({ objectMode: true })), + ["c", "d"], + ); + // a byte stream before a function: the Duplex writes bytes + const bytes = compose(new PassThrough(), numbered); + await drain("bytes", bytes, ["e"]); + // a function before a Transform + const tail = new Transform({ + objectMode: true, + transform(chunk: unknown, _enc: string, cb: (err?: Error | null, data?: unknown) => void): void { + cb(null, `<${String(chunk)}>`); + }, + }); + await drain("transform", compose(upper, tail), ["f", "g"]); + // one stage + await drain("single", compose(numbered), ["h", "i", "j"]); + for (const line of log) console.log(line); +} + +void main(); diff --git a/tests/corpus/7255-compose-stages.mjs b/tests/corpus/7255-compose-stages.mjs new file mode 100644 index 000000000..ef6542ece --- /dev/null +++ b/tests/corpus/7255-compose-stages.mjs @@ -0,0 +1,55 @@ +// stream.compose in JavaScript with an iterable or generator first stage +// (the Duplex is not writable), a stage that throws (the Duplex errors), +// a destroyed composition (the stages see an AbortError), and compose() +// with no stages (ERR_MISSING_ARGS). +import { PassThrough, compose } from "node:stream"; + +const log = []; + +const double = async function* (source) { + for await (const n of source) yield n * 2; +}; + +const fromArray = compose([1, 2, 3], double); +log.push(`array r=${fromArray.readable} w=${fromArray.writable}`); +for await (const n of fromArray) log.push(`array ${n}`); + +function* counter() { + yield 10; + yield 20; +} +const fromGen = compose(counter(), double, async function* (source) { + for await (const n of source) yield `n=${n}`; +}); +for await (const s of fromGen) log.push(`gen ${s}`); + +const failing = compose(new PassThrough({ objectMode: true }), async function* (source) { + for await (const chunk of source) { + if (chunk === "stop") throw new Error("stage failed on " + chunk); + yield chunk; + } +}, new PassThrough({ objectMode: true })); +failing.on("data", (c) => log.push(`failing data ${c}`)); +failing.on("error", (e) => log.push(`failing error ${e.message}`)); +failing.on("close", () => log.push(`failing close ${failing.destroyed}`)); +failing.write("go"); +failing.write("stop"); +await new Promise((resolve) => failing.on("close", resolve)); + +const stopped = compose(new PassThrough({ objectMode: true }), double); +stopped.on("error", (e) => log.push(`stopped error ${e.name} ${e.code}`)); +stopped.on("close", () => log.push("stopped close")); +stopped.write(4); +stopped.once("data", (n) => { + log.push(`stopped data ${n}`); + stopped.destroy(); +}); +await new Promise((resolve) => stopped.on("close", resolve)); + +try { + compose(); +} catch (e) { + log.push(`empty ${e.code} ${e.message}`); +} + +console.log(log.join("\n")); diff --git a/tests/corpus/7256-readable-compose.mjs b/tests/corpus/7256-readable-compose.mjs new file mode 100644 index 000000000..a9622173e --- /dev/null +++ b/tests/corpus/7256-readable-compose.mjs @@ -0,0 +1,53 @@ +// readable.compose(stage, { signal }): compose(this, stage), with the +// signal aborting the composed Duplex. The stage may be an async +// generator function or a stream; the result reads like the stage's +// output and is not writable when the receiver heads it. +import { PassThrough, Readable, Transform } from "node:stream"; + +const log = []; + +const WORDS = ["alpha", "beta", "gamma"]; +const lengths = async function* (source) { + for await (const w of source) yield `${w}:${w.length}`; +}; + +const source = Readable.from(WORDS); +const viaFn = source.compose(lengths); +log.push(`fn r=${viaFn.readable} w=${viaFn.writable} ro=${viaFn.readableObjectMode}`); +for await (const s of viaFn) log.push(`fn ${s}`); + +const shout = new Transform({ + objectMode: true, + transform(chunk, _enc, cb) { + cb(null, chunk.toUpperCase()); + }, +}); +const source2 = Readable.from(WORDS); +const viaStream = source2.compose(shout); +log.push(`stream ${(await viaStream.toArray()).join(",")}`); + +const pass = new PassThrough({ objectMode: true }); +const chained = pass.compose(lengths).compose(async function* (source) { + for await (const s of source) yield `[${s}]`; +}); +chained.on("data", (s) => log.push(`chained ${s}`)); +pass.write("x"); +pass.end("yz"); +await new Promise((resolve) => chained.on("end", resolve)); + +const ac = new AbortController(); +const signal = ac.signal; +const slow = new Readable({ objectMode: true, read() {} }); +const aborted = slow.compose(lengths, { signal }); +aborted.on("data", (s) => log.push(`aborted data ${s}`)); +aborted.on("error", (e) => log.push(`aborted error ${e.name} ${e.code}`)); +aborted.on("close", () => log.push(`aborted close ${aborted.destroyed}`)); +slow.push("one"); +setTimeout(() => ac.abort(), 10); +await new Promise((resolve) => aborted.on("close", resolve)); + +const source3 = Readable.from(WORDS); +const unsignaled = source3.compose(lengths, undefined); +log.push(`undefined options ${(await unsignaled.toArray()).length}`); + +console.log(log.join("\n")); diff --git a/tests/corpus/7257-pipeline-generator-destination.mjs b/tests/corpus/7257-pipeline-generator-destination.mjs new file mode 100644 index 000000000..0996e06eb --- /dev/null +++ b/tests/corpus/7257-pipeline-generator-destination.mjs @@ -0,0 +1,61 @@ +// pipeline() whose destination is a function: an async generator +// destination is pumped into the object-mode PassThrough pipeline() +// returns, and an async function destination's resolved value is written +// to it (null and undefined are not). The callback runs with the error +// and the destination's value; a generator yielding null fails the +// pipeline with ERR_STREAM_NULL_VALUES. +import { Readable, pipeline } from "node:stream"; + +const log = []; + +const out = pipeline( + Readable.from(["a", "b", "c"]), + async function* (source) { + for await (const chunk of source) yield { chunk, at: log.length }; + }, + (err, value) => log.push(`gen cb ${err} ${value}`), +); +log.push(`gen out r=${out.readable} w=${out.writable} ro=${out.readableObjectMode} wo=${out.writableObjectMode}`); +for await (const item of out) log.push(`gen read ${JSON.stringify(item)}`); + +const summary = pipeline( + Readable.from([3, 4]), + async function* (source) { + for await (const n of source) yield n * n; + }, + async function (source) { + let total = 0; + for await (const n of source) total += n; + return { total }; + }, + (err, value) => log.push(`fn cb ${err} ${JSON.stringify(value)}`), +); +summary.on("data", (v) => log.push(`fn data ${JSON.stringify(v)}`)); +summary.on("end", () => log.push("fn end")); +await new Promise((resolve) => summary.on("close", resolve)); + +const nothing = pipeline( + Readable.from(["x"]), + async function (source) { + for await (const _ of source); + return undefined; + }, + (err) => log.push(`undefined cb ${err}`), +); +nothing.on("data", () => log.push("undefined data")); +nothing.on("end", () => log.push("undefined end")); +await new Promise((resolve) => nothing.on("close", resolve)); + +const bad = pipeline( + Readable.from(["g"]), + async function* (source) { + for await (const chunk of source) yield chunk; + yield null; + }, + (err) => log.push(`null cb ${err && err.code}`), +); +bad.on("data", (c) => log.push(`null data ${c}`)); +bad.on("error", (e) => log.push(`null error ${e.code}`)); +await new Promise((resolve) => bad.on("close", resolve)); + +console.log(log.join("\n")); diff --git a/tests/corpus/7258-pipeline-generator-destination.ts b/tests/corpus/7258-pipeline-generator-destination.ts new file mode 100644 index 000000000..b22b68dce --- /dev/null +++ b/tests/corpus/7258-pipeline-generator-destination.ts @@ -0,0 +1,47 @@ +// pipeline() with a function destination in typed code. @types/node and +// the fallback declarations type the result as the destination stream; +// for a function destination it is the object-mode PassThrough that +// pipeline() returns, read here as a PassThrough. +import { PassThrough, Readable, pipeline } from "node:stream"; + +const log: string[] = []; + +async function* tokens(source: AsyncIterable): AsyncGenerator { + for await (const line of source) { + for (const word of String(line).split(" ")) yield word; + } +} + +async function* tagged(source: AsyncIterable): AsyncGenerator<{ word: string; n: number }> { + let n = 0; + for await (const word of source) yield { word, n: ++n }; +} + +async function main(): Promise { + const out = pipeline(Readable.from(["to be", "or not"]), tokens, tagged, (err: Error | null) => { + log.push(`callback ${err === null || err === undefined ? "ok" : err.message}`); + }) as PassThrough; + log.push(`object mode ${out.readableObjectMode} ${out.writableObjectMode}`); + for await (const item of out) { + const entry = item as { word: string; n: number }; + log.push(`${entry.n} ${entry.word}`); + } + log.push(`ended ${out.readableEnded}`); + + const counted = pipeline( + Readable.from(["x", "y", "z"]), + async function (source: AsyncIterable): Promise { + let n = 0; + for await (const _ of source) n++; + return n; + }, + (err: Error | null) => log.push(`count callback ${err === null || err === undefined}`), + ) as PassThrough; + const values: unknown[] = []; + counted.on("data", (v: unknown) => values.push(v)); + await new Promise((resolve) => counted.on("end", () => resolve())); + log.push(`count values ${JSON.stringify(values)}`); + for (const line of log) console.log(line); +} + +void main(); diff --git a/tests/corpus/7259-promises-pipeline-stage-options.cjs b/tests/corpus/7259-promises-pipeline-stage-options.cjs new file mode 100644 index 000000000..a4a3ce396 --- /dev/null +++ b/tests/corpus/7259-promises-pipeline-stage-options.cjs @@ -0,0 +1,65 @@ +// stream/promises pipeline() with function stages or an iterable source +// and the options argument: end: false leaves the destination open, a +// signal aborts the pipeline (the stages see their own signal abort), an +// async function destination's value resolves the promise, and a value +// that is not an AbortSignal rejects with ERR_INVALID_ARG_TYPE. +const { PassThrough, Writable } = require("node:stream"); +const { pipeline } = require("node:stream/promises"); + +const log = []; + +const upper = async function* (source) { + for await (const chunk of source) yield String(chunk).toUpperCase(); +}; + +function sink(name) { + return new Writable({ + objectMode: true, + write(chunk, _enc, cb) { + log.push(`${name} got ${chunk}`); + cb(); + }, + }); +} + +async function main() { + const joined = await pipeline(["a", "b"], upper, async function (source) { + let s = ""; + for await (const c of source) s += c; + return s; + }, { end: true }); + log.push(`joined ${joined}`); + + const open = new PassThrough({ objectMode: true }); + await pipeline(["k"], upper, open, { end: false }); + log.push(`end false: ended ${open.writableEnded}`); + open.end("last"); + const rest = []; + for await (const c of open) rest.push(c); + log.push(`end false: read ${rest.join(",")}`); + + const ac = new AbortController(); + const slow = async function* (source, { signal }) { + signal.addEventListener("abort", () => log.push(`stage signal aborted ${signal.aborted}`)); + for await (const chunk of source) { + yield chunk; + await new Promise((resolve) => setTimeout(resolve, 30)); + } + }; + const aborting = pipeline(["s1", "s2", "s3"], slow, sink("abort"), { signal: ac.signal }); + setTimeout(() => ac.abort(), 10); + try { + await aborting; + } catch (e) { + log.push(`aborted ${e.name} ${e.code}`); + } + + try { + await pipeline(["v"], upper, sink("bad"), { signal: "nope" }); + } catch (e) { + log.push(`bad signal ${e.code}`); + } + console.log(log.join("\n")); +} + +main(); diff --git a/tests/harness/corpus-costs/7252-duplex-from-generator.ts.json b/tests/harness/corpus-costs/7252-duplex-from-generator.ts.json new file mode 100644 index 000000000..47f7b9ed9 --- /dev/null +++ b/tests/harness/corpus-costs/7252-duplex-from-generator.ts.json @@ -0,0 +1 @@ +{ "plain": 2522, "san": 3330 } diff --git a/tests/harness/corpus-costs/7253-duplex-from.mjs.json b/tests/harness/corpus-costs/7253-duplex-from.mjs.json new file mode 100644 index 000000000..7743ca7c8 --- /dev/null +++ b/tests/harness/corpus-costs/7253-duplex-from.mjs.json @@ -0,0 +1 @@ +{ "plain": 1849, "san": 2440 } diff --git a/tests/harness/corpus-costs/7254-compose-functions.ts.json b/tests/harness/corpus-costs/7254-compose-functions.ts.json new file mode 100644 index 000000000..19d896024 --- /dev/null +++ b/tests/harness/corpus-costs/7254-compose-functions.ts.json @@ -0,0 +1 @@ +{ "plain": 1782, "san": 2350 } diff --git a/tests/harness/corpus-costs/7255-compose-stages.mjs.json b/tests/harness/corpus-costs/7255-compose-stages.mjs.json new file mode 100644 index 000000000..e8e0ab3d1 --- /dev/null +++ b/tests/harness/corpus-costs/7255-compose-stages.mjs.json @@ -0,0 +1 @@ +{ "plain": 2033, "san": 2680 } diff --git a/tests/harness/corpus-costs/7256-readable-compose.mjs.json b/tests/harness/corpus-costs/7256-readable-compose.mjs.json new file mode 100644 index 000000000..fe1307a1f --- /dev/null +++ b/tests/harness/corpus-costs/7256-readable-compose.mjs.json @@ -0,0 +1 @@ +{ "plain": 2146, "san": 2830 } diff --git a/tests/harness/corpus-costs/7257-pipeline-generator-destination.mjs.json b/tests/harness/corpus-costs/7257-pipeline-generator-destination.mjs.json new file mode 100644 index 000000000..ebaa9214c --- /dev/null +++ b/tests/harness/corpus-costs/7257-pipeline-generator-destination.mjs.json @@ -0,0 +1 @@ +{ "plain": 2008, "san": 2731 } diff --git a/tests/harness/corpus-costs/7258-pipeline-generator-destination.ts.json b/tests/harness/corpus-costs/7258-pipeline-generator-destination.ts.json new file mode 100644 index 000000000..15ff27f21 --- /dev/null +++ b/tests/harness/corpus-costs/7258-pipeline-generator-destination.ts.json @@ -0,0 +1 @@ +{ "plain": 1751, "san": 2442 } diff --git a/tests/harness/corpus-costs/7259-promises-pipeline-stage-options.cjs.json b/tests/harness/corpus-costs/7259-promises-pipeline-stage-options.cjs.json new file mode 100644 index 000000000..6bda6d818 --- /dev/null +++ b/tests/harness/corpus-costs/7259-promises-pipeline-stage-options.cjs.json @@ -0,0 +1 @@ +{ "plain": 2504, "san": 3707 } diff --git a/tests/harness/self-hosting-native-toolchain.test.ts b/tests/harness/self-hosting-native-toolchain.test.ts index c0176eb93..c4013d288 100644 --- a/tests/harness/self-hosting-native-toolchain.test.ts +++ b/tests/harness/self-hosting-native-toolchain.test.ts @@ -108,7 +108,7 @@ for (const backend of ["llvm"] as const) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 27, marker: "scr_runtime_abi_v27" }, + runtime_abi: { version: 28, marker: "scr_runtime_abi_v28" }, compiler: { command: "fixture", identity: "fixture", target: target.llvmTriple }, macros: { executable: [], excluded: [], sanitizer: "external-toolchain-required" }, flavors: { From 5ceec5659c799a280e2f47a4a52920a86c903007 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 11:06:14 -0500 Subject: [PATCH 050/197] Keep key orders exact through write values, live views and spreads Three cases where a value's key order could differ from Node: - A write of an optional key moved the key before its value was evaluated. A value that adds another key to the same record, as in `o.b = ((o.c = 1), 2)`, then put the keys in the wrong order. The write now evaluates its value first, then tests and moves the key. - A value seen as a JavaScript value bound the field hooks of its order when it was boxed, and a later write that moved a key did not change them. Shapes whose values can change order after they are built (IrRecordShape.orderMoves) now read through the value's view, which follows the current order. - A literal that spreads a value with orders and then sets a key the value may lack, as in `{ ...value, tag: "x" }`, put that key where the source's order has it. Node puts it there only when the value has it, and after the value's keys otherwise. The literal now picks its order from the source's order and which of those keys the source has. Corpus 7136 and 7138 cover each case. --- .../src/backend/llvm/expr-stream-bridges.ts | 10 +- .../lowering/key-order-witness.test.ts | 108 ++++++++++++++ .../frontend/lowering/key-order-witness.ts | 138 +++++++++++++----- packages/compiler/src/ir/ir.ts | 4 + tests/corpus/7136-key-order-copies.ts | 13 ++ tests/corpus/7138-key-order-late-writes.ts | 14 +- .../7136-key-order-copies.ts.json | 2 +- .../7138-key-order-late-writes.ts.json | 2 +- 8 files changed, 251 insertions(+), 40 deletions(-) diff --git a/packages/compiler/src/backend/llvm/expr-stream-bridges.ts b/packages/compiler/src/backend/llvm/expr-stream-bridges.ts index b64204122..398384e75 100644 --- a/packages/compiler/src/backend/llvm/expr-stream-bridges.ts +++ b/packages/compiler/src/backend/llvm/expr-stream-bridges.ts @@ -1071,9 +1071,15 @@ export function streamTypedRefMaterializeAdapter( }; // Published before the field converters, like the array hooks below. // A Dirent row reads through its finished view (the prototype and - // the non-enumerable %dtype). + // the non-enumerable %dtype). A capsule binds the ops of its value's + // order when it is built, so a shape whose values can change order + // later (orderMoves) reads through its view. const dirent = isDirentShape(shape); - if (!dirent && orders.every((order) => liveRecordOpsEligible(fieldsOf(order)))) { + if ( + !dirent && + shape.orderMoves === undefined && + orders.every((order) => liveRecordOpsEligible(fieldsOf(order))) + ) { if (orders.length === 1) { adapter.recordOps = `@${snapshot}_rops`; liveRecordOps(host, t, snapshot, fieldsOf(orders[0]!)); diff --git a/packages/compiler/src/frontend/lowering/key-order-witness.test.ts b/packages/compiler/src/frontend/lowering/key-order-witness.test.ts index 56b65a3d3..3b64d65ce 100644 --- a/packages/compiler/src/frontend/lowering/key-order-witness.test.ts +++ b/packages/compiler/src/frontend/lowering/key-order-witness.test.ts @@ -468,3 +468,111 @@ test("re-adding a key a literal creates earlier moves it to the end of that valu ["b", "a"], ]); }); + +test("a moving write evaluates its value before it tests and moves the key", () => { + const f = fixture(); + const o = f.record( + [ + ["id", F64], + ["x", OPT], + ["y", OPT], + ], + ["id", "x", "y"], + ); + // `o.x = ((o.y = 2), 3)`: y is created first. + const value: IrExpr = { + kind: "seqExpr", + stmts: [set(ref("o", o), "y")], + result: num(3), + type: F64, + loc, + }; + const write: IrStmt = { + kind: "recordSet", + obj: ref("o", o), + shapeId: o.shapeId, + field: "x", + value, + loc, + }; + const fn = f.run([ + decl("o", f.literal(o, ["id"])), + observe(f.literal(o, ["x", "id"])), + write, + observe(ref("o", o)), + ]); + const shape = f.shapes.get(o.shapeId)!; + expect(shape.orderVariants).toBeDefined(); + expect(shape.orderMoves).toBe(true); + const block = fn.body[2] as IrStmt & { kind: "block" }; + expect(block.kind).toBe("block"); + const [first, guard, written] = block.body; + expect(first).toMatchObject({ kind: "varDecl", init: { kind: "seqExpr" } }); + const local = (first as IrStmt & { kind: "varDecl" }).localId; + expect(guard).toMatchObject({ kind: "if", cond: { kind: "unary", op: "!" } }); + expect(written).toMatchObject({ + kind: "recordSet", + field: "x", + value: { kind: "varRef", localId: local }, + }); + // The write inside the value moves y in its own block. + const init = (first as IrStmt & { kind: "varDecl" }).init as IrExpr & { kind: "seqExpr" }; + expect(init.stmts[0]).toMatchObject({ kind: "block" }); +}); + +test("a key after a spread that the source may lack picks its order from the source's keys", () => { + const f = fixture(); + const s = f.record( + [ + ["a", F64], + ["b", OPT], + ["c", OPT], + ], + ["a", "b", "c"], + ); + const t = f.record( + [ + ["a", F64], + ["b", F64], + ["c", OPT], + ], + ["a", "b", "c"], + ); + const read = (field: string): IrExpr => ({ + kind: "recordGet", + obj: ref("x", s), + shapeId: s.shapeId, + field, + type: OPT, + loc, + }); + // `{ ...x, b: 1 }` + const merged = f.notes.literal( + { + kind: "recordLit", + fields: [ + { name: "a", value: read("a") }, + { name: "c", value: read("c") }, + { name: "b", value: num(1) }, + ], + type: t, + loc, + }, + [ + { kind: "spread", shapeId: s.shapeId, maybeAbsent: false }, + { kind: "key", name: "b", definite: true }, + ], + ); + f.run([decl("x", f.literal(s, ["c", "a"])), observe(f.literal(s, ["a", "c"])), observe(merged)]); + // b keeps its place in the source's order when the source has it, and + // follows the source's keys when it does not. + expect(f.shapes.get(t.shapeId)!.orderVariants).toEqual([ + ["a", "c", "b"], + ["c", "a", "b"], + ["a", "b", "c"], + ]); + expect(orderIndexOf(merged)).toMatchObject({ + kind: "ternary", + cond: { kind: "recordHas", obj: { kind: "varRef", localId: "x" }, field: "b" }, + }); +}); diff --git a/packages/compiler/src/frontend/lowering/key-order-witness.ts b/packages/compiler/src/frontend/lowering/key-order-witness.ts index c35860d43..4eef5dc54 100644 --- a/packages/compiler/src/frontend/lowering/key-order-witness.ts +++ b/packages/compiler/src/frontend/lowering/key-order-witness.ts @@ -768,7 +768,8 @@ export function applyKeyOrderWitnesses(lowerer: Lowerer, fns: readonly IrFunctio // table, as before. // A write of an optional key a value lacks moves the key to the end of // the value's order, as in Node: the orders include every order such - // writes reach, and each write site updates the index first. + // writes reach, and each write site updates the index after it evaluates + // the value and before it writes the key. const MAX_ORDERS = 16; const unresolved = (id: string): boolean => result.get(id) === "unknown"; /** `order` over every key of `table`: keys the order lacks follow it. */ @@ -809,6 +810,30 @@ export function applyKeyOrderWitnesses(lowerer: Lowerer, fns: readonly IrFunctio } return [...moving]; }; + /** `values[i]` for the order index `current` holds, as a chain of + * tests. */ + const indexChain = (current: IrExpr, values: readonly number[], loc: SrcLoc): IrExpr => { + let out: IrExpr = { kind: "numLit", value: values[values.length - 1]!, type: F64, loc }; + for (let i = values.length - 2; i >= 0; i--) { + if (out.kind === "numLit" && out.value === values[i]) continue; + out = { + kind: "ternary", + cond: { + kind: "bin", + op: "===", + left: current, + right: { kind: "numLit", value: i, type: F64, loc }, + type: BOOL, + loc, + }, + then: { kind: "numLit", value: values[i]!, type: F64, loc }, + else_: out, + type: F64, + loc, + }; + } + return out; + }; const moves = new Map>(); /** Adds to `orders` every order writes of `moving` keys reach from them * and records each key's transitions; false past MAX_ORDERS. */ @@ -887,8 +912,17 @@ export function applyKeyOrderWitnesses(lowerer: Lowerer, fns: readonly IrFunctio // way: copies (rest bindings, width copies), or literals spreading the // value between the same explicit keys. Order i is the source's order i // over the shape's keys, after the keys before the spread and before - // the keys after it. - type Template = { source: string; prefix: string[]; suffix: string[] }; + // the keys after it. A key after the spread that the source value may + // lack goes where the source has it, or after the source's keys when it + // has not: such a literal's orders are one per source order and set of + // those keys the source has, and `select.index[present][i]` is the order + // for source order i with the keys of bit set `present`. + type Template = { + source: string; + prefix: string[]; + suffix: string[]; + select?: { keys: string[]; index: number[][] }; + }; const templateOf = (segments: readonly KeyOrderSegment[], id: string): Template | null => { const prefix: string[] = []; const suffix: string[] = []; @@ -924,11 +958,39 @@ export function applyKeyOrderWitnesses(lowerer: Lowerer, fns: readonly IrFunctio const orders = variants.get(first.source); if (orders === undefined || mirrors.has(first.source)) continue; const table = tableOf(candidate.shape); - const mirrored = orders.map((o) => { + const tested = first.suffix.filter((k) => maybeAbsent(first.source, k)); + if (tested.length > 4) continue; + const mirrorOf = (o: readonly string[], lacks: ReadonlySet): string[] => { const order = [...first.prefix]; - for (const k of [...o, ...first.suffix]) if (!order.includes(k)) order.push(k); + for (const k of o) if (!lacks.has(k) && !order.includes(k)) order.push(k); + for (const k of first.suffix) if (!order.includes(k)) order.push(k); return complete(order, table); - }); + }; + let mirrored: string[][]; + let template: Template = first; + if (tested.length === 0) mirrored = orders.map((o) => mirrorOf(o, new Set())); + else { + mirrored = []; + const index: number[][] = []; + for (let present = 0; present < 1 << tested.length; present++) { + const lacks = new Set(tested.filter((_, j) => (present & (1 << j)) === 0)); + const row: number[] = []; + for (const o of orders) { + const order = mirrorOf(o, lacks); + let i = indexOfOrder(mirrored, order); + if (i < 0) i = mirrored.push(order) - 1; + row.push(i); + } + index.push(row); + } + if (mirrored.length > MAX_ORDERS) continue; + template = { + source: first.source, + prefix: first.prefix, + suffix: first.suffix, + select: { keys: tested, index }, + }; + } if (!closeUnderMoves(id, mirrored, candidate.moving)) continue; if (mirrored.every((o) => sameOrder(o, mirrored[0]!))) { // Every order of the source agrees over these keys: one table. @@ -940,7 +1002,7 @@ export function applyKeyOrderWitnesses(lowerer: Lowerer, fns: readonly IrFunctio continue; } variants.set(id, mirrored); - mirrors.set(id, first); + mirrors.set(id, template); } if (variants.size === 0) return; @@ -995,7 +1057,7 @@ export function applyKeyOrderWitnesses(lowerer: Lowerer, fns: readonly IrFunctio if (segments !== undefined && mirror !== undefined) { const obj = copySource(lit, mirror.source); if (obj === null) return null; - return { + const sourceIndex: IrExpr = { kind: "recordGet", obj, shapeId: mirror.source, @@ -1003,6 +1065,22 @@ export function applyKeyOrderWitnesses(lowerer: Lowerer, fns: readonly IrFunctio type: F64, loc: lit.loc, }; + const select = mirror.select; + if (select === undefined) return sourceIndex; + // Tests the source for each key in turn, then picks by its index. + const pick = (j: number, present: number): IrExpr => { + if (j === select.keys.length) + return indexChain(sourceIndex, select.index[present]!, lit.loc); + return { + kind: "ternary", + cond: lowerer.recordFieldPresent(obj, mirror.source, select.keys[j]!, lit.loc), + then: pick(j + 1, present | (1 << j)), + else_: pick(j + 1, present), + type: F64, + loc: lit.loc, + }; + }; + return pick(0, 0); } if (segments !== undefined) { const order = literalOrder(segments); @@ -1072,7 +1150,7 @@ export function applyKeyOrderWitnesses(lowerer: Lowerer, fns: readonly IrFunctio typeof index === "number" ? { kind: "numLit", value: index, type: F64, loc: lit.loc } : index; lit.fields.push({ name: KEY_ORDER_FIELD, value }); } - let receivers = 0; + let temporaries = 0; const replacements = new Map>(); for (const { st, fn } of writeSites) { if (dropped.has(st.shapeId)) continue; @@ -1084,7 +1162,7 @@ export function applyKeyOrderWitnesses(lowerer: Lowerer, fns: readonly IrFunctio const prologue: IrStmt[] = []; let obj = st.obj; if (plain.kind !== "varRef") { - const localId = `%keyOrderReceiver.${receivers++}`; + const localId = `%keyOrderReceiver.${temporaries++}`; fn.locals.push({ id: localId, name: "%keyOrderReceiver", type: st.obj.type, mutable: false }); prologue.push({ kind: "varDecl", localId, init: st.obj, loc }); obj = { kind: "varRef", localId, type: st.obj.type, loc }; @@ -1097,29 +1175,20 @@ export function applyKeyOrderWitnesses(lowerer: Lowerer, fns: readonly IrFunctio type: F64, loc, }; - // transitions[i] for the current index i, as a chain of tests. - let next: IrExpr = { - kind: "numLit", - value: transitions[transitions.length - 1]!, - type: F64, - loc, - }; - for (let i = transitions.length - 2; i >= 0; i--) { - next = { - kind: "ternary", - cond: { - kind: "bin", - op: "===", - left: current, - right: { kind: "numLit", value: i, type: F64, loc }, - type: BOOL, - loc, - }, - then: { kind: "numLit", value: transitions[i]!, type: F64, loc }, - else_: next, - type: F64, - loc, - }; + const next = indexChain(current, transitions, loc); + // The value evaluates before the key is tested and moved: it may add + // other keys to the same value, which then come first. + let value = st.value; + const pure = + value.kind === "numLit" || + value.kind === "strLit" || + value.kind === "boolLit" || + value.kind === "varRef"; + if (!pure) { + const localId = `%keyOrderValue.${temporaries++}`; + fn.locals.push({ id: localId, name: "%keyOrderValue", type: value.type, mutable: false }); + prologue.push({ kind: "varDecl", localId, init: value, loc }); + value = { kind: "varRef", localId, type: value.type, loc }; } const absent: IrExpr = { kind: "unary", @@ -1142,7 +1211,7 @@ export function applyKeyOrderWitnesses(lowerer: Lowerer, fns: readonly IrFunctio obj, shapeId: st.shapeId, field: st.field, - value: st.value, + value, loc, }; const guard: IrStmt = { kind: "if", cond: absent, then: [update], else_: null, loc }; @@ -1172,5 +1241,6 @@ export function applyKeyOrderWitnesses(lowerer: Lowerer, fns: readonly IrFunctio tables.push(table); } shape.orderVariants = tables; + if (moves.has(id)) shape.orderMoves = true; } } diff --git a/packages/compiler/src/ir/ir.ts b/packages/compiler/src/ir/ir.ts index 89a5ee73a..a9f5dda24 100644 --- a/packages/compiler/src/ir/ir.ts +++ b/packages/compiler/src/ir/ir.ts @@ -1673,6 +1673,10 @@ export interface IrRecordShape { * and every other construction leaves 0. Each key-order surface * dispatches on that field. */ orderVariants?: string[][]; + /** A shape with orderVariants whose values can change order after they + * are built: a write of an optional key a value lacks updates its + * KEY_ORDER_FIELD. */ + orderMoves?: true; } /** The hidden field of a shape with orderVariants: the index of the value's diff --git a/tests/corpus/7136-key-order-copies.ts b/tests/corpus/7136-key-order-copies.ts index daafba23b..5d754bced 100644 --- a/tests/corpus/7136-key-order-copies.ts +++ b/tests/corpus/7136-key-order-copies.ts @@ -17,3 +17,16 @@ for (const value of [a, b, c]) { console.log(name, JSON.stringify(rest), Object.keys(rest).join(",")); console.log(JSON.stringify(assigned), JSON.stringify(cloned)); } +// Keys after a spread that the spread value may lack: a key the value has +// keeps its place, and a key it lacks goes after the value's keys. +type Options = { id: number; label?: string; size?: number; tag?: string }; +const options: Options[] = [ + { id: 1, size: 2 }, + { size: 3, id: 2 }, + { label: "c", id: 3, tag: "t" }, + { tag: "u", size: 4, id: 4 }, +]; +for (const value of options) { + const merged = { ...value, tag: "x", label: "y", done: true }; + console.log(Object.keys(merged).join(","), JSON.stringify(merged)); +} diff --git a/tests/corpus/7138-key-order-late-writes.ts b/tests/corpus/7138-key-order-late-writes.ts index b1afd8761..d0adef3da 100644 --- a/tests/corpus/7138-key-order-late-writes.ts +++ b/tests/corpus/7138-key-order-late-writes.ts @@ -23,5 +23,15 @@ for (let i = 0; i < 2; i++) { console.log(Object.keys(built).join(",")); } console.log(built); -// d -// e +// A write's value runs first, so a key it adds comes before the written +// key. +const nested: Item = { count: 3, id: 4 }; +nested.name = ((nested.note = "first"), "second"); +console.log(Object.keys(nested).join(","), JSON.stringify(nested)); +// A value seen as a JavaScript value enumerates later writes in order. +const seen: Item = { id: 5, count: 4 }; +const view: unknown = seen; +console.log(Object.keys(view as object).join(",")); +seen.note = "n"; +seen.name = "m"; +console.log(Object.keys(view as object).join(","), JSON.stringify(view)); diff --git a/tests/harness/corpus-costs/7136-key-order-copies.ts.json b/tests/harness/corpus-costs/7136-key-order-copies.ts.json index ba3e270ee..9d76a548c 100644 --- a/tests/harness/corpus-costs/7136-key-order-copies.ts.json +++ b/tests/harness/corpus-costs/7136-key-order-copies.ts.json @@ -1 +1 @@ -{ "plain": 1700, "san": 2400 } +{ "plain": 1900, "san": 3500 } diff --git a/tests/harness/corpus-costs/7138-key-order-late-writes.ts.json b/tests/harness/corpus-costs/7138-key-order-late-writes.ts.json index 0b945a1c9..2b1aad28d 100644 --- a/tests/harness/corpus-costs/7138-key-order-late-writes.ts.json +++ b/tests/harness/corpus-costs/7138-key-order-late-writes.ts.json @@ -1 +1 @@ -{ "plain": 1700, "san": 2500 } +{ "plain": 1700, "san": 2800 } From 291587aae17242e55cb6a1ab9801735cb22d4e49 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 11:08:55 -0500 Subject: [PATCH 051/197] Scan regular expressions after jump statements, and give up on non-ASCII whitespace --- .../src/frontend/token-equivalence.test.ts | 22 +++++++++++ .../src/frontend/token-equivalence.ts | 37 ++++++++++++++----- 2 files changed, 49 insertions(+), 10 deletions(-) diff --git a/packages/compiler/src/frontend/token-equivalence.test.ts b/packages/compiler/src/frontend/token-equivalence.test.ts index afd81d9b6..18bfb32e6 100644 --- a/packages/compiler/src/frontend/token-equivalence.test.ts +++ b/packages/compiler/src/frontend/token-equivalence.test.ts @@ -37,6 +37,28 @@ test("comment-like text inside regular expressions is code", () => { ).toBe(false); }); +test("a regular expression can start after a statement that ends at its keyword or label", () => { + const loop = (body: string) => `a: for (;;) {\n${body}\n}\n`; + expect( + equivalent(loop("break\n/[//]/.test(x); c = 1;"), loop("break\n/[//]/.test(x); c = 2;")), + ).toBe(false); + expect( + equivalent(loop("break a\n/[//]/.test(x); c = 1;"), loop("break a\n/[//]/.test(x); c = 2;")), + ).toBe(false); + expect( + equivalent("debugger\n/[//]/.test(x); c = 1;\n", "debugger\n/[//]/.test(x); c = 2;\n"), + ).toBe(false); + // The next line's `x` is no label: it starts a new statement, and `/` divides. + expect(equivalent(loop("continue\nx / 2; // one"), loop("continue\nx / 2; // two"))).toBe(true); +}); + +test("whitespace outside ASCII is never equivalent", () => { + expect( + equivalent("return\u3000/[//]/.test(x); c = 1;\n", "return\u3000/[//]/.test(x); c = 2;\n"), + ).toBe(false); + expect(equivalent("const a = 1;\u00a0// one\n", "const a = 1;\u00a0// two\n")).toBe(false); +}); + test("U+2028 and U+2029 end a line comment, so their files are never equivalent", () => { expect(equivalent("// a\u2028c = 1;\n", "// a\u2028c = 2;\n")).toBe(false); expect(equivalent("/* a\u2029*/ c = 1;\n", "/* b\u2029*/ c = 1;\n")).toBe(false); diff --git a/packages/compiler/src/frontend/token-equivalence.ts b/packages/compiler/src/frontend/token-equivalence.ts index 94c60d836..b13b67498 100644 --- a/packages/compiler/src/frontend/token-equivalence.ts +++ b/packages/compiler/src/frontend/token-equivalence.ts @@ -19,10 +19,13 @@ * cannot tell which, so it gives up, as it does for escaped identifiers, * which can spell keywords. * - Template literals track their `${` substitutions on a brace stack. - * - JSX files, files that contain U+2028 or U+2029 anywhere (they end a - * line comment and separate lines for automatic semicolon insertion), - * and unterminated strings, comments, templates and regular expressions - * give up. + * - After `break`, `continue` and `debugger`, and after the label of a + * `break` or `continue`, a `/` starts a regular expression: it can only + * follow there once automatic semicolon insertion ends the statement. + * - JSX files, files that contain whitespace or a line terminator outside + * ASCII anywhere (the identifier scan would take it as part of a word, and + * U+2028 and U+2029 end a line comment), and unterminated strings, + * comments, templates and regular expressions give up. * * Plain character scans keep the scanner free of Unicode property escapes: * any non-ASCII code unit outside strings, templates, regular expressions @@ -55,6 +58,12 @@ const EXPRESSION_KEYWORDS = new Set([ "extends", ]); +/** Statements that end at their keyword, or at the label after it. */ +const JUMP_KEYWORDS = new Set(["break", "continue", "debugger"]); + +/** Whitespace and line terminators outside ASCII. */ +const NON_ASCII_SPACE = /[\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]/; + /** Keywords whose parenthesized header is followed by a statement. */ const HEADER_KEYWORDS = new Set(["if", "while", "for", "with"]); @@ -120,7 +129,7 @@ function isSpace(code: number): boolean { /** The tokens of `source`, or null when the scanner cannot classify it. */ function scan(path: string, source: string): Token[] | null { if (path.endsWith(".tsx") || path.endsWith(".jsx")) return null; - if (source.includes("\u2028") || source.includes("\u2029")) return null; + if (NON_ASCII_SPACE.test(source)) return null; const typeScript = isTypeScript(path); const tokens: Token[] = []; // Brace kinds: true for a template substitution, false otherwise. @@ -128,14 +137,23 @@ function scan(path: string, source: string): Token[] | null { // Paren kinds: true for a statement header. const parens: boolean[] = []; let lineBreakBefore = false; - // The previous token and the one before it, and whether a `)` closed a - // statement header. - const last: { token: string | null; before: string | null } = { token: null, before: null }; + // The previous token and the one before it, whether the previous token is + // the label of a `break` or `continue` on its line, and whether a `)` + // closed a statement header. + const last: { token: string | null; before: string | null; label: boolean } = { + token: null, + before: null, + label: false, + }; let closedHeader = false; const push = (text: string, comment = false): void => { tokens.push({ text, comment, lineBreakBefore }); lineBreakBefore = false; if (!comment) { + last.label = + (last.token === "break" || last.token === "continue") && + !tokens[tokens.length - 1]!.lineBreakBefore && + /^[A-Za-z_$]/.test(text); last.before = last.token; last.token = text; } @@ -184,7 +202,6 @@ function scan(path: string, source: string): Token[] | null { index++; continue; } - if (code > 127 && (code === 0xa0 || code === 0xfeff)) return null; const next = source.charCodeAt(index + 1); // Comments. if (code === 47 && (next === 47 || next === 42)) { @@ -276,7 +293,7 @@ function scan(path: string, source: string): Token[] | null { else if (before === "++" || before === "--") return null; else if (before === "]") regex = false; else if (/^[A-Za-z0-9_$\\]/.test(before) || before.charCodeAt(0) > 127) - regex = EXPRESSION_KEYWORDS.has(before); + regex = EXPRESSION_KEYWORDS.has(before) || JUMP_KEYWORDS.has(before) || last.label; else if (before.startsWith("'") || before.startsWith('"') || before.endsWith("`")) regex = false; else if (before.startsWith("/") && before.length > 1 && before !== "/=") regex = false; From 2b657e6e3cca7bb12f60ef1cb199ce79e73ac198 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 11:12:02 -0500 Subject: [PATCH 052/197] Format the split tools' output with the tools checkout's oxfmt configuration The split tools ran oxfmt in the checkout they rewrite. oxfmt looks up its configuration from that directory and from the formatted files' directories, and runs an oxfmt.config.ts it finds there, so a branch that rehome-split rewrites could run code with the maintainer's permissions. The tools now format through format.mjs, which passes the tools checkout's .oxfmtrc.jsonc with --config and disables the nested lookup, so no configuration file of the rewritten checkout is read or run. With no files to format it does nothing, where oxfmt would format the whole directory. The split-tools tests cover split-methods and move-state on a checkout whose root and src directory commit an oxfmt.config.ts that records it ran. --- scripts/split/README.md | 4 ++- scripts/split/format.mjs | 20 +++++++++++ scripts/split/move-state.mjs | 5 ++- scripts/split/rehome-split.mjs | 6 ++-- scripts/split/split-methods.mjs | 7 ++-- scripts/split/split-module.mjs | 7 ++-- tests/harness/split-tools.test.ts | 57 +++++++++++++++++++++++++++++++ 7 files changed, 88 insertions(+), 18 deletions(-) create mode 100644 scripts/split/format.mjs diff --git a/scripts/split/README.md b/scripts/split/README.md index e3cdfd077..be8c948ac 100644 --- a/scripts/split/README.md +++ b/scripts/split/README.md @@ -5,13 +5,15 @@ These scripts split a large source file into smaller files without changing beha Run them from a checkout with `node_modules` installed. That checkout is the tools directory (`--tools`) and supplies `typescript` and `oxfmt`. ## Transforms + - `split-module.mjs plan|apply ` moves top-level declarations into new files. `plan` computes a manifest from seed declarations, and `apply` does the move. - `split-methods.mjs apply ` moves class methods into exported functions that take the instance as a parameter (`emitter`). The class keeps a one-line stub with the same signature, so callers don't change. - `move-state.mjs apply ` moves a group of class fields into a feature's state object, defined in the feature's file with `defineModuleState` (`packages/compiler/src/backend/llvm/emitter-state.ts`). It rewrites `this.` and `emitter.` to `KEY.of(this).` and creates files from templates. It also applies the change's other edits as literal replacements, in order. A replacement whose text isn't found the expected number of times is reported and skipped. -Each transform takes `--repo

` (the tree to rewrite) and `--tools `, and formats what it writes with oxfmt. The tools read and write the tree's files only at real paths inside `--repo`, and a config's templates and manifest only inside the config's directory. A path that resolves elsewhere, through a symlink or `..`, stops the tool with an error (`confine.mjs`). `configs/` holds the configs of splits that have landed. For example, `configs/loop-state/` moves the loop optimizations' state out of `LlEmitter`. +Each transform takes `--repo ` (the tree to rewrite) and `--tools `, and formats what it writes with the tools checkout's oxfmt and its `.oxfmtrc.jsonc` (`format.mjs`). The formatter reads no configuration from `--repo`, so an `oxfmt.config.ts` a branch commits is never run. The tools read and write the tree's files only at real paths inside `--repo`, and a config's templates and manifest only inside the config's directory. A path that resolves elsewhere, through a symlink or `..`, stops the tool with an error (`confine.mjs`). `configs/` holds the configs of splits that have landed. For example, `configs/loop-state/` moves the loop optimizations' state out of `LlEmitter`. ## Rehoming an open branch + ```sh git fetch origin pull//head:pr- node scripts/split/rehome-split.mjs --branch pr- --onto origin/main --config scripts/split/configs//.json diff --git a/scripts/split/format.mjs b/scripts/split/format.mjs new file mode 100644 index 000000000..10c1fbc8a --- /dev/null +++ b/scripts/split/format.mjs @@ -0,0 +1,20 @@ +/* Formatting for the split tools. oxfmt reads its configuration from the + * directory it runs in, its parents and the formatted files' directories, + * and an oxfmt.config.ts there is code it runs. The tools format files in + * a checkout they rewrite, which can be a pull request's, so they pass the + * tools checkout's own configuration and turn the nested lookup off: no + * configuration file of the rewritten checkout is read or run. */ +import { execFileSync } from "node:child_process"; +import { join } from "node:path"; + +/** Formats `files` (paths relative to `cwd`) with the tools checkout's + * oxfmt and configuration. */ +export function formatFiles(toolsDir, files, { cwd, stdio }) { + // With no file arguments oxfmt would format the whole directory. + if (files.length === 0) return; + execFileSync( + join(toolsDir, "node_modules/.bin/oxfmt"), + ["--config", join(toolsDir, ".oxfmtrc.jsonc"), "--disable-nested-config", ...files], + { cwd, stdio }, + ); +} diff --git a/scripts/split/move-state.mjs b/scripts/split/move-state.mjs index b70023c67..71306f5d1 100644 --- a/scripts/split/move-state.mjs +++ b/scripts/split/move-state.mjs @@ -26,8 +26,8 @@ import { readFileSync } from "node:fs"; import { dirname, join, resolve } from "node:path"; import { createRequire } from "node:module"; -import { execFileSync } from "node:child_process"; import { confine } from "./confine.mjs"; +import { formatFiles } from "./format.mjs"; const [command, configPath, ...flags] = process.argv.slice(2); if (command !== "apply") throw new Error("usage: move-state.mjs apply "); @@ -199,8 +199,7 @@ for (const r of config.replacements ?? []) { checkout.write(path, text.split(r.find).join(r.replace)); touched.add(path); } -const oxfmt = join(toolsDir, "node_modules/.bin/oxfmt"); -execFileSync(oxfmt, [...touched], { cwd: repo, stdio: "inherit" }); +formatFiles(toolsDir, [...touched], { cwd: repo, stdio: "inherit" }); if (skipped) console.warn(`${skipped} replacement(s) skipped`); console.log( `moved ${config.groups.map((g) => g.present.length).reduce((a, b) => a + b, 0)} fields; rewrote ${touched.size} files`, diff --git a/scripts/split/rehome-split.mjs b/scripts/split/rehome-split.mjs index 89c32d6b7..6533301f8 100644 --- a/scripts/split/rehome-split.mjs +++ b/scripts/split/rehome-split.mjs @@ -29,6 +29,7 @@ import { createRequire } from "node:module"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { confine } from "./confine.mjs"; +import { formatFiles } from "./format.mjs"; const args = process.argv.slice(2); const opt = (name, fallback) => { @@ -173,10 +174,7 @@ try { splitFiles.has(file) && resolveImportConflicts(confine(resultDir), join(resultDir, file)) ) { - execFileSync(join(tools, "node_modules/.bin/oxfmt"), [file], { - cwd: resultDir, - stdio: ["ignore", "ignore", "inherit"], - }); + formatFiles(tools, [file], { cwd: resultDir, stdio: ["ignore", "ignore", "inherit"] }); gitIn(resultDir, "add", file); console.log(`${file}: import conflicts resolved to both sides' imports`); } diff --git a/scripts/split/split-methods.mjs b/scripts/split/split-methods.mjs index 05d72c4be..4d57644af 100644 --- a/scripts/split/split-methods.mjs +++ b/scripts/split/split-methods.mjs @@ -23,8 +23,8 @@ import { readFileSync } from "node:fs"; import { dirname, join, relative, resolve, posix } from "node:path"; import { createRequire } from "node:module"; -import { execFileSync } from "node:child_process"; import { confine, confineHost } from "./confine.mjs"; +import { formatFiles } from "./format.mjs"; const [command, configPath, ...flags] = process.argv.slice(2); if (command !== "apply") throw new Error("usage: split-methods.mjs apply "); @@ -413,8 +413,5 @@ const touched = [ modulePath, ...[...files].filter(([, e]) => e.decls.length).map(([f]) => join(moduleDir, f)), ]; -execFileSync(join(toolsDir, "node_modules/.bin/oxfmt"), touched, { - cwd: repo, - stdio: ["ignore", "ignore", "inherit"], -}); +formatFiles(toolsDir, touched, { cwd: repo, stdio: ["ignore", "ignore", "inherit"] }); console.log(`${moved} methods moved; ${publicize.size} members reached through host`); diff --git a/scripts/split/split-module.mjs b/scripts/split/split-module.mjs index ba340df40..4b8764d80 100644 --- a/scripts/split/split-module.mjs +++ b/scripts/split/split-module.mjs @@ -39,8 +39,8 @@ import { readFileSync } from "node:fs"; import { dirname, join, relative, resolve, posix } from "node:path"; import { createRequire } from "node:module"; -import { execFileSync } from "node:child_process"; import { confine, confineHost } from "./confine.mjs"; +import { formatFiles } from "./format.mjs"; const [command, configPath, ...flags] = process.argv.slice(2); const config = JSON.parse(readFileSync(configPath, "utf8")); @@ -533,10 +533,7 @@ function apply() { // Format what changed with the tools checkout's oxfmt, so the split and // every rehome of it are formatted the same way. const touched = [modulePath, ...[...files.keys()].map((f) => join(moduleDir, f))]; - execFileSync(join(toolsDir, "node_modules/.bin/oxfmt"), touched, { - cwd: repo, - stdio: ["ignore", "ignore", "inherit"], - }); + formatFiles(toolsDir, touched, { cwd: repo, stdio: ["ignore", "ignore", "inherit"] }); for (const [file, entry] of files) console.log(`${file}: ${entry.decls.length} declarations`); } diff --git a/tests/harness/split-tools.test.ts b/tests/harness/split-tools.test.ts index 08d89e850..8bdefdf0e 100644 --- a/tests/harness/split-tools.test.ts +++ b/tests/harness/split-tools.test.ts @@ -97,6 +97,63 @@ test("split-methods keeps an import a staying method uses as a shorthand propert expect(diagnostics(dir)).toEqual([]); }); +/** An oxfmt.config.ts that records it ran by writing `marker`. */ +const recordingConfig = (marker: string): string => + [ + 'import { writeFileSync } from "node:fs";', + `writeFileSync(${JSON.stringify(marker)}, "ran");`, + "export default {};", + "", + ].join("\n"); + +test("split-methods formats with the tools checkout's configuration, not the branch's", () => { + const dir = checkout({ + "src/host.ts": [ + "export class Host {", + " value = 1;", + " moved(): number {", + " return this.value + 1;", + " }", + "}", + "", + ].join("\n"), + "split.json": JSON.stringify({ + module: "src/host.ts", + class: "Host", + methods: { "moved.ts": ["moved"] }, + }), + }); + const markers = [join(dir, "root-config-ran"), join(dir, "nested-config-ran")]; + writeFileSync(join(dir, "oxfmt.config.ts"), recordingConfig(markers[0]!)); + writeFileSync(join(dir, "src/oxfmt.config.ts"), recordingConfig(markers[1]!)); + run("split-methods.mjs", ["apply", join(dir, "split.json")], dir); + expect(readFileSync(join(dir, "src/moved.ts"), "utf8")).toContain("export function moved"); + for (const marker of markers) expect(existsSync(marker)).toBe(false); +}); + +test("move-state formats with the tools checkout's configuration, not the branch's", () => { + const dir = checkout({ + "src/host.ts": + "export class Host {\n count = 0;\n moved(): number {\n return this.count + 1;\n }\n}\n", + "state.template": "", + "split.json": JSON.stringify({ + module: "src/host.ts", + class: "Host", + dir: "src", + create: { "host-state.ts": "state.template" }, + groups: [ + { file: "host-state.ts", key: "COUNTS", interfaceName: "Counts", fields: ["count"] }, + ], + }), + }); + const markers = [join(dir, "root-config-ran"), join(dir, "nested-config-ran")]; + writeFileSync(join(dir, "oxfmt.config.ts"), recordingConfig(markers[0]!)); + writeFileSync(join(dir, "src/oxfmt.config.ts"), recordingConfig(markers[1]!)); + run("move-state.mjs", ["apply", join(dir, "split.json")], dir); + expect(readFileSync(join(dir, "src/host.ts"), "utf8")).toContain("COUNTS.of(this).count"); + for (const marker of markers) expect(existsSync(marker)).toBe(false); +}); + test("split-module imports a moved class's local base class as a value", () => { const dir = checkout({ "src/shapes.ts": [ From fd40a8c84a136cdeaa18b08d5cf4c566e32eadd7 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 11:18:30 -0500 Subject: [PATCH 053/197] Try each PATH entry with exec when spawning with a replacement env The PATH lookup for spawns with an `env` option picked the first executable regular file and stopped there, so a script whose interpreter is missing ended the search with ENOENT. Node's libuv runs execvp in the child, which moves on to the next PATH entry when an exec fails with ENOENT, ENOTDIR or EACCES. Each candidate now runs through posix_spawn in order with the same rule, and relative entries are resolved by the child in its cwd. --- packages/runtime/src/scr_child.c | 76 +++++++--------------- tests/corpus/7036-child-env-path-lookup.ts | 15 ++++- 2 files changed, 36 insertions(+), 55 deletions(-) diff --git a/packages/runtime/src/scr_child.c b/packages/runtime/src/scr_child.c index c1d98b35b..df7543500 100644 --- a/packages/runtime/src/scr_child.c +++ b/packages/runtime/src/scr_child.c @@ -2907,7 +2907,6 @@ void scr_children_poll(void) { #include #include #include -#include #include #include #include @@ -3112,21 +3111,23 @@ static ScrSpawnRes *scr_spawn_res_new(bool has_status, double status, static char **scr_exec_envp(ScrArr *pairs); static void scr_exec_envp_free(char **envp); -/* The executable `file` names when `envp` replaces the child's - * environment, found as libuv finds it: a name with a '/' is used as - * given; otherwise each entry of the replacement's PATH (an empty entry - * is the current directory), or of the default path when the replacement - * has no PATH, is tried in order. Relative candidates are checked from - * `cwd`, the directory the child starts in (NULL: this process's). Returns - * a malloc'd path, or NULL with *err set: EACCES when a candidate exists - * but cannot run, else ENOENT. */ -static char *scr_spawn_path_lookup(const char *file, char *const *envp, const char *cwd, - int *err) { - if (strchr(file, '/')) { - char *copy = strdup(file); - if (!copy) scr_child_oom(); - return copy; - } +/* posix_spawnp for the child_process spawns. Without `envp` the command + * resolves on this process's PATH over `environ`. With `envp` (a + * replacement environment) it resolves the way Node's libuv runs execvp + * in the child: a name with a '/' runs as given; otherwise each entry of + * the replacement's PATH (an empty entry is the current directory), or of + * the default path when the replacement has no PATH, is tried in order, + * and a candidate whose exec fails with ENOENT, ENOTDIR or EACCES (a + * missing file, or a script whose interpreter is missing) moves on to the + * next entry. The last such failure is EACCES if any candidate was denied, + * else ENOENT. Relative entries are resolved by the child, which starts in + * `cwd`. Returns posix_spawn's error number. */ +static int scr_spawn_command(pid_t *pid, const char *file, const posix_spawn_file_actions_t *fa, + const posix_spawnattr_t *attr, char *const *argv, + char *const *envp, const char *cwd) { + (void)cwd; /* the file actions change into it before the exec */ + if (envp == NULL) return posix_spawnp(pid, file, fa, attr, argv, environ); + if (strchr(file, '/')) return posix_spawn(pid, file, fa, attr, argv, envp); const char *path = "/usr/bin:/bin"; for (char *const *e = envp; *e; e++) { if (strncmp(*e, "PATH=", 5) == 0) { @@ -3135,7 +3136,6 @@ static char *scr_spawn_path_lookup(const char *file, char *const *envp, const ch } } size_t file_len = strlen(file); - size_t cwd_len = cwd ? strlen(cwd) : 0; bool denied = false; for (const char *dir = path;;) { const char *end = strchr(dir, ':'); @@ -3147,51 +3147,19 @@ static char *scr_spawn_path_lookup(const char *file, char *const *envp, const ch memcpy(candidate, prefix, prefix_len); candidate[prefix_len] = '/'; memcpy(candidate + prefix_len + 1, file, file_len + 1); - char *checked = candidate; - if (cwd_len > 0 && candidate[0] != '/') { - checked = malloc(cwd_len + 1 + strlen(candidate) + 1); - if (!checked) scr_child_oom(); - memcpy(checked, cwd, cwd_len); - checked[cwd_len] = '/'; - strcpy(checked + cwd_len + 1, candidate); - } - struct stat st; - bool runs = false; - if (stat(checked, &st) == 0) { - runs = S_ISREG(st.st_mode) && access(checked, X_OK) == 0; - if (!runs) denied = true; - } else if (errno == EACCES) { - denied = true; - } - if (checked != candidate) free(checked); - if (runs) return candidate; + int err = posix_spawn(pid, candidate, fa, attr, argv, envp); free(candidate); + if (err == EACCES) denied = true; + else if (err != ENOENT && err != ENOTDIR) return err; if (!end) break; dir = end + 1; } - *err = denied ? EACCES : ENOENT; - return NULL; -} - -/* posix_spawnp for the child_process spawns: with `envp` (a replacement - * environment) the command resolves on that environment's PATH - * (scr_spawn_path_lookup) as Node's does; without it, on this process's - * PATH over `environ`. Returns posix_spawn's error number. */ -static int scr_spawn_command(pid_t *pid, const char *file, const posix_spawn_file_actions_t *fa, - const posix_spawnattr_t *attr, char *const *argv, - char *const *envp, const char *cwd) { - if (envp == NULL) return posix_spawnp(pid, file, fa, attr, argv, environ); - int err = 0; - char *resolved = scr_spawn_path_lookup(file, envp, cwd, &err); - if (!resolved) return err; - err = posix_spawn(pid, resolved, fa, attr, argv, envp); - free(resolved); - return err; + return denied ? EACCES : ENOENT; } /* env_pairs: NULL inherits the parent's environment; else [k,v,...] * replaces it, as Node's `env` option does, and the command is looked up - * on the replacement's PATH (scr_spawn_path_lookup). */ + * on the replacement's PATH (scr_spawn_command). */ static ScrSpawnRes *scr_spawn_sync_core(ScrStr *cmd, ScrArr *args, double timeout_ms, int killsig, int in_mode, int out_mode, int err_mode, ScrArr *env_pairs) { diff --git a/tests/corpus/7036-child-env-path-lookup.ts b/tests/corpus/7036-child-env-path-lookup.ts index cac0670e9..97c93d835 100644 --- a/tests/corpus/7036-child-env-path-lookup.ts +++ b/tests/corpus/7036-child-env-path-lookup.ts @@ -11,6 +11,12 @@ mkdirSync(join(dir, "tools")); writeFileSync(join(dir, "tools", "child-tool"), "#!/bin/sh\necho child-tool $1\n"); chmodSync(join(dir, "tools", "child-tool"), 0o755); +// An earlier PATH entry holding a script whose interpreter is missing +// moves the lookup on to the next entry, as execvp does. +mkdirSync(join(dir, "broken")); +writeFileSync(join(dir, "broken", "child-tool"), "#!/nonexistent/interpreter\n"); +chmodSync(join(dir, "broken", "child-tool"), 0o755); + const show = (e: unknown): string => { const err = e as NodeJS.ErrnoException; return `${err.code} ${err.message.replace(dir, "")}`; @@ -23,11 +29,18 @@ try { } catch (e) { console.log(show(e)); } +const twoEntries = `${join(dir, "broken")}:${join(dir, "tools")}`; +console.log(execFileSync("child-tool", ["fallthrough"], { env: { PATH: twoEntries }, encoding: "utf8" }).trim()); +try { + execFileSync("child-tool", [], { env: { PATH: join(dir, "broken") }, encoding: "utf8" }); +} catch (e) { + console.log(show(e)); +} console.log( execFileSync("child-tool", ["relative"], { env: { PATH: "tools" }, cwd: dir, encoding: "utf8" }).trim(), ); -const child = spawn("child-tool", ["async"], { env: { PATH: join(dir, "tools") }, stdio: ["ignore", "pipe", "ignore"] }); +const child = spawn("child-tool", ["async"], { env: { PATH: twoEntries }, stdio: ["ignore", "pipe", "ignore"] }); let out = ""; child.stdout!.on("data", (chunk: Buffer) => (out += chunk.toString())); child.on("close", (code) => { From 7cce36354f20204698523128fc3072b5c58ca5ad Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 11:21:10 -0500 Subject: [PATCH 054/197] Dispatch checked-dynamic method calls on method ids A method call on a value of unknown type (untyped JavaScript, `any`) picked its builtin by comparing the method name with strcmp against each candidate, in long ladders per receiver kind: a string method could test dozens of names before it matched. - The name is now looked up once per call in a generated hash table (scr_dyn_methods.h, from scripts/gen-dyn-methods.mjs), and each ladder compares the resulting id. The table verifies the name's bytes, so a name is matched by its content, as before. - The worker event methods compare the first character before strcmp. - bench/dynamic-methods calls string, array, Map, Set, number and own methods from untyped JavaScript. --- benchmarks/runtime/README.md | 1 + .../runtime/workloads/dynamic-methods.json | 6 + .../runtime/workloads/dynamic-methods.mjs | 36 +++ packages/runtime/src/scr_dyn_invoke.c | 255 +++++++++--------- packages/runtime/src/scr_dyn_methods.h | 160 +++++++++++ packages/runtime/src/scr_dyn_methods.test.ts | 10 + packages/runtime/src/scr_worker_events.c | 40 +-- scripts/gen-dyn-methods.mjs | 101 +++++++ scripts/zero-cost-compare.mjs | 4 +- 9 files changed, 473 insertions(+), 140 deletions(-) create mode 100644 benchmarks/runtime/workloads/dynamic-methods.json create mode 100644 benchmarks/runtime/workloads/dynamic-methods.mjs create mode 100644 packages/runtime/src/scr_dyn_methods.h create mode 100644 packages/runtime/src/scr_dyn_methods.test.ts create mode 100644 scripts/gen-dyn-methods.mjs diff --git a/benchmarks/runtime/README.md b/benchmarks/runtime/README.md index 2653ec51a..e8acbfbb5 100644 --- a/benchmarks/runtime/README.md +++ b/benchmarks/runtime/README.md @@ -11,6 +11,7 @@ Application-shaped workloads for measuring the speed of compiled executables. Ea | `channel-messages` | cpu | 200,000 MessageChannel messages in one thread, half numbers and half small objects, all posted before the port delivers any | | `csv-numbers` | cpu | `split`, `Number`, `parseInt`, `toFixed` | | `cyclic-graphs` | cpu | object graphs with back references built and dropped: trees with parent and sibling links, rings of records sharing a long-lived registry, closures capturing their owner | +| `dynamic-methods` | cpu | untyped JavaScript calling builtin methods on values it does not type: string `slice`/`indexOf`/`toUpperCase`/`padStart`, array `push`/`pop`/`includes`, `Map` and `Set`, and an object's own methods | | `functional-pipeline` | cpu | `filter`/`map`/`reduce`/`flatMap` chains, closures, small records | | `intern-table`, `intern-table-map`, `intern-table-1`, `intern-table-map-1` | cpu | worker threads interning the same type-like objects under string keys and records under number keys, then hits and misses: one shared `InternTable` per kind of key, or a module-level `Map` per thread (the `-map` entries), with 8 threads or 1 (the `-1` entries); under Node.js every thread has its own tables | | `json-records` | cpu | `JSON.stringify`, typed `JSON.parse`, Map aggregation | diff --git a/benchmarks/runtime/workloads/dynamic-methods.json b/benchmarks/runtime/workloads/dynamic-methods.json new file mode 100644 index 000000000..fa27f4ea8 --- /dev/null +++ b/benchmarks/runtime/workloads/dynamic-methods.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/dynamic-methods.mjs", + "args": [ + "300000" + ] +} diff --git a/benchmarks/runtime/workloads/dynamic-methods.mjs b/benchmarks/runtime/workloads/dynamic-methods.mjs new file mode 100644 index 000000000..bbf217701 --- /dev/null +++ b/benchmarks/runtime/workloads/dynamic-methods.mjs @@ -0,0 +1,36 @@ +// Untyped JavaScript calling builtin methods on values it never declares +// types for: string slice/indexOf/toUpperCase/startsWith/padStart, array +// push/pop/at/join/includes/indexOf, Map and Set get/set/has/add, number +// toFixed, and calls of an object's own methods. Argument: the number of +// iterations. +const rounds = Number(process.argv[2] ?? "300000"); +const words = ["alpha", "beta", "gamma", "delta", "epsilon", "zeta", "eta", "theta"]; +const counts = new Map(); +const seen = new Set(); +const stack = []; +const recent = []; +const tally = { + total: 0, + add(value) { + this.total += value; + return this.total; + }, +}; +let checksum = 0; +for (let i = 0; i < rounds; i++) { + const word = words[i % words.length]; + const head = word.slice(0, 3).toUpperCase(); + const key = head.padStart(4, "-"); + if (word.indexOf("e") >= 0 && !word.startsWith("z")) checksum += 1; + counts.set(key, (counts.get(key) ?? 0) + 1); + seen.add(i % 97); + if (seen.has(i % 89)) checksum += 2; + stack.push(i); + if (stack.length > 32) checksum += stack.pop() % 3; + recent.push(word); + if (recent.length > 4) recent.shift(); + if (recent.includes("gamma")) checksum += recent.indexOf("gamma"); + checksum += tally.add(i % 5) % 11; + if (i % 1000 === 0) checksum += Number((i / 7).toFixed(2)) | 0; +} +console.log(checksum, counts.size, seen.size, stack.at(-1), recent.join(","), tally.total); diff --git a/packages/runtime/src/scr_dyn_invoke.c b/packages/runtime/src/scr_dyn_invoke.c index 80e01b08c..b0f30ac74 100644 --- a/packages/runtime/src/scr_dyn_invoke.c +++ b/packages/runtime/src/scr_dyn_invoke.c @@ -8,6 +8,7 @@ * scr_json.c; this unit uses only its public surface plus ScrJsonBuf. */ #include "scr_runtime.h" +#include "scr_dyn_methods.h" #include #include @@ -549,7 +550,11 @@ ScrDyn *scr_dyn_array_proto_call(ScrDyn *recv, ScrStr *method, ScrDyn *args) { return scr_dyn_retain(scr_dyn_undefined()); } -static bool dyn_name_is(const char *m, const char *n) { return strcmp(m, n) == 0; } +/* The dispatch below maps a call's method name to an id once + * (dyn_method_of, scr_dyn_methods.h) and its branches compare ids. Names + * compare by content, and any name the branches do not test is DYN_M_NONE. + * scripts/gen-dyn-methods.mjs writes the header from the DYN_M_ ids used + * here. */ /* Handle receivers: hand the whole call to the tag's ops (installed by * the owning unit at main() — a missing install is an internal error the @@ -692,25 +697,30 @@ static bool dyn_arr_reversed_copy(ScrDyn *out, const ScrDyn *source) { /* Names each prototype declares BEYOND what's implemented here — these * throw the "not supported yet" Error instead of mis-answering "is not a * function". */ -static bool dyn_arr_proto_unimpl(const char *m) { - static const char *names[] = { "toString", "toLocaleString", NULL }; - for (size_t i = 0; names[i]; i++) if (dyn_name_is(m, names[i])) return true; - return false; +static bool dyn_arr_proto_unimpl(int m) { + return m == DYN_M_toString || m == DYN_M_toLocaleString; } -static bool dyn_arr_proto_mutates(const char *m) { - static const char *names[] = { - "push", "pop", "shift", "unshift", "reverse", "sort", "fill", "copyWithin", "splice", NULL - }; - for (size_t i = 0; names[i]; i++) if (dyn_name_is(m, names[i])) return true; - return false; +static bool dyn_arr_proto_mutates(int m) { + switch (m) { + case DYN_M_push: case DYN_M_pop: case DYN_M_shift: case DYN_M_unshift: case DYN_M_reverse: + case DYN_M_sort: case DYN_M_fill: case DYN_M_copyWithin: case DYN_M_splice: + return true; + default: + return false; + } } -static bool dyn_bytes_proto_real(const char *m) { - static const char *names[] = { "slice", "at", "indexOf", "lastIndexOf", "includes", "join", - "forEach", "map", "filter", "some", "every", "find", "findIndex", "reverse", "fill", "set", - "subarray", "sort", "keys", "values", "entries", "reduce", "reduceRight", "copyWithin", - "toString", "toLocaleString", NULL }; - for (size_t i = 0; names[i]; i++) if (dyn_name_is(m, names[i])) return true; - return false; +static bool dyn_bytes_proto_real(int m) { + switch (m) { + case DYN_M_slice: case DYN_M_at: case DYN_M_indexOf: case DYN_M_lastIndexOf: case DYN_M_includes: + case DYN_M_join: case DYN_M_forEach: case DYN_M_map: case DYN_M_filter: case DYN_M_some: + case DYN_M_every: case DYN_M_find: case DYN_M_findIndex: case DYN_M_reverse: case DYN_M_fill: + case DYN_M_set: case DYN_M_subarray: case DYN_M_sort: case DYN_M_keys: case DYN_M_values: + case DYN_M_entries: case DYN_M_reduce: case DYN_M_reduceRight: case DYN_M_copyWithin: + case DYN_M_toString: case DYN_M_toLocaleString: + return true; + default: + return false; + } } static ScrDyn *scr_dyn_invoke_impl( @@ -724,18 +734,18 @@ static ScrDyn *scr_dyn_invoke_impl( * array. Anything that could observe more — a hole, argument coercion, a * multi-value push, a value the element type rejects — sets *handled false * and takes the snapshot path. */ -static ScrDyn *dyn_live_array_method(ScrDyn *recv, const char *method, ScrDyn *const *args, +static ScrDyn *dyn_live_array_method(ScrDyn *recv, int mid, ScrDyn *const *args, size_t argc, const char *what, bool *handled) { const ScrDynTypedArrayOps *ops = scr_dyn_typed_ref_array_ops(recv); if (!ops) return NULL; ScrArr *arr = recv->v.typed_ref.ptr; size_t len = arr->len; - if (dyn_name_is(method, "push") && argc == 1 && ops->set) { + if ((mid == DYN_M_push) && argc == 1 && ops->set) { if (!ops->set(arr, (double)len, args[0])) return NULL; *handled = true; return scr_exc_pending() ? NULL : scr_dyn_new_num((double)arr->len); } - if (dyn_name_is(method, "pop")) { + if ((mid == DYN_M_pop)) { if (len == 0) { *handled = true; return scr_dyn_retain(scr_dyn_undefined()); @@ -746,7 +756,7 @@ static ScrDyn *dyn_live_array_method(ScrDyn *recv, const char *method, ScrDyn *c *handled = true; return last; } - if (dyn_name_is(method, "at") && (argc == 0 || args[0]->kind == SCR_DYN_NUM || args[0]->kind == SCR_DYN_UNDEF)) { + if ((mid == DYN_M_at) && (argc == 0 || args[0]->kind == SCR_DYN_NUM || args[0]->kind == SCR_DYN_UNDEF)) { double index = dyn_index_arg(args, argc, 0, 0, what); if (index < 0) index += (double)len; if (index < 0 || index >= (double)len) { @@ -792,6 +802,7 @@ ScrDyn *scr_dyn_invoke(ScrDyn *recv, const char *method, static ScrDyn *scr_dyn_invoke_impl( ScrDyn *recv, const char *method, ScrDyn *const *args, size_t argc, const char *what, ScrDyn *callback_recv) { + const int mid = dyn_method_of(method); if (recv->kind == SCR_DYN_UNDEF || recv->kind == SCR_DYN_NULL) { ScrJsonBuf b; scr_jb_init(&b); @@ -811,7 +822,7 @@ static ScrDyn *scr_dyn_invoke_impl( if (recv->kind == SCR_DYN_HANDLE) { return scr_dynh_dispatch(recv, method, args, argc, what); } - if (scr_dyn_is_array_prototype(recv) && dyn_arr_proto_mutates(method)) { + if (scr_dyn_is_array_prototype(recv) && dyn_arr_proto_mutates(mid)) { scr_dyn_array_prototype_element_fence(recv); return NULL; } @@ -836,7 +847,7 @@ static ScrDyn *scr_dyn_invoke_impl( * snapshot identity back to the externally visible capsule. */ if (recv->kind == SCR_DYN_TYPED_REF) { bool handled = false; - ScrDyn *fast = dyn_live_array_method(recv, method, args, argc, what, &handled); + ScrDyn *fast = dyn_live_array_method(recv, mid, args, argc, what, &handled); if (handled) return fast; /* Iterator helpers on a compiled generator. */ if (scr_dyn_generator_helper_hook && @@ -861,7 +872,7 @@ static ScrDyn *scr_dyn_invoke_impl( return NULL; } bool mutates = materialized->kind == SCR_DYN_ARR && - dyn_arr_proto_mutates(method); + dyn_arr_proto_mutates(mid); ScrDyn *result = scr_dyn_invoke_impl( materialized, method, args, argc, what, recv); if (mutates && !scr_exc_pending()) scr_dyn_typed_ref_commit(recv); @@ -892,18 +903,18 @@ static ScrDyn *scr_dyn_invoke_impl( } if (recv->kind == SCR_DYN_FUNC) { - if (dyn_name_is(method, "apply")) { + if ((mid == DYN_M_apply)) { return scr_dyn_apply_array_like(recv, argc ? args[0] : scr_dyn_undefined(), argc > 1 ? args[1] : scr_dyn_undefined(), what); } - if (dyn_name_is(method, "call")) { + if ((mid == DYN_M_call)) { scr_dyn_this_push_dyn(argc >= 1 ? args[0] : scr_dyn_undefined()); ScrDyn *r = scr_dyn_call(recv, argc > 1 ? args + 1 : NULL, argc > 1 ? argc - 1 : 0, what); scr_dyn_this_pop(); return r; } - if (dyn_name_is(method, "bind")) return scr_dyn_bind(recv, args, argc); - if (dyn_name_is(method, "toString")) { + if ((mid == DYN_M_bind)) return scr_dyn_bind(recv, args, argc); + if ((mid == DYN_M_toString)) { ScrDyn *member = scr_dyn_fn_get(recv, method, strlen(method)); if (member && member->kind != SCR_DYN_UNDEF) { scr_dyn_this_push_dyn(recv); @@ -938,9 +949,9 @@ static ScrDyn *scr_dyn_invoke_impl( } if ((recv->kind == SCR_DYN_SYMBOL || recv->kind == SCR_DYN_BIGINT || recv->kind == SCR_DYN_NUM || recv->kind == SCR_DYN_BOOL || recv->kind == SCR_DYN_STR) && - dyn_name_is(method, "valueOf")) return scr_dyn_retain(recv); + (mid == DYN_M_valueOf)) return scr_dyn_retain(recv); - if (recv->kind == SCR_DYN_NUM && dyn_name_is(method, "toFixed")) { + if (recv->kind == SCR_DYN_NUM && (mid == DYN_M_toFixed)) { double digits = dyn_index_arg(args, argc, 0, 0, what); if (scr_exc_pending()) return NULL; ScrStr *text = scr_num_to_fixed(recv->v.num, digits); @@ -949,7 +960,7 @@ static ScrDyn *scr_dyn_invoke_impl( scr_str_release(text); return result; } - if (recv->kind == SCR_DYN_NUM && dyn_name_is(method, "toExponential")) { + if (recv->kind == SCR_DYN_NUM && (mid == DYN_M_toExponential)) { if (argc && args[0]->kind != SCR_DYN_UNDEF) { dyn_throw_unsupported("Number", method); return NULL; @@ -962,7 +973,7 @@ static ScrDyn *scr_dyn_invoke_impl( if (recv->kind == SCR_DYN_STR) { ScrStr *s = recv->v.str; - if (dyn_name_is(method, "slice")) { + if ((mid == DYN_M_slice)) { double start = dyn_index_arg(args, argc, 0, 0, what); if (scr_exc_pending()) return NULL; double end = dyn_index_arg(args, argc, 1, scr_str_utf16_len(s), what); @@ -972,16 +983,16 @@ static ScrDyn *scr_dyn_invoke_impl( scr_str_release(piece); return r; } - if (dyn_name_is(method, "at") || dyn_name_is(method, "concat") || - dyn_name_is(method, "indexOf") || dyn_name_is(method, "lastIndexOf") || - dyn_name_is(method, "includes")) { - if ((dyn_name_is(method, "indexOf") || dyn_name_is(method, "lastIndexOf") || - dyn_name_is(method, "includes")) && + if ((mid == DYN_M_at) || (mid == DYN_M_concat) || + (mid == DYN_M_indexOf) || (mid == DYN_M_lastIndexOf) || + (mid == DYN_M_includes)) { + if (((mid == DYN_M_indexOf) || (mid == DYN_M_lastIndexOf) || + (mid == DYN_M_includes)) && argc >= 1 && args[0]->kind == SCR_DYN_STR) { - double position = dyn_index_arg(args, argc, 1, dyn_name_is(method, "lastIndexOf") ? INFINITY : 0, what); + double position = dyn_index_arg(args, argc, 1, (mid == DYN_M_lastIndexOf) ? INFINITY : 0, what); if (scr_exc_pending()) return NULL; - if (dyn_name_is(method, "includes")) return scr_dyn_new_bool(scr_str_index_of(s, args[0]->v.str, position) >= 0); - if (dyn_name_is(method, "indexOf")) return scr_dyn_new_num(scr_str_index_of(s, args[0]->v.str, position)); + if ((mid == DYN_M_includes)) return scr_dyn_new_bool(scr_str_index_of(s, args[0]->v.str, position) >= 0); + if ((mid == DYN_M_indexOf)) return scr_dyn_new_num(scr_str_index_of(s, args[0]->v.str, position)); return scr_dyn_new_num(scr_str_last_index_of_from(s, args[0]->v.str, position)); } dyn_throw_unsupported("String", method); @@ -992,19 +1003,19 @@ static ScrDyn *scr_dyn_invoke_impl( } if (recv->kind == SCR_DYN_ARR) { - if (dyn_name_is(method, "keys") || dyn_name_is(method, "values") || dyn_name_is(method, "entries")) + if ((mid == DYN_M_keys) || (mid == DYN_M_values) || (mid == DYN_M_entries)) return scr_dyn_native_handle_iterator(callback_recv ? callback_recv : recv, - dyn_name_is(method, "keys") ? 1 : dyn_name_is(method, "entries") ? 2 : 0); + (mid == DYN_M_keys) ? 1 : (mid == DYN_M_entries) ? 2 : 0); size_t len = recv->v.arr.len; if (recv->v.arr.presence) { bool sparse = false; for (size_t i = 0; i < len; i++) if (!recv->v.arr.presence[i]) { sparse = true; break; } if (!sparse) { free(recv->v.arr.presence); recv->v.arr.presence = NULL; } - else if (dyn_name_is(method, "forEach") || dyn_name_is(method, "map") || dyn_name_is(method, "filter") || - dyn_name_is(method, "some") || dyn_name_is(method, "every") || dyn_name_is(method, "find") || - dyn_name_is(method, "findIndex") || dyn_name_is(method, "findLast") || dyn_name_is(method, "findLastIndex") || - dyn_name_is(method, "reduce") || dyn_name_is(method, "reduceRight") || dyn_name_is(method, "slice") || - dyn_name_is(method, "indexOf") || dyn_name_is(method, "lastIndexOf") || dyn_name_is(method, "includes")) { + else if ((mid == DYN_M_forEach) || (mid == DYN_M_map) || (mid == DYN_M_filter) || + (mid == DYN_M_some) || (mid == DYN_M_every) || (mid == DYN_M_find) || + (mid == DYN_M_findIndex) || (mid == DYN_M_findLast) || (mid == DYN_M_findLastIndex) || + (mid == DYN_M_reduce) || (mid == DYN_M_reduceRight) || (mid == DYN_M_slice) || + (mid == DYN_M_indexOf) || (mid == DYN_M_lastIndexOf) || (mid == DYN_M_includes)) { ScrDyn *pack = scr_dyn_new_arr(); for (size_t i = 0; i < argc; i++) scr_dyn_arr_push(pack, scr_dyn_retain(args[i])); ScrStr *name = scr_str_new(method, strlen(method)); @@ -1012,7 +1023,7 @@ static ScrDyn *scr_dyn_invoke_impl( scr_str_release(name); scr_dyn_release(pack); return result; - } else if (dyn_arr_proto_mutates(method) && !dyn_name_is(method, "fill") && !dyn_name_is(method, "push") && !dyn_name_is(method, "pop") && !dyn_name_is(method, "shift")) { + } else if (dyn_arr_proto_mutates(mid) && !(mid == DYN_M_fill) && !(mid == DYN_M_push) && !(mid == DYN_M_pop) && !(mid == DYN_M_shift)) { static const char message[] = "This checked array mutation over holes is not supported yet"; scr_throw_error_msg_code(SCR_ERR_ERROR, message, sizeof message - 1, "SC2020"); return NULL; @@ -1020,11 +1031,11 @@ static ScrDyn *scr_dyn_invoke_impl( } size_t splice_start = 0; size_t splice_removed = 0; - if (dyn_arr_proto_mutates(method)) { - bool grows = dyn_name_is(method, "push") && argc > 0; - grows = grows || (dyn_name_is(method, "unshift") && argc > 0); - bool shrinks = (dyn_name_is(method, "pop") || dyn_name_is(method, "shift")) && len > 0; - if (dyn_name_is(method, "splice")) { + if (dyn_arr_proto_mutates(mid)) { + bool grows = (mid == DYN_M_push) && argc > 0; + grows = grows || ((mid == DYN_M_unshift) && argc > 0); + bool shrinks = ((mid == DYN_M_pop) || (mid == DYN_M_shift)) && len > 0; + if ((mid == DYN_M_splice)) { double startD = dyn_index_arg(args, argc, 0, 0, what); if (scr_exc_pending()) return NULL; splice_start = dyn_rel_index(startD, len); @@ -1061,33 +1072,33 @@ static ScrDyn *scr_dyn_invoke_impl( return NULL; } } - if (recv->v.arr.frozen && (dyn_name_is(method, "push") || - dyn_name_is(method, "pop") || dyn_name_is(method, "shift") || - dyn_name_is(method, "unshift") || dyn_name_is(method, "splice"))) { + if (recv->v.arr.frozen && ((mid == DYN_M_push) || + (mid == DYN_M_pop) || (mid == DYN_M_shift) || + (mid == DYN_M_unshift) || (mid == DYN_M_splice))) { static const char message[] = "Cannot assign to read only property 'length'"; scr_throw_error_msg(SCR_ERR_TYPE, message, sizeof message - 1); return NULL; } - if ((grows && recv->non_extensible) || (shrinks && recv->v.arr.sealed && !dyn_name_is(method, "shift")) || - (recv->v.arr.frozen && ((dyn_name_is(method, "sort") && len > 1) || - (dyn_name_is(method, "reverse") && len > 1)))) { + if ((grows && recv->non_extensible) || (shrinks && recv->v.arr.sealed && !(mid == DYN_M_shift)) || + (recv->v.arr.frozen && (((mid == DYN_M_sort) && len > 1) || + ((mid == DYN_M_reverse) && len > 1)))) { static const char message[] = "Cannot modify a sealed or frozen array"; scr_throw_error_msg(SCR_ERR_TYPE, message, sizeof message - 1); return NULL; } } - if (dyn_name_is(method, "push")) { + if ((mid == DYN_M_push)) { for (size_t i = 0; i < argc; i++) scr_dyn_arr_push(recv, scr_dyn_retain(args[i])); return scr_dyn_new_num((double)recv->v.arr.len); } - if (dyn_name_is(method, "pop")) { + if ((mid == DYN_M_pop)) { if (len == 0) return scr_dyn_retain(scr_dyn_undefined()); ScrDyn *result = scr_dyn_arr_at(recv, (double)(len - 1)); if (!result) return NULL; scr_dyn_release(recv->v.arr.items[--recv->v.arr.len]); return result; } - if (dyn_name_is(method, "shift")) { + if ((mid == DYN_M_shift)) { if (len == 0) return scr_dyn_retain(scr_dyn_undefined()); if (recv->v.arr.sealed) { if (recv->v.arr.frozen) { @@ -1109,7 +1120,7 @@ static ScrDyn *scr_dyn_invoke_impl( recv->v.arr.len = len - 1; return first; /* ownership moves out */ } - if (dyn_name_is(method, "unshift")) { + if ((mid == DYN_M_unshift)) { /* Grow the backing store before shifting the old elements. The * appended placeholders are immortal, so overwriting them does not * abandon retained argument references. */ @@ -1121,7 +1132,7 @@ static ScrDyn *scr_dyn_invoke_impl( } return scr_dyn_new_num((double)recv->v.arr.len); } - if (dyn_name_is(method, "splice")) { + if ((mid == DYN_M_splice)) { size_t start = splice_start; size_t removed = splice_removed; size_t inserted = argc > 2 ? argc - 2 : 0; @@ -1151,7 +1162,7 @@ static ScrDyn *scr_dyn_invoke_impl( recv->v.arr.len = next_len; return out; } - if (dyn_name_is(method, "slice")) { + if ((mid == DYN_M_slice)) { double startD = dyn_index_arg(args, argc, 0, 0, what); if (scr_exc_pending()) return NULL; double endD = dyn_index_arg(args, argc, 1, (double)len, what); @@ -1163,7 +1174,7 @@ static ScrDyn *scr_dyn_invoke_impl( for (size_t i = start; i < end; i++) scr_dyn_arr_push(out, scr_dyn_retain(recv->v.arr.items[i])); return out; } - if (dyn_name_is(method, "at")) { + if ((mid == DYN_M_at)) { double iD = dyn_index_arg(args, argc, 0, 0, what); if (scr_exc_pending()) return NULL; if (!dyn_arr_length_unchanged(recv, len)) return NULL; @@ -1171,17 +1182,17 @@ static ScrDyn *scr_dyn_invoke_impl( if (idx < 0 || idx >= (double)len) return scr_dyn_retain(scr_dyn_undefined()); return scr_dyn_arr_at(recv, idx); } - if (dyn_name_is(method, "indexOf") || dyn_name_is(method, "lastIndexOf") || - dyn_name_is(method, "includes")) { - if (len == 0) return dyn_name_is(method, "includes") ? scr_dyn_new_bool(false) : scr_dyn_new_num(-1); + if ((mid == DYN_M_indexOf) || (mid == DYN_M_lastIndexOf) || + (mid == DYN_M_includes)) { + if (len == 0) return (mid == DYN_M_includes) ? scr_dyn_new_bool(false) : scr_dyn_new_num(-1); ScrDyn *needle = argc > 0 ? args[0] : scr_dyn_undefined(); - double from = dyn_name_is(method, "lastIndexOf") ? (double)len - 1 : 0; + double from = (mid == DYN_M_lastIndexOf) ? (double)len - 1 : 0; if (argc > 1) { from = dyn_index_arg(args, argc, 1, 0, what); if (scr_exc_pending()) return NULL; } if (!dyn_arr_length_unchanged(recv, len)) return NULL; - if (dyn_name_is(method, "lastIndexOf")) { + if ((mid == DYN_M_lastIndexOf)) { double first = from < 0 ? (double)len + from : from; if (first >= (double)len) first = (double)len - 1; for (size_t i = first < 0 ? 0 : (size_t)first + 1; i > 0; i--) { @@ -1193,16 +1204,16 @@ static ScrDyn *scr_dyn_invoke_impl( if (first < 0) first = 0; for (size_t i = first >= (double)len ? len : (size_t)first; i < len; i++) { const ScrDyn *item = recv->v.arr.items[i]; - bool nan_match = dyn_name_is(method, "includes") && + bool nan_match = (mid == DYN_M_includes) && item->kind == SCR_DYN_NUM && needle->kind == SCR_DYN_NUM && item->v.num != item->v.num && needle->v.num != needle->v.num; if (nan_match || scr_dyn_strict_eq(item, needle)) { - return dyn_name_is(method, "includes") ? scr_dyn_new_bool(true) : scr_dyn_new_num((double)i); + return (mid == DYN_M_includes) ? scr_dyn_new_bool(true) : scr_dyn_new_num((double)i); } } - return dyn_name_is(method, "includes") ? scr_dyn_new_bool(false) : scr_dyn_new_num(-1); + return (mid == DYN_M_includes) ? scr_dyn_new_bool(false) : scr_dyn_new_num(-1); } - if (dyn_name_is(method, "findLast") || dyn_name_is(method, "findLastIndex")) { + if ((mid == DYN_M_findLast) || (mid == DYN_M_findLastIndex)) { if (!dyn_cb_check(args, argc)) return NULL; ScrDyn *visible_recv = callback_recv ? callback_recv : recv; for (size_t i = len; i > 0; i--) { @@ -1215,16 +1226,16 @@ static ScrDyn *scr_dyn_invoke_impl( bool matched = scr_dyn_truthy(result); scr_dyn_release(result); if (matched) { - if (dyn_name_is(method, "findLast")) return item; + if ((mid == DYN_M_findLast)) return item; scr_dyn_release(item); return scr_dyn_new_num((double)index); } scr_dyn_release(item); } - return dyn_name_is(method, "findLast") + return (mid == DYN_M_findLast) ? scr_dyn_retain(scr_dyn_undefined()) : scr_dyn_new_num(-1); } - if (dyn_name_is(method, "join")) { + if ((mid == DYN_M_join)) { ScrJsonBuf b; scr_jb_init(&b); ScrStr *separator = argc > 0 && args[0]->kind != SCR_DYN_UNDEF @@ -1250,7 +1261,7 @@ static ScrDyn *scr_dyn_invoke_impl( scr_str_release(joined); return r; } - if (dyn_name_is(method, "concat")) { + if ((mid == DYN_M_concat)) { ScrDyn *out = scr_dyn_new_arr(); for (size_t i = 0; i < len; i++) { if (!dyn_arr_copy_item(out, recv, i, true)) { scr_dyn_release(out); return NULL; } @@ -1271,7 +1282,7 @@ static ScrDyn *scr_dyn_invoke_impl( } return out; } - if (dyn_name_is(method, "flat")) { + if ((mid == DYN_M_flat)) { double depth = dyn_index_arg(args, argc, 0, 1, what); if (scr_exc_pending()) return NULL; if (!dyn_arr_length_unchanged(recv, len)) return NULL; @@ -1283,7 +1294,7 @@ static ScrDyn *scr_dyn_invoke_impl( } return out; } - if (dyn_name_is(method, "reverse")) { + if ((mid == DYN_M_reverse)) { for (size_t i = 0; i < len / 2; i++) { ScrDyn *tmp = recv->v.arr.items[i]; recv->v.arr.items[i] = recv->v.arr.items[len - 1 - i]; @@ -1291,12 +1302,12 @@ static ScrDyn *scr_dyn_invoke_impl( } return scr_dyn_retain(recv); } - if (dyn_name_is(method, "toReversed")) { + if ((mid == DYN_M_toReversed)) { ScrDyn *out = scr_dyn_new_arr(); if (!dyn_arr_reversed_copy(out, recv)) { scr_dyn_release(out); return NULL; } return out; } - if (dyn_name_is(method, "with")) { + if ((mid == DYN_M_with)) { double indexD = dyn_index_arg(args, argc, 0, 0, what); if (scr_exc_pending()) return NULL; if (!dyn_arr_length_unchanged(recv, len)) return NULL; @@ -1314,7 +1325,7 @@ static ScrDyn *scr_dyn_invoke_impl( } return out; } - if (dyn_name_is(method, "toSpliced")) { + if ((mid == DYN_M_toSpliced)) { double startD = dyn_index_arg(args, argc, 0, 0, what); if (scr_exc_pending()) return NULL; size_t start = dyn_rel_index(startD, len); @@ -1331,7 +1342,7 @@ static ScrDyn *scr_dyn_invoke_impl( for (size_t i = start + delete_count; i < len; i++) if (!dyn_arr_copy_item(out, recv, i, false)) { scr_dyn_release(out); return NULL; } return out; } - if (dyn_name_is(method, "fill")) { + if ((mid == DYN_M_fill)) { double startD = dyn_index_arg(args, argc, 1, 0, what); if (scr_exc_pending()) return NULL; double endD = dyn_index_arg(args, argc, 2, (double)len, what); @@ -1359,7 +1370,7 @@ static ScrDyn *scr_dyn_invoke_impl( } return scr_dyn_retain(recv); } - if (dyn_name_is(method, "copyWithin")) { + if ((mid == DYN_M_copyWithin)) { double targetD = dyn_index_arg(args, argc, 0, 0, what); if (scr_exc_pending()) return NULL; double startD = dyn_index_arg(args, argc, 1, 0, what); @@ -1392,13 +1403,13 @@ static ScrDyn *scr_dyn_invoke_impl( } return scr_dyn_retain(recv); } - if (dyn_name_is(method, "forEach") || dyn_name_is(method, "map") || - dyn_name_is(method, "filter") || dyn_name_is(method, "some") || - dyn_name_is(method, "every") || dyn_name_is(method, "find") || - dyn_name_is(method, "findIndex")) { + if ((mid == DYN_M_forEach) || (mid == DYN_M_map) || + (mid == DYN_M_filter) || (mid == DYN_M_some) || + (mid == DYN_M_every) || (mid == DYN_M_find) || + (mid == DYN_M_findIndex)) { if (!dyn_cb_check(args, argc)) return NULL; ScrDyn *cb = args[0]; - ScrDyn *out = (dyn_name_is(method, "map") || dyn_name_is(method, "filter")) ? scr_dyn_new_arr() : NULL; + ScrDyn *out = ((mid == DYN_M_map) || (mid == DYN_M_filter)) ? scr_dyn_new_arr() : NULL; /* Array iteration methods snapshot length before the first callback. * A shrinking callback can remove a later dense entry; an expanding * callback must not make the new tail observable to this pass. Live @@ -1413,30 +1424,30 @@ static ScrDyn *scr_dyn_invoke_impl( ScrDyn *item = scr_dyn_retain(recv->v.arr.items[i]); ScrDyn *r = dyn_call_cb(cb, item, i, visible_recv); if (!r) { scr_dyn_release(item); scr_dyn_release(out); return NULL; } - if (dyn_name_is(method, "map")) { + if ((mid == DYN_M_map)) { scr_dyn_arr_push(out, r); /* ownership moves in */ r = NULL; } else { bool truthy = scr_dyn_truthy(r); scr_dyn_release(r); - if (dyn_name_is(method, "filter") && truthy) scr_dyn_arr_push(out, scr_dyn_retain(item)); - if (dyn_name_is(method, "some") && truthy) { scr_dyn_release(item); return scr_dyn_new_bool(true); } - if (dyn_name_is(method, "every") && !truthy) { scr_dyn_release(item); return scr_dyn_new_bool(false); } - if (dyn_name_is(method, "find") && truthy) return item; /* +1 moves out */ - if (dyn_name_is(method, "findIndex") && truthy) { scr_dyn_release(item); return scr_dyn_new_num((double)i); } + if ((mid == DYN_M_filter) && truthy) scr_dyn_arr_push(out, scr_dyn_retain(item)); + if ((mid == DYN_M_some) && truthy) { scr_dyn_release(item); return scr_dyn_new_bool(true); } + if ((mid == DYN_M_every) && !truthy) { scr_dyn_release(item); return scr_dyn_new_bool(false); } + if ((mid == DYN_M_find) && truthy) return item; /* +1 moves out */ + if ((mid == DYN_M_findIndex) && truthy) { scr_dyn_release(item); return scr_dyn_new_num((double)i); } } scr_dyn_release(item); } if (out) return out; - if (dyn_name_is(method, "some")) return scr_dyn_new_bool(false); - if (dyn_name_is(method, "every")) return scr_dyn_new_bool(true); - if (dyn_name_is(method, "find")) return scr_dyn_retain(scr_dyn_undefined()); - if (dyn_name_is(method, "findIndex")) return scr_dyn_new_num(-1); + if ((mid == DYN_M_some)) return scr_dyn_new_bool(false); + if ((mid == DYN_M_every)) return scr_dyn_new_bool(true); + if ((mid == DYN_M_find)) return scr_dyn_retain(scr_dyn_undefined()); + if ((mid == DYN_M_findIndex)) return scr_dyn_new_num(-1); return scr_dyn_retain(scr_dyn_undefined()); /* forEach */ } - if (dyn_name_is(method, "reduce") || dyn_name_is(method, "reduceRight")) { + if ((mid == DYN_M_reduce) || (mid == DYN_M_reduceRight)) { if (!dyn_cb_check(args, argc)) return NULL; - bool backward = dyn_name_is(method, "reduceRight"); + bool backward = (mid == DYN_M_reduceRight); ScrDyn *acc = argc > 1 ? scr_dyn_retain(args[1]) : NULL; ScrDyn *visible_recv = callback_recv ? callback_recv : recv; for (size_t step = 0; step < len; step++) { @@ -1458,7 +1469,7 @@ static ScrDyn *scr_dyn_invoke_impl( scr_throw_error_msg(SCR_ERR_TYPE, message, sizeof message - 1); return NULL; } - if (dyn_name_is(method, "flatMap")) { + if ((mid == DYN_M_flatMap)) { /* JS Array.prototype.flatMap over a dyn array: map + a depth-1 * flatten. Native dyn-array results flatten element-by-element; a * WRAPPED engine array flattens through the routed keyed reads @@ -1496,7 +1507,7 @@ static ScrDyn *scr_dyn_invoke_impl( } return out; } - if (dyn_name_is(method, "sort")) { + if ((mid == DYN_M_sort)) { ScrDyn *cmp = argc > 0 ? args[0] : scr_dyn_undefined(); if (cmp->kind != SCR_DYN_UNDEF && cmp->kind != SCR_DYN_FUNC) { /* V8 appends the received value's string image. */ @@ -1510,7 +1521,7 @@ static ScrDyn *scr_dyn_invoke_impl( if (len > 1 && !dyn_arr_sort(recv, cmp->kind == SCR_DYN_FUNC ? cmp : NULL)) return NULL; return scr_dyn_retain(recv); } - if (dyn_name_is(method, "toSorted")) { + if ((mid == DYN_M_toSorted)) { ScrDyn *cmp = argc > 0 ? args[0] : scr_dyn_undefined(); if (cmp->kind != SCR_DYN_UNDEF && cmp->kind != SCR_DYN_FUNC) { static const char message[] = "The comparison function must be either a function or undefined"; @@ -1525,7 +1536,7 @@ static ScrDyn *scr_dyn_invoke_impl( } return out; } - if (dyn_arr_proto_unimpl(method)) { + if (dyn_arr_proto_unimpl(mid)) { dyn_throw_unsupported("Array", method); return NULL; } @@ -1541,14 +1552,14 @@ static ScrDyn *scr_dyn_invoke_impl( #ifdef SCR_LIB scr_trap("scriptc: promise dispatch is unavailable in library mode\n"); #else - if (dyn_name_is(method, "then")) { + if ((mid == DYN_M_then)) { return scr_dyn_promise_then(recv->v.promise, argc >= 1 ? args[0] : NULL, argc >= 2 ? args[1] : NULL, NULL); } - if (dyn_name_is(method, "catch")) { + if ((mid == DYN_M_catch)) { return scr_dyn_promise_then(recv->v.promise, NULL, argc >= 1 ? args[0] : NULL, NULL); } - if (dyn_name_is(method, "finally")) { + if ((mid == DYN_M_finally)) { return scr_dyn_promise_then(recv->v.promise, NULL, NULL, argc >= 1 ? args[0] : NULL); } dyn_throw_not_fn(what); @@ -1600,44 +1611,44 @@ static ScrDyn *scr_dyn_invoke_impl( } /* The typed array methods below are not DataView's; it inherits * Object.prototype's. */ - if (dyn_name_is(method, "toString") || dyn_name_is(method, "toLocaleString")) { + if ((mid == DYN_M_toString) || (mid == DYN_M_toLocaleString)) { ScrStr *tag = scr_str_new("[object DataView]", 17); ScrDyn *out = scr_dyn_new_str(tag); scr_str_release(tag); return out; } - if (dyn_name_is(method, "valueOf")) return scr_dyn_retain(recv); + if ((mid == DYN_M_valueOf)) return scr_dyn_retain(recv); dyn_throw_not_fn(what); return NULL; } size_t blen = bytes->len; - if (dyn_name_is(method, "at")) { + if ((mid == DYN_M_at)) { double iD = dyn_bytes_index_arg(args, argc, 0, 0); if (scr_exc_pending()) return NULL; double idx = iD < 0 ? (double)blen + iD : iD; if (idx < 0 || idx >= (double)blen) return scr_dyn_retain(scr_dyn_undefined()); return scr_dyn_new_num(scr_bytes_get(bytes, idx)); } - if (dyn_name_is(method, "slice") || dyn_name_is(method, "subarray")) { + if ((mid == DYN_M_slice) || (mid == DYN_M_subarray)) { /* subarray and Buffer.slice alias their source. TypedArray.slice * owns an independent copy; all keep the receiver's Buffer flavor. */ double startD = dyn_bytes_index_arg(args, argc, 0, 0); if (scr_exc_pending()) return NULL; double endD = dyn_bytes_index_arg(args, argc, 1, (double)blen); if (scr_exc_pending()) return NULL; - ScrBytes *out = recv->buffer || dyn_name_is(method, "subarray") + ScrBytes *out = recv->buffer || (mid == DYN_M_subarray) ? scr_bytes_subarray(bytes, startD, endD) : scr_bytes_slice(bytes, startD, endD); ScrDyn *d = recv->buffer ? scr_dyn_new_buffer(out) : scr_dyn_new_bytes(out); scr_bytes_release(out); return d; } - if (dyn_name_is(method, "set")) { + if ((mid == DYN_M_set)) { double offset = dyn_bytes_index_arg(args, argc, 1, 0); if (scr_exc_pending()) return NULL; scr_bytes_set_from_dyn(bytes, argc ? args[0] : scr_dyn_undefined(), offset); return scr_exc_pending() ? NULL : scr_dyn_retain(scr_dyn_undefined()); } - if (dyn_name_is(method, "copyWithin")) { + if ((mid == DYN_M_copyWithin)) { double target = dyn_bytes_index_arg(args, argc, 0, 0); if (scr_exc_pending()) return NULL; double start = dyn_bytes_index_arg(args, argc, 1, 0); @@ -1647,7 +1658,7 @@ static ScrDyn *scr_dyn_invoke_impl( scr_bytes_release(scr_bytes_copy_within(bytes, target, start, end)); return scr_dyn_retain(recv); } - if (dyn_name_is(method, "fill") && !recv->buffer) { + if ((mid == DYN_M_fill) && !recv->buffer) { double value; if (!scr_dyn_number_coerce_js(argc ? args[0] : scr_dyn_undefined(), &value)) return NULL; double start = dyn_bytes_index_arg(args, argc, 1, 0); @@ -1657,8 +1668,8 @@ static ScrDyn *scr_dyn_invoke_impl( scr_bytes_release(scr_bytes_fill_elem(bytes, value, start, end)); return scr_dyn_retain(recv); } - if (dyn_name_is(method, "join") || (dyn_name_is(method, "toString") && !recv->buffer)) { - ScrStr *separator = dyn_name_is(method, "join") && argc && args[0]->kind != SCR_DYN_UNDEF + if ((mid == DYN_M_join) || ((mid == DYN_M_toString) && !recv->buffer)) { + ScrStr *separator = (mid == DYN_M_join) && argc && args[0]->kind != SCR_DYN_UNDEF ? scr_dyn_string_coerce_js(args[0]) : scr_str_new(",", 1); if (scr_exc_pending()) { scr_str_release(separator); return NULL; } ScrStr *joined = scr_bytes_join(bytes, separator); @@ -1667,7 +1678,7 @@ static ScrDyn *scr_dyn_invoke_impl( scr_str_release(joined); return out; } - if (dyn_bytes_proto_real(method)) { + if (dyn_bytes_proto_real(mid)) { dyn_throw_unsupported(scr_bytes_elem_name(bytes->elem), method); return NULL; } diff --git a/packages/runtime/src/scr_dyn_methods.h b/packages/runtime/src/scr_dyn_methods.h new file mode 100644 index 000000000..49e5c7254 --- /dev/null +++ b/packages/runtime/src/scr_dyn_methods.h @@ -0,0 +1,160 @@ +/* Generated by scripts/gen-dyn-methods.mjs from the DYN_M_ ids scr_dyn_invoke.c + * uses. Do not edit by hand. */ +#ifndef SCR_DYN_METHODS_H +#define SCR_DYN_METHODS_H + +#include +#include + +enum { + DYN_M_NONE, + DYN_M_apply, + DYN_M_at, + DYN_M_bind, + DYN_M_call, + DYN_M_catch, + DYN_M_concat, + DYN_M_copyWithin, + DYN_M_entries, + DYN_M_every, + DYN_M_fill, + DYN_M_filter, + DYN_M_finally, + DYN_M_find, + DYN_M_findIndex, + DYN_M_findLast, + DYN_M_findLastIndex, + DYN_M_flat, + DYN_M_flatMap, + DYN_M_forEach, + DYN_M_includes, + DYN_M_indexOf, + DYN_M_join, + DYN_M_keys, + DYN_M_lastIndexOf, + DYN_M_map, + DYN_M_pop, + DYN_M_push, + DYN_M_reduce, + DYN_M_reduceRight, + DYN_M_reverse, + DYN_M_set, + DYN_M_shift, + DYN_M_slice, + DYN_M_some, + DYN_M_sort, + DYN_M_splice, + DYN_M_subarray, + DYN_M_then, + DYN_M_toExponential, + DYN_M_toFixed, + DYN_M_toLocaleString, + DYN_M_toReversed, + DYN_M_toSorted, + DYN_M_toSpliced, + DYN_M_toString, + DYN_M_unshift, + DYN_M_valueOf, + DYN_M_values, + DYN_M_with, + DYN_M_COUNT +}; + +static const char *const dyn_method_names[DYN_M_COUNT] = { + NULL, + "apply", + "at", + "bind", + "call", + "catch", + "concat", + "copyWithin", + "entries", + "every", + "fill", + "filter", + "finally", + "find", + "findIndex", + "findLast", + "findLastIndex", + "flat", + "flatMap", + "forEach", + "includes", + "indexOf", + "join", + "keys", + "lastIndexOf", + "map", + "pop", + "push", + "reduce", + "reduceRight", + "reverse", + "set", + "shift", + "slice", + "some", + "sort", + "splice", + "subarray", + "then", + "toExponential", + "toFixed", + "toLocaleString", + "toReversed", + "toSorted", + "toSpliced", + "toString", + "unshift", + "valueOf", + "values", + "with", +}; + +static const unsigned char dyn_method_lengths[DYN_M_COUNT] = { + 0, 5, 2, 4, 4, 5, 6, 10, 7, 5, 4, 6, 7, 4, 9, 8, + 13, 4, 7, 7, 8, 7, 4, 4, 11, 3, 3, 4, 6, 11, 7, 3, + 5, 5, 4, 4, 6, 8, 4, 13, 7, 14, 10, 8, 9, 8, 7, 7, + 6, 4, +}; + +/* Each name's id at its slot (or the next free one); 0 ends a probe. */ +static const unsigned char dyn_method_slots[256] = { + 0, 0, 0, 0, 12, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 14, 0, 0, 27, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 28, 3, 0, + 22, 0, 20, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, + 0, 0, 0, 0, 0, 33, 31, 0, 0, 0, 0, 0, 36, 35, 0, 0, + 0, 0, 0, 0, 23, 4, 32, 0, 5, 0, 0, 0, 0, 0, 0, 29, + 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 6, 0, 0, 37, 0, + 0, 40, 0, 0, 0, 0, 0, 0, 43, 0, 0, 45, 0, 0, 0, 44, + 0, 7, 0, 0, 0, 0, 42, 0, 25, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, + 46, 0, 0, 39, 0, 41, 0, 9, 0, 0, 13, 0, 0, 0, 0, 8, + 0, 47, 10, 0, 0, 0, 0, 48, 0, 0, 17, 0, 0, 49, 0, 0, + 0, 0, 0, 19, 0, 0, 11, 0, 0, 0, 0, 18, 0, 0, 0, 0, +}; + +#define DYN_METHOD_MAX_LENGTH 14 + +/* The id of a method name, compared by content; DYN_M_NONE for any other + * name. */ +static inline int dyn_method_of(const char *m) { + size_t length = strlen(m); + if (length == 0 || length > DYN_METHOD_MAX_LENGTH) return DYN_M_NONE; + unsigned slot = (unsigned)(length * 7 + (unsigned char)m[0] * 31 + + (unsigned char)m[length - 1]) & 255; + for (;;) { + unsigned id = dyn_method_slots[slot]; + if (id == 0) return DYN_M_NONE; + if (dyn_method_lengths[id] == length && memcmp(m, dyn_method_names[id], length) == 0) return (int)id; + slot = (slot + 1) & 255; + } +} + +#endif diff --git a/packages/runtime/src/scr_dyn_methods.test.ts b/packages/runtime/src/scr_dyn_methods.test.ts new file mode 100644 index 000000000..77911aa2d --- /dev/null +++ b/packages/runtime/src/scr_dyn_methods.test.ts @@ -0,0 +1,10 @@ +import { execFileSync } from "node:child_process"; +import { join } from "node:path"; +import { expect, test } from "vitest"; + +test("scr_dyn_methods.h lists every method id the checked-dynamic dispatch compares", () => { + const script = join(import.meta.dirname, "../../../scripts/gen-dyn-methods.mjs"); + expect(() => + execFileSync(process.execPath, [script, "--check"], { stdio: "pipe" }), + ).not.toThrow(); +}); diff --git a/packages/runtime/src/scr_worker_events.c b/packages/runtime/src/scr_worker_events.c index 3d10183f4..a23395c91 100644 --- a/packages/runtime/src/scr_worker_events.c +++ b/packages/runtime/src/scr_worker_events.c @@ -354,17 +354,23 @@ static bool scr_worker_transfer_has(const ScrDyn *transfer, const ScrWorkerView return false; } +/* Whether a method name is `name`: first characters compare inline, so + * the dispatch ladders below call strcmp only for names that can match. */ +static inline bool method_is(const char *method, const char *name) { + return method[0] == name[0] && strcmp(method, name) == 0; +} + static ScrDyn *scr_worker_view_invoke(void *ptr, ScrDyn *self, const char *method, ScrDyn *const *args, size_t argc, const char *what) { (void)what; ScrWorkerView *view = ptr; - if (!strcmp(method, "ref") || !strcmp(method, "unref")) { - view->referenced = !strcmp(method, "ref"); + if (method_is(method, "ref") || method_is(method, "unref")) { + view->referenced = method_is(method, "ref"); return scr_dyn_retain(scr_dyn_undefined()); } - if (view->port && !strcmp(method, "hasRef")) + if (view->port && method_is(method, "hasRef")) return scr_dyn_new_bool(view->referenced && (!view->closed || (view->channel && view->closing))); - if (!view->port && !strcmp(method, "terminate")) { + if (!view->port && method_is(method, "terminate")) { if (!view->termination) { view->termination = scr_promise_new(); if (view->closed) scr_promise_fulfill_ref(view->termination, @@ -373,7 +379,7 @@ static ScrDyn *scr_worker_view_invoke(void *ptr, ScrDyn *self, const char *metho } return scr_dyn_new_promise(view->termination); } - if (!strcmp(method, "postMessage")) { + if (method_is(method, "postMessage")) { /* Worker drops messages after exit before validating or cloning. A * closed MessagePort still validates its payload, as Node does. */ if (!view->port && view->closed) return scr_dyn_retain(scr_dyn_undefined()); @@ -405,9 +411,9 @@ static ScrDyn *scr_worker_view_invoke(void *ptr, ScrDyn *self, const char *metho else scr_worker_thread_post(view->thread, message); return scr_dyn_retain(scr_dyn_undefined()); } - bool once = !strcmp(method, "once") || !strcmp(method, "prependOnceListener"); - bool prepend = !strcmp(method, "prependListener") || !strcmp(method, "prependOnceListener"); - if (once || prepend || !strcmp(method, "on") || !strcmp(method, "addListener")) { + bool once = method_is(method, "once") || method_is(method, "prependOnceListener"); + bool prepend = method_is(method, "prependListener") || method_is(method, "prependOnceListener"); + if (once || prepend || method_is(method, "on") || method_is(method, "addListener")) { ScrDyn *listener = argc > 1 ? args[1] : scr_dyn_undefined(); scr_dyn_check_listener(listener, "listener"); if (scr_exc_pending()) return NULL; @@ -416,7 +422,7 @@ static ScrDyn *scr_worker_view_invoke(void *ptr, ScrDyn *self, const char *metho scr_emitter_release(scr_emitter_on_flex(view->events, args[0]->v.str, listener, once, prepend)); return scr_exc_pending() ? NULL : scr_dyn_retain(self); } - if (!strcmp(method, "off") || !strcmp(method, "removeListener")) { + if (method_is(method, "off") || method_is(method, "removeListener")) { ScrDyn *listener = argc > 1 ? args[1] : scr_dyn_undefined(); scr_dyn_check_listener(listener, "listener"); if (scr_exc_pending()) return NULL; @@ -425,21 +431,21 @@ static ScrDyn *scr_worker_view_invoke(void *ptr, ScrDyn *self, const char *metho scr_emitter_release(scr_emitter_off_dyn(view->events, args[0]->v.str, listener)); return scr_exc_pending() ? NULL : scr_dyn_retain(self); } - if (!strcmp(method, "removeAllListeners")) { + if (method_is(method, "removeAllListeners")) { bool all = !argc || args[0]->kind == SCR_DYN_UNDEF; if (!all && args[0]->kind != SCR_DYN_STR) return scr_worker_refusal("Worker event names must be strings"); scr_emitter_release(scr_emitter_remove_all(view->events, all ? NULL : args[0]->v.str, all)); return scr_exc_pending() ? NULL : scr_dyn_retain(self); } - if (!strcmp(method, "listenerCount")) { + if (method_is(method, "listenerCount")) { if (!argc || args[0]->kind != SCR_DYN_STR) return scr_worker_refusal("Worker event names must be strings"); return scr_dyn_new_num(argc > 1 && args[1]->kind != SCR_DYN_UNDEF ? scr_emitter_listener_count_dyn(view->events, args[0]->v.str, args[1]) : scr_emitter_listener_count(view->events, args[0]->v.str)); } - if (view->port && !strcmp(method, "close")) { + if (view->port && method_is(method, "close")) { if (argc && args[0]->kind != SCR_DYN_UNDEF) { scr_dyn_check_listener(args[0], "callback"); if (scr_exc_pending()) return NULL; @@ -460,7 +466,7 @@ static ScrDyn *scr_worker_view_invoke(void *ptr, ScrDyn *self, const char *metho } return scr_dyn_retain(scr_dyn_undefined()); } - if (view->port && !strcmp(method, "start")) { + if (view->port && method_is(method, "start")) { view->started = true; return scr_dyn_retain(scr_dyn_undefined()); } @@ -1021,7 +1027,7 @@ static ScrDyn *scr_broadcast_invoke(void *ptr, ScrDyn *self, const char *method, ScrDyn *const *args, size_t argc, const char *what) { (void)what; ScrWorkerView *view = ptr; - if (!strcmp(method, "postMessage")) { + if (method_is(method, "postMessage")) { if (view->closed) { scr_throw_domex("InvalidStateError", "BroadcastChannel is closed."); return NULL; @@ -1045,7 +1051,7 @@ static ScrDyn *scr_broadcast_invoke(void *ptr, ScrDyn *self, const char *method, scr_message_free(first); return scr_exc_pending() ? NULL : scr_dyn_retain(scr_dyn_undefined()); } - if (!strcmp(method, "close")) { + if (method_is(method, "close")) { if (!view->closed) { view->closed = true; scr_broadcast_leave(view->channel); @@ -1053,8 +1059,8 @@ static ScrDyn *scr_broadcast_invoke(void *ptr, ScrDyn *self, const char *method, } return scr_dyn_retain(scr_dyn_undefined()); } - if (!strcmp(method, "ref") || !strcmp(method, "unref")) { - view->referenced = !strcmp(method, "ref"); + if (method_is(method, "ref") || method_is(method, "unref")) { + view->referenced = method_is(method, "ref"); return scr_dyn_retain(self); } ScrDyn *out = NULL; diff --git a/scripts/gen-dyn-methods.mjs b/scripts/gen-dyn-methods.mjs new file mode 100644 index 000000000..fe4f81a90 --- /dev/null +++ b/scripts/gen-dyn-methods.mjs @@ -0,0 +1,101 @@ +#!/usr/bin/env node +/* Writes packages/runtime/src/scr_dyn_methods.h: the method-name ids the + * checked-dynamic dispatch in scr_dyn_invoke.c compares (DYN_M_), the + * names, and an open-addressed hash table mapping a name to its id. The ids + * are the DYN_M_ names scr_dyn_invoke.c uses, so the header follows the + * dispatch code. + * + * node scripts/gen-dyn-methods.mjs write the header + * node scripts/gen-dyn-methods.mjs --check fail when it is out of date */ +import { readFileSync, writeFileSync } from "node:fs"; + +const source = new URL("../packages/runtime/src/scr_dyn_invoke.c", import.meta.url); +const target = new URL("../packages/runtime/src/scr_dyn_methods.h", import.meta.url); +const SLOTS = 256; + +const used = new Set(); +for (const match of readFileSync(source, "utf8").matchAll(/\bDYN_M_([A-Za-z][A-Za-z0-9]*)\b/g)) + if (match[1] !== "NONE" && match[1] !== "COUNT") used.add(match[1]); +const names = [...used].sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)); +if (names.length >= SLOTS / 2) throw new Error("too many method names for the hash table"); + +// The slot dyn_method_of in the header computes. +const slotOf = (name) => + (name.length * 7 + name.charCodeAt(0) * 31 + name.charCodeAt(name.length - 1)) & (SLOTS - 1); +const slots = new Array(SLOTS).fill(0); +names.forEach((name, i) => { + let slot = slotOf(name); + while (slots[slot] !== 0) slot = (slot + 1) & (SLOTS - 1); + slots[slot] = i + 1; +}); + +const rows = (values, width) => { + const out = []; + for (let i = 0; i < values.length; i += width) + out.push(` ${values.slice(i, i + width).join(", ")},`); + return out; +}; +const text = [ + "/* Generated by scripts/gen-dyn-methods.mjs from the DYN_M_ ids scr_dyn_invoke.c", + " * uses. Do not edit by hand. */", + "#ifndef SCR_DYN_METHODS_H", + "#define SCR_DYN_METHODS_H", + "", + "#include ", + "#include ", + "", + "enum {", + " DYN_M_NONE,", + ...names.map((name) => ` DYN_M_${name},`), + " DYN_M_COUNT", + "};", + "", + "static const char *const dyn_method_names[DYN_M_COUNT] = {", + " NULL,", + ...names.map((name) => ` ${JSON.stringify(name)},`), + "};", + "", + "static const unsigned char dyn_method_lengths[DYN_M_COUNT] = {", + ...rows([0, ...names.map((name) => name.length)], 16), + "};", + "", + `/* Each name's id at its slot (or the next free one); 0 ends a probe. */`, + `static const unsigned char dyn_method_slots[${SLOTS}] = {`, + ...rows(slots, 16), + "};", + "", + `#define DYN_METHOD_MAX_LENGTH ${Math.max(...names.map((name) => name.length))}`, + "", + "/* The id of a method name, compared by content; DYN_M_NONE for any other", + " * name. */", + "static inline int dyn_method_of(const char *m) {", + " size_t length = strlen(m);", + " if (length == 0 || length > DYN_METHOD_MAX_LENGTH) return DYN_M_NONE;", + " unsigned slot = (unsigned)(length * 7 + (unsigned char)m[0] * 31 +", + ` (unsigned char)m[length - 1]) & ${SLOTS - 1};`, + " for (;;) {", + " unsigned id = dyn_method_slots[slot];", + " if (id == 0) return DYN_M_NONE;", + " if (dyn_method_lengths[id] == length && memcmp(m, dyn_method_names[id], length) == 0) return (int)id;", + ` slot = (slot + 1) & ${SLOTS - 1};`, + " }", + "}", + "", + "#endif", + "", +].join("\n"); + +if (process.argv.includes("--check")) { + let current = ""; + try { + current = readFileSync(target, "utf8"); + } catch {} + if (current !== text) { + console.error( + "packages/runtime/src/scr_dyn_methods.h is out of date: run node scripts/gen-dyn-methods.mjs", + ); + process.exit(1); + } +} else { + writeFileSync(target, text); +} diff --git a/scripts/zero-cost-compare.mjs b/scripts/zero-cost-compare.mjs index 3158de997..160e33438 100644 --- a/scripts/zero-cost-compare.mjs +++ b/scripts/zero-cost-compare.mjs @@ -394,7 +394,8 @@ export function dynamicTierSymbols(names) { * not yet traced to its typed root. bench/thread-slots, * bench/channel-messages and bench/worker-messages, added later, link them * through worker messages: - * node:worker_threads values are dyn objects. Every other bench workload is fully typed and must link none. The gate and + * node:worker_threads values are dyn objects. bench/dynamic-methods is + * untyped JavaScript. Every other bench workload is fully typed and must link none. The gate and * tests/harness/bench-dynamic-tier.test.ts enforce that, and the test also * fails when a listed workload stops linking them, so remove a workload here * when a leak fix makes its binary clean. */ @@ -408,6 +409,7 @@ export const DYNAMIC_WORKLOADS = new Set([ "bench/intern-table", "bench/thread-slots", "bench/channel-messages", + "bench/dynamic-methods", "bench/worker-messages", "bench/typed-array-scan-workers", "bench/log-summary", From e5b2e62b45c16f9f87b5f4ca934526e8f7ab050f Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 11:25:30 -0500 Subject: [PATCH 055/197] Emit function bodies on forked workers that merge into one emitter's text On hosts that can fork, the native compiler now emits a large module's function bodies on worker processes (SCRIPTC_EMIT_WORKERS, 4 by default). The function list splits into contiguous ranges, each emitted by a forked child of the emitter, and the ranges merge in function order into the text one emitter gives. A range whose worker fails is emitted in the parent. Modules under 2,000 functions, Node and Windows keep the serial path. A range's emitter records an ordered journal (emit-journal.ts): - every helper it numbers from a counter (the dyn conversions and matchers, message listeners and readers, JSON writers, publication walkers, stream adapters and resolve thunks), recorded when its number is taken, with the entry point and arguments that create it; - the first request of each key of a module-level table (declarations, literals, C strings, unit and absent instances, retype guards, regex and template-strings instances, map descriptors, alias tags) made outside a helper's definition; - unkeyed sites (inline caches, speculative loop sites) taken outside a definition. A speculative loop site's global is now `@sc_spec_bails_N`, named like the other numbered sites. The merging emitter replays each range's journal in order. Replay calls the same entry points, so every helper gets the number and definition the serial emission gives it, and table entries land in the serial order. The range's text is then renamed to the merged numbers in one pass. Sets and flags the ranges set (await callees, function values, RC helper requests) merge after the replay. LlvmTargetOptions.functionChunks runs the same merge in one process, for tests. The fork, exit, wait and lock primitives come from the native host (host.c, process-fork.ts); the native compiler passes them to the emitter as LlvmTargetOptions.forkPrimitives. --- packages/compiler/native/host.c | 60 +++ packages/compiler/native/host.ffi.json | 8 +- packages/compiler/src/backend/llvm/dyn.ts | 91 +++++ .../compiler/src/backend/llvm/emit-journal.ts | 238 ++++++++++++ .../compiler/src/backend/llvm/emit-module.ts | 149 +++++++- .../src/backend/llvm/emit-statements.ts | 3 +- packages/compiler/src/backend/llvm/emitter.ts | 360 +++++++++++++++++- .../src/backend/llvm/expr-callbacks.ts | 79 ++++ .../compiler/src/backend/llvm/expr-calls.ts | 44 ++- .../src/backend/llvm/expr-containers.ts | 5 + .../src/backend/llvm/expr-primitives.ts | 2 + .../src/backend/llvm/expr-stream-bridges.ts | 25 ++ .../src/backend/llvm/expr-stream-callbacks.ts | 42 ++ .../compiler/src/backend/llvm/lib-connect.ts | 7 + .../compiler/src/backend/llvm/lib-datagram.ts | 8 + .../compiler/src/backend/llvm/lib-http2.ts | 5 + .../compiler/src/backend/llvm/lib-lookup.ts | 28 +- packages/compiler/src/backend/llvm/publish.ts | 20 + packages/compiler/src/backend/llvm/shapes.ts | 8 + packages/compiler/src/backend/llvm/walkers.ts | 21 + .../compiler/src/frontend/process-fork.ts | 26 ++ packages/compiler/src/native/cli.ts | 16 + packages/compiler/src/native/prepare.ts | 24 +- .../compiler/test/chunked-emission.test.ts | 69 ++++ 24 files changed, 1302 insertions(+), 36 deletions(-) create mode 100644 packages/compiler/src/backend/llvm/emit-journal.ts create mode 100644 packages/compiler/src/frontend/process-fork.ts create mode 100644 packages/compiler/test/chunked-emission.test.ts diff --git a/packages/compiler/native/host.c b/packages/compiler/native/host.c index 6aa1b2a28..afc645ab3 100644 --- a/packages/compiler/native/host.c +++ b/packages/compiler/native/host.c @@ -56,3 +56,63 @@ void scr_collect_cycles_phase(void); void scriptc_native_collect_cycles(void) { scr_collect_cycles_phase(); } + +/* Forked lowering workers (frontend/process-fork.ts). A worker shares the + * parent's heap copy-on-write, writes its results to a file and leaves with + * _exit, so no buffered output or exit handler of the parent runs twice. + * Checker requests from workers hold an exclusive flock on a file each + * worker opens after the fork, so every worker has its own open file + * description and the lock excludes the other workers. */ +#ifdef _WIN32 +int32_t scriptc_native_fork(void) { return -1; } +void scriptc_native_exit(int32_t code) { ExitProcess((UINT)code); } +int32_t scriptc_native_wait(int32_t pid) { (void)pid; return -1; } +uint8_t scriptc_native_lock_open(const uint8_t *bytes, size_t size) { (void)bytes; (void)size; return 0; } +void scriptc_native_lock(void) {} +void scriptc_native_unlock(void) {} +#else +#include +#include +#include +#include + +int32_t scriptc_native_fork(void) { return (int32_t)fork(); } + +void scriptc_native_exit(int32_t code) { _exit(code); } + +/* The worker's exit status, 128 plus the signal for a worker a signal + * ended, or -1 when waitpid fails. */ +int32_t scriptc_native_wait(int32_t pid) { + int status = 0; + while (waitpid((pid_t)pid, &status, 0) < 0) { + if (errno != EINTR) return -1; + } + if (WIFEXITED(status)) return WEXITSTATUS(status); + if (WIFSIGNALED(status)) return 128 + WTERMSIG(status); + return -1; +} + +static int scriptc_native_lock_fd = -1; + +uint8_t scriptc_native_lock_open(const uint8_t *bytes, size_t size) { + if (size == 0 || memchr(bytes, 0, size) != NULL) return 0; + char *path = malloc(size + 1); + if (!path) return 0; + memcpy(path, bytes, size); + path[size] = 0; + if (scriptc_native_lock_fd >= 0) close(scriptc_native_lock_fd); + scriptc_native_lock_fd = open(path, O_CREAT | O_RDWR | O_CLOEXEC, 0600); + free(path); + return scriptc_native_lock_fd >= 0; +} + +void scriptc_native_lock(void) { + if (scriptc_native_lock_fd < 0) return; + while (flock(scriptc_native_lock_fd, LOCK_EX) < 0 && errno == EINTR) { + } +} + +void scriptc_native_unlock(void) { + if (scriptc_native_lock_fd >= 0) flock(scriptc_native_lock_fd, LOCK_UN); +} +#endif diff --git a/packages/compiler/native/host.ffi.json b/packages/compiler/native/host.ffi.json index d7e0e92aa..2bad2a0ce 100644 --- a/packages/compiler/native/host.ffi.json +++ b/packages/compiler/native/host.ffi.json @@ -2,6 +2,12 @@ "ffi_format": 6, "functions": [ { "name": "compilerNativePrivateDirectory", "symbol": "scriptc_native_private_directory", "params": ["string", "bool"], "returns": "bool" }, - { "name": "compilerNativeCollectCycles", "symbol": "scriptc_native_collect_cycles", "params": [], "returns": "void" } + { "name": "compilerNativeCollectCycles", "symbol": "scriptc_native_collect_cycles", "params": [], "returns": "void" }, + { "name": "compilerNativeFork", "symbol": "scriptc_native_fork", "params": [], "returns": "i32" }, + { "name": "compilerNativeExit", "symbol": "scriptc_native_exit", "params": ["i32"], "returns": "void" }, + { "name": "compilerNativeWait", "symbol": "scriptc_native_wait", "params": ["i32"], "returns": "i32" }, + { "name": "compilerNativeLockOpen", "symbol": "scriptc_native_lock_open", "params": ["string"], "returns": "bool" }, + { "name": "compilerNativeLock", "symbol": "scriptc_native_lock", "params": [], "returns": "void" }, + { "name": "compilerNativeUnlock", "symbol": "scriptc_native_unlock", "params": [], "returns": "void" } ] } diff --git a/packages/compiler/src/backend/llvm/dyn.ts b/packages/compiler/src/backend/llvm/dyn.ts index 5c9d8e60f..594634893 100644 --- a/packages/compiler/src/backend/llvm/dyn.ts +++ b/packages/compiler/src/backend/llvm/dyn.ts @@ -196,6 +196,55 @@ export class LlDyn { constructor(private readonly host: DynHost) {} + /** Journal replay (emit-journal.ts): builds or finds the helper a chunk + * emitter recorded, and answers its name. */ + replayHelper(op: string, args: readonly unknown[]): string { + switch (op) { + case "dyn.collectionDynOps": + return this.collectionDynOps(args[0] as IrType & { kind: "map" | "set" }); + case "dyn.dynMatchHelper": + return this.dynMatchHelper(args[0] as IrType, args[1] as boolean); + case "dyn.jsonSchema": + return this.jsonSchema(args[0] as IrType); + case "dyn.recordNameTable": { + const shapeId = args[0] as string; + return this.recordNameTable(shapeId, this.host.recordsById.get(shapeId)!).names; + } + case "dyn.jsonParseHelper": + return this.jsonParseHelper(args[0] as IrType)!; + case "dyn.dynCheckHelper": + return this.dynCheckHelper(args[0] as IrType, args[1] as boolean); + case "dyn.toDynHelper": + return this.toDynHelper(args[0] as IrType); + case "dyn.promiseDynAdapterHelper": + return this.promiseDynAdapterHelper(args[0] as IrType); + case "dyn.dynFuncThunkHelper": + return this.dynFuncThunkHelper(args[0] as IrType & { kind: "func" }); + case "dyn.dynFuncBoxHelper": + return this.dynFuncBoxHelper(args[0] as IrType & { kind: "func" }); + case "dyn.dynFuncAdapterHelper": + return this.dynFuncAdapterHelper(args[0] as IrType & { kind: "func" }); + case "dyn.dynToStrHelper": + return this.dynToStrHelper(); + case "dyn.dynIterNHelper": + return this.dynIterNHelper(); + case "dyn.dynDestrCheckHelper": + return this.dynDestrCheckHelper(); + case "dyn.dynKeyGetHelper": + return this.dynKeyGetHelper(); + case "dyn.dynComputedKeyGetHelper": + return this.dynComputedKeyGetHelper(); + case "dyn.caughtToDynHelper": + return this.caughtToDynHelper(); + case "dyn.messageListenerHelper": + return this.messageListenerHelper(args[0] as IrType & { kind: "func" }) ?? ""; + case "dyn.messageReaderHelper": + return this.messageReaderHelper(args[0] as IrType); + default: + throw new InternalCompilerError(`llvm emitter bug: no dyn helper replay ${op}`); + } + } + /** The types a dyn matcher or builder for `t` checks structurally: record * fields and index values, array elements and union arms. */ private structuralChildren(t: IrType): readonly IrType[] { @@ -385,6 +434,7 @@ export class LlDyn { if (prior) return prior; const name = `sc_map_dyn_${this.mapDynOps.size}`; this.mapDynOps.set(key, name); + this.host.journalAllocated?.(name, "dyn.collectionDynOps", [t]); const keyType = t.kind === "map" ? t.key : t.elem; const valueType: IrType = t.kind === "map" ? t.value : { kind: "f64" }; const emitSlot = (type: IrType, role: string): void => { @@ -442,6 +492,7 @@ export class LlDyn { `@${name} = internal constant { ptr, ptr, ptr, ptr, ptr, ptr } { ptr ${this.host.cstr(key)}, ptr @${name}_key_box, ptr @${name}_key_unbox, ptr @${this.dynMatchHelper(keyType)}, ptr @${name}_value_box, ptr @${name}_value_unbox }`, "", ); + this.host.journalBuilt?.(); return name; } @@ -840,6 +891,7 @@ export class LlDyn { if (existing) return existing; const name = `sc_dm_${this.dynMatchers.size}`; this.dynMatchers.set(matcherKey, name); + this.host.journalAllocated?.(name, "dyn.dynMatchHelper", [t, preserveRefs]); const recursive = this.isRecursiveShape(t); const wrapped = recursive || this.holdsRecursiveShape(t); const B = new BlockBuilder(); @@ -1200,6 +1252,7 @@ export class LlDyn { ``, ); if (wrapped) this.emitVisitWrapper(name, t, recursive ? "match" : "visit"); + this.host.journalBuilt?.(); return name; } @@ -1293,6 +1346,7 @@ export class LlDyn { const S = host.sizeType; const name = `sc_jsch_${this.jsonSchemas.size}`; this.jsonSchemas.set(key, name); + this.host.journalAllocated?.(name, "dyn.jsonSchema", [t]); let body: string; switch (t.kind) { case "f64": @@ -1364,6 +1418,7 @@ export class LlDyn { `@${name} = internal constant { ${S}, ${S}, ptr, ptr, ptr, ptr, ptr, ptr, ${S} } { ${body} }`, ``, ); + this.host.journalBuilt?.(); return name; } @@ -1453,6 +1508,7 @@ export class LlDyn { const base = `sc_rnames_${this.recordNameTables.size}`; const table = { names: `@${base}`, lens: `@${base}_lens` }; this.recordNameTables.set(shapeId, table); + this.host.journalAllocated?.(base, "dyn.recordNameTable", [shapeId]); const n = shape.fields.length; const names = recordSourceFieldOrder(shape); this.defs.push( @@ -1460,6 +1516,7 @@ export class LlDyn { `@${base}_lens = internal constant [${n} x ${S}] [${names.map((name) => `${S} ${Buffer.byteLength(name, "utf8")}`).join(", ")}]`, ``, ); + this.host.journalBuilt?.(); return table; } @@ -1478,6 +1535,7 @@ export class LlDyn { const host = this.host; const name = `sc_jp_${this.jsonParsers.size}`; this.jsonParsers.set(key, name); + this.host.journalAllocated?.(name, "dyn.jsonParseHelper", [t]); const schema = this.jsonSchema(t); const check = this.dynCheckHelper(t); host.declare(`declare ptr @scr_json_parse_schema(ptr, ptr)`); @@ -1504,6 +1562,7 @@ export class LlDyn { `}`, ``, ); + this.host.journalBuilt?.(); return name; } @@ -1519,6 +1578,7 @@ export class LlDyn { if (existing) return existing; const name = `sc_dc_${this.dynBuilders.size}`; this.dynBuilders.set(builderKey, name); + this.host.journalAllocated?.(name, "dyn.dynCheckHelper", [t, preserveRefs]); const recursive = this.isRecursiveShape(t); const memo = !recursive && this.holdsRecursiveShape(t) && this.valTy(t) === "ptr"; const host = this.host; @@ -1541,6 +1601,7 @@ export class LlDyn { `}`, ``, ); + this.host.journalBuilt?.(); return name; } if (isRefCounted(t) && t.kind !== "dyn") { @@ -2537,6 +2598,7 @@ export class LlDyn { ``, ); if (recursive || memo) this.emitVisitWrapper(name, t, recursive ? "build" : "memo"); + this.host.journalBuilt?.(); return name; } @@ -2551,6 +2613,7 @@ export class LlDyn { if (existing) return existing; const name = `sc_td_${this.toDynFns.size}`; this.toDynFns.set(key, name); + this.host.journalAllocated?.(name, "dyn.toDynHelper", [t]); const host = this.host; const B = new BlockBuilder(); let sourceAccessor: { name: string; release: string } | null = null; @@ -3157,6 +3220,7 @@ export class LlDyn { ``, ); } + this.host.journalBuilt?.(); return name; } @@ -3173,6 +3237,7 @@ export class LlDyn { if (existing) return existing; const name = `sc_pda_${this.promiseDynAdapters.size}`; this.promiseDynAdapters.set(key, name); + this.host.journalAllocated?.(name, "dyn.promiseDynAdapterHelper", [inner]); const host = this.host; const B = new BlockBuilder(); host.declare(`declare void @scr_promise_fulfill_ref(ptr, ptr, ptr, ptr, ptr)`); @@ -3250,6 +3315,7 @@ export class LlDyn { `}`, ``, ); + this.host.journalBuilt?.(); return name; } @@ -3262,6 +3328,7 @@ export class LlDyn { if (this.dynToStrFn) return this.dynToStrFn; const name = "sc_ds"; this.dynToStrFn = name; + this.host.journalAllocated?.(name, "dyn.dynToStrHelper", []); const host = this.host; host.declare(`declare void @scr_jb_init(ptr)`); host.declare(`declare ptr @scr_jb_finish(ptr)`); @@ -3566,6 +3633,7 @@ export class LlDyn { ``, ); } + this.host.journalBuilt?.(); return name; } @@ -3577,6 +3645,7 @@ export class LlDyn { if (this.caughtToDynFn) return this.caughtToDynFn; const name = "sc_cd"; this.caughtToDynFn = name; + this.host.journalAllocated?.(name, "dyn.caughtToDynHelper", []); const host = this.host; const B = new BlockBuilder(); const kp = B.tmp(); @@ -3682,6 +3751,7 @@ export class LlDyn { `}`, ``, ); + this.host.journalBuilt?.(); return name; } @@ -3695,6 +3765,7 @@ export class LlDyn { this.helperMemo.set(name, name); const B = new BlockBuilder(); const host = this.host; + this.host.journalAllocated?.(name, "dyn.dynComputedKeyGetHelper", []); const stringHelper = this.dynKeyGetHelper(); host.declare(`declare ptr @scr_dyn_property_key_value(ptr)`); host.declare(`declare ptr @scr_dyn_symbol_key_get(ptr, ptr, i1 zeroext)`); @@ -3732,6 +3803,7 @@ export class LlDyn { "}", "", ); + this.host.journalBuilt?.(); return name; } @@ -3741,6 +3813,7 @@ export class LlDyn { if (existing) return existing; const name = "sc_dyn_key_get"; this.dynBuilders.set(memoKey, name); + this.host.journalAllocated?.(name, "dyn.dynKeyGetHelper", []); const host = this.host; const B = new BlockBuilder(); const kd = this.kindOf(B, "%d"); @@ -4134,6 +4207,7 @@ export class LlDyn { `}`, ``, ); + this.host.journalBuilt?.(); return name; } @@ -4145,6 +4219,7 @@ export class LlDyn { if (existing) return existing; const name = "sc_dyn_destr_check"; this.dynBuilders.set(memoKey, name); + this.host.journalAllocated?.(name, "dyn.dynDestrCheckHelper", []); const host = this.host; host.declare(`declare void @scr_jb_init(ptr)`); host.declare(`declare ptr @scr_jb_finish(ptr)`); @@ -4199,6 +4274,7 @@ export class LlDyn { `}`, ``, ); + this.host.journalBuilt?.(); return name; } @@ -4210,6 +4286,7 @@ export class LlDyn { if (existing) return existing; const name = "sc_dyn_iter_n"; this.dynBuilders.set(memoKey, name); + this.host.journalAllocated?.(name, "dyn.dynIterNHelper", []); const host = this.host; host.declare(`declare ptr @scr_dyn_new_arr()`); host.declare(`declare void @scr_dyn_arr_push(ptr, ptr)`); @@ -4414,6 +4491,7 @@ export class LlDyn { `}`, ``, ); + this.host.journalBuilt?.(); return name; } @@ -4445,6 +4523,7 @@ export class LlDyn { if (existing) return existing; const name = `sc_dfk_${this.dynFuncThunks.size}`; this.dynFuncThunks.set(key, name); + this.host.journalAllocated?.(name, "dyn.dynFuncThunkHelper", [t]); const host = this.host; const B = new BlockBuilder(); const argNames: string[] = []; @@ -4594,6 +4673,7 @@ export class LlDyn { `}`, ``, ); + this.host.journalBuilt?.(); return name; } @@ -4604,6 +4684,7 @@ export class LlDyn { if (existing) return existing; const name = `sc_dfb_${this.dynFuncBoxes.size}`; this.dynFuncBoxes.set(key, name); + this.host.journalAllocated?.(name, "dyn.dynFuncBoxHelper", [t]); const host = this.host; const thunk = this.dynFuncThunkHelper(t); host.declare(`declare ptr @scr_closure_retain_v(ptr)`); @@ -4618,6 +4699,7 @@ export class LlDyn { `}`, ``, ); + this.host.journalBuilt?.(); return name; } @@ -4640,11 +4722,15 @@ export class LlDyn { (t.ret.kind === "void" || t.ret.kind === "f64" || t.ret.kind === "bool") && this.messageReadable(param, new Set()); if (!supported) { + // A signature without an entry still takes a number. this.messageListeners.set(key, null); + this.host.journalAllocated?.("", "dyn.messageListenerHelper", [t]); + this.host.journalBuilt?.(); return null; } const name = `sc_dml_${this.messageListeners.size}`; this.messageListeners.set(key, name); + this.host.journalAllocated?.(name, "dyn.messageListenerHelper", [t]); const B = new BlockBuilder(); B.entryAllocas.push(`%fail = alloca i1`); B.line(`store i1 false, ptr %fail`); @@ -4676,6 +4762,7 @@ export class LlDyn { `}`, ``, ); + this.host.journalBuilt?.(); return name; } @@ -4737,6 +4824,7 @@ export class LlDyn { if (existing) return existing; const name = `sc_dmr_${this.messageReaders.size}`; this.messageReaders.set(key, name); + this.host.journalAllocated?.(name, "dyn.messageReaderHelper", [t]); const host = this.host; const S = host.sizeType; if (t.kind !== "record") host.declare(`declare i32 @scr_message_kind(ptr, ${S})`); @@ -4909,6 +4997,7 @@ export class LlDyn { `}`, ``, ); + this.host.journalBuilt?.(); return name; } @@ -4920,6 +5009,7 @@ export class LlDyn { if (existing) return existing; const name = `sc_dfa_${this.dynFuncAdapters.size}`; this.dynFuncAdapters.set(key, name); + this.host.journalAllocated?.(name, "dyn.dynFuncAdapterHelper", [t]); const host = this.host; const B = new BlockBuilder(); host.declare(`declare ptr @scr_box_get_ref(ptr)`); @@ -5022,6 +5112,7 @@ export class LlDyn { `}`, ``, ); + this.host.journalBuilt?.(); return name; } } diff --git a/packages/compiler/src/backend/llvm/emit-journal.ts b/packages/compiler/src/backend/llvm/emit-journal.ts new file mode 100644 index 000000000..a778aa8f0 --- /dev/null +++ b/packages/compiler/src/backend/llvm/emit-journal.ts @@ -0,0 +1,238 @@ +/* Chunked function emission (emit-module.ts). An emitter that emits one + * contiguous range of a module's functions records here the helpers and + * sites it numbers from a counter (`sc_td_${size}` and the like). The + * emitter that merges the ranges replays each range's records in order: + * replay calls the same entry point with the same arguments, so every + * helper gets the number and the definition the serial emission gives it, + * and the range's text is renamed to those numbers. */ +import type { LlEmitter } from "./emitter.js"; + +/** One event of a range's emission, in order. */ +export type JournalEntry = + /** A helper or site the range's emitter numbered: the name it gave it + * (a counter prefix and a number), and the replay entry point + * (registerReplay) with its arguments, which are plain IR values. */ + | { + readonly kind: "op"; + readonly name: string; + readonly op: string; + readonly args: readonly unknown[]; + } + /** The first request of a key of a module-level table made outside any + * replayable definition, with the value the range's emitter stored. */ + | { + readonly kind: "entry"; + readonly table: string; + readonly key: string; + readonly value: unknown; + }; + +export class EmitJournal { + readonly entries: JournalEntry[] = []; + /** The replayable definitions being built, innermost last. */ + private readonly open: string[] = []; + + /** The definitions still open (empty between functions). */ + get unclosed(): readonly string[] { + return this.open; + } + /** Keys already recorded per table. */ + private readonly recorded = new Map>(); + + /** A keyed helper, recorded when its number is taken, before anything its + * definition creates in turn: that keeps the serial numbering order. + * Replaying a helper that exists answers its name, so helpers created + * inside another helper's definition are recorded too. Opens the + * definition, which the creating entry point closes with `built`. */ + allocated(name: string, op: string, args: readonly unknown[]): void { + this.entries.push({ kind: "op", name, op, args }); + this.open.push(op); + } + + /** Opens a definition whose helper is not replayed itself (its + * entry point records the request with `used`). */ + opened(op: string): void { + this.open.push(op); + } + + /** A request, made outside any definition, of the helper `name` that an + * entry point answered; recorded once per name. Replay makes the same + * request. */ + used(name: string, op: string, args: readonly unknown[]): void { + if (this.open.length !== 0) return; + if (this.usedNames.has(name)) return; + this.usedNames.add(name); + this.entries.push({ kind: "op", name, op, args }); + } + private readonly usedNames = new Set(); + + /** Closes the definition `allocated` or `opened` opened. */ + built(): void { + if (this.open.pop() === undefined) this.overclosed = true; + } + + /** A `built` closed a definition no `allocated` opened. */ + overclosed = false; + + /** An unkeyed site (one per use). Sites taken while a definition is built + * are taken again when the merging emitter replays that definition. */ + site(name: string, op: string, args: readonly unknown[] = []): void { + if (this.open.length === 0) this.entries.push({ kind: "op", name, op, args }); + } + + /** A request of `key` in a module-level table. Only the first request + * made outside any definition is recorded: requests made while a + * definition is built happen again when it is replayed. */ + entry(table: string, key: string, value: unknown): void { + if (this.open.length !== 0) return; + let keys = this.recorded.get(table); + if (keys === undefined) this.recorded.set(table, (keys = new Set())); + if (keys.has(key)) return; + keys.add(key); + this.entries.push({ kind: "entry", table, key, value }); + } +} + +/** A range emitter's result (LlEmitter.chunkResult): plain data, so a + * worker process can write it. */ +export interface ChunkResult { + readonly entries: readonly JournalEntry[]; + readonly flags: { + readonly usesCompactRefs: boolean; + readonly usesCoroutines: boolean; + readonly usesTimers: boolean; + readonly usesGenSlots: boolean; + readonly needsUnionNone: boolean; + readonly needsNullableAbsent: boolean; + readonly needsOom: boolean; + readonly needsBadTag: boolean; + readonly needsBadKey: boolean; + readonly needsRetainBox: boolean; + }; + readonly publishedNamesSym: string | null; + readonly rcHelpers: readonly string[]; + readonly immortalValues: readonly string[]; + readonly fnValues: readonly string[]; + readonly arraySearchKinds: readonly string[]; + readonly recordCloneShapes: readonly string[]; + readonly awaitCallees: readonly string[]; + readonly adoptCallees: readonly string[]; + readonly directAwaitCallees: readonly string[]; + readonly inlineAwaitCallees: readonly string[]; +} + +type Replay = (emitter: LlEmitter, args: readonly unknown[]) => string; +const replays = new Map(); + +/** Registers the entry point that creates (or finds) a numbered helper or + * site from its recorded arguments and answers its name. */ +export function registerReplay(op: string, replay: Replay): void { + replays.set(op, replay); +} + +/** Replays a range's entries on the merging emitter, in record order, and + * records the renames from the range's names to its own in `names`. */ +export function replayJournal( + emitter: LlEmitter, + entries: readonly JournalEntry[], + names: Map, + tags: Map, +): void { + for (const entry of entries) { + if (entry.kind === "entry") { + emitter.applyJournalEntry(entry.table, entry.key, entry.value, names, tags); + continue; + } + const replay = replays.get(entry.op); + if (replay === undefined) + throw new Error(`llvm emitter bug: no replay for journal entry ${entry.op}`); + const own = stripSigil(replay(emitter, entry.args)); + const local = stripSigil(entry.name); + if (own !== local) names.set(local, own); + } +} + +/** The rename of a range's text: numbered names (`names`, without the + * sigil) and field alias tags (`tags`). One scan for each kind of name: + * a numbered name starts `sc_` after a character that cannot be part of a + * name, and ends at the last digit after its counter prefix. */ +export function renameChunk( + names: ReadonlyMap, + tags: ReadonlyMap, +): (text: string) => string { + const prefixes = new Set(); + for (const name of names.keys()) prefixes.add(name.replace(/\d+$/, "")); + if (prefixes.size === 0 && tags.size === 0) return (text) => text; + return (text) => { + let out = prefixes.size === 0 ? text : renameNames(text, names, prefixes); + if (tags.size > 0) out = renameTags(out, tags); + return out; + }; +} + +function renameNames( + text: string, + names: ReadonlyMap, + prefixes: ReadonlySet, +): string { + const parts: string[] = []; + let copied = 0; + for (let at = text.indexOf("sc_"); at >= 0; at = text.indexOf("sc_", at + 3)) { + if (at > 0 && isNameChar(text.charCodeAt(at - 1))) continue; + // The counter prefix ends where the digits start; the number ends at + // the last digit. + let end = at + 3; + while (end < text.length && isNameChar(text.charCodeAt(end)) && !isDigit(text.charCodeAt(end))) + end++; + if (end === text.length || !isDigit(text.charCodeAt(end))) continue; + if (!prefixes.has(text.slice(at, end))) continue; + let last = end; + while (last < text.length && isDigit(text.charCodeAt(last))) last++; + const own = names.get(text.slice(at, last)); + if (own === undefined) continue; + parts.push(text.slice(copied, at), own); + copied = last; + } + if (copied === 0) return text; + parts.push(text.slice(copied)); + return parts.join(""); +} + +function renameTags(text: string, tags: ReadonlyMap): string { + const marker = "!tbaa !"; + const parts: string[] = []; + let copied = 0; + for (let at = text.indexOf(marker); at >= 0; at = text.indexOf(marker, at + marker.length)) { + const start = at + marker.length; + let end = start; + while (end < text.length && isDigit(text.charCodeAt(end))) end++; + const own = tags.get(Number(text.slice(start, end))); + if (own === undefined) continue; + parts.push(text.slice(copied, start), String(own)); + copied = end; + } + if (copied === 0) return text; + parts.push(text.slice(copied)); + return parts.join(""); +} + +/** `[-$._A-Za-z0-9]`: a character of an LLVM identifier. */ +function isNameChar(code: number): boolean { + return ( + (code >= 48 && code <= 57) || + (code >= 65 && code <= 90) || + (code >= 97 && code <= 122) || + code === 95 || + code === 36 || + code === 45 || + code === 46 + ); +} + +function isDigit(code: number): boolean { + return code >= 48 && code <= 57; +} + +function stripSigil(name: string): string { + return name.startsWith("@") ? name.slice(1) : name; +} diff --git a/packages/compiler/src/backend/llvm/emit-module.ts b/packages/compiler/src/backend/llvm/emit-module.ts index 3d9d41072..75e6ad50f 100644 --- a/packages/compiler/src/backend/llvm/emit-module.ts +++ b/packages/compiler/src/backend/llvm/emit-module.ts @@ -2,6 +2,10 @@ import { relieveHeapPressure } from "../../heap-pressure.js"; import { emitWasmModules } from "./wasm-modules.js"; import { type LlEmitter, ffiCallbackDummyLl, llStrBytes } from "./emitter.js"; +import { type ChunkResult, EmitJournal } from "./emit-journal.js"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { JS_IC_WORDS } from "./js-ic.js"; import { mangleAsyncSpawn, @@ -59,16 +63,147 @@ import { STRING_READ_HELPERS, stringScanDecl } from "./string-reads.js"; import { emitLibraryIdentityLines } from "../library-identity-markers.js"; import { InternalCompilerError } from "../../errors.js"; +/** The function bodies emitted in `chunks` contiguous ranges, each by its + * own emitter (LlEmitter.chunkEmitter), then merged in function order: the + * text one emitter gives. */ +function emitFunctionsInChunks(emitter: LlEmitter, chunks: number): string[] { + const functions = emitter.mod.functions; + const size = Math.ceil(functions.length / chunks); + const ranges: { chunk: LlEmitter; texts: string[] }[] = []; + for (let start = 0; start < functions.length; start += size) { + const chunk = emitter.chunkEmitter(); + const texts = functions.slice(start, start + size).map((fn) => { + const text = chunk.emitFunction(fn); + if (chunk.journal!.overclosed) + throw new Error(`llvm emitter bug: journal definition closed twice in ${fn.name}`); + if (chunk.journal!.unclosed.length > 0) + throw new Error( + `llvm emitter bug: unclosed journal definitions after ${fn.name}: ${chunk.journal!.unclosed.join(", ")}`, + ); + return text; + }); + ranges.push({ chunk, texts }); + } + const fnDefs: string[] = []; + for (const { chunk, texts } of ranges) { + // The plain-data result a worker process would write, round-tripped. + const result = JSON.parse(JSON.stringify(chunk.chunkResult())) as ChunkResult; + const rename = emitter.mergeChunk(result); + for (const text of texts) { + const def = rename(text); + emitter.fnDefText.set(functions[fnDefs.length]!.name, def); + fnDefs.push(def); + } + } + return fnDefs; +} + +/** Modules with fewer functions emit in one process: forking and merging + * would cost more than the workers save (SCRIPTC_EMIT_WORKER_FUNCTIONS + * lowers it, for tests). */ +function workerMinFunctions(): number { + const value = Number(process.env["SCRIPTC_EMIT_WORKER_FUNCTIONS"] ?? "2000"); + return Number.isFinite(value) && value > 0 ? value : 2000; +} + +/** The function bodies emitted on forked worker processes, `workers` at a + * time: the function list splits into contiguous ranges, each emitted in + * its own process, which inherits this emitter as it is at the fork and + * records a journal (emit-journal.ts). Ranges merge in function order, so + * the text is the serial text. A range whose worker fails is emitted here + * when its turn comes. Null without fork primitives (Node, Windows, inside + * a worker) or when the module is small. */ +function emitFunctionsOnWorkers(emitter: LlEmitter, workers: number): string[] | null { + const fork = emitter.targetOptions.forkPrimitives; + const functions = emitter.mod.functions; + if (fork === undefined || workers < 2) return null; + if (functions.length < workerMinFunctions()) return null; + // More ranges than workers: a range's merge overlaps the later ranges' + // emission, and a slow range holds back less. + const count = Math.min(functions.length, workers * 3); + const size = Math.ceil(functions.length / count); + const dir = mkdtempSync(join(tmpdir(), "scriptc-emit-")); + const pids: number[] = []; + const start = (range: number): void => { + const pid = fork.fork(); + if (pid === 0) { + let code = 1; + try { + emitter.journal = new EmitJournal(); + const texts: string[] = []; + for (const fn of functions.slice(range * size, (range + 1) * size)) { + texts.push(emitter.emitFunction(fn)); + relieveHeapPressure(); + } + writeFileSync(join(dir, `${range}.json`), JSON.stringify(emitter.chunkResult())); + // Function text never holds NUL, so it separates the functions. + writeFileSync(join(dir, `${range}.ll`), texts.join("\0")); + code = 0; + } catch { + // The parent emits the range itself. + } + fork.exit(code); + } + pids.push(pid); + }; + const fnDefs: string[] = []; + try { + for (let range = 0; range < Math.min(workers, count); range++) start(range); + for (let range = 0; range < count; range++) { + const pid = pids[range]!; + const status = pid > 0 ? fork.wait(pid) : -1; + // The next worker starts from this emitter as merged so far, which + // the journal allows: it records requests, not table contents. + const merged = status === 0 ? readRange(dir, range) : null; + const slice = functions.slice(range * size, (range + 1) * size); + if (merged !== null && merged.texts.length === slice.length) { + const rename = emitter.mergeChunk(merged.result); + slice.forEach((fn, index) => { + const def = rename(merged.texts[index]!); + emitter.fnDefText.set(fn.name, def); + fnDefs.push(def); + }); + } else + for (const fn of slice) { + const def = emitter.emitFunction(fn); + emitter.fnDefText.set(fn.name, def); + fnDefs.push(def); + relieveHeapPressure(); + } + if (pids.length < count) start(pids.length); + } + } finally { + rmSync(dir, { recursive: true, force: true }); + } + return fnDefs; +} + +/** A worker's range result, or null when its files are missing. */ +function readRange(dir: string, range: number): { result: ChunkResult; texts: string[] } | null { + try { + const result = JSON.parse(readFileSync(join(dir, `${range}.json`), "utf8")) as ChunkResult; + const texts = readFileSync(join(dir, `${range}.ll`), "utf8").split("\0"); + return { result, texts }; + } catch { + return null; + } +} + export function emitModuleParts(emitter: LlEmitter): string[] { // Function bodies first (the literal/unit/fn-value tables fill as they // emit), then the file assembles around them — the runtime ABI’s order. - const fnDefs: string[] = []; - for (const fn of emitter.mod.functions) { - const def = emitter.emitFunction(fn); - fnDefs.push(def); - emitter.fnDefText.set(fn.name, def); - relieveHeapPressure(); - } + const chunks = emitter.targetOptions.functionChunks ?? 1; + const fnDefs = + (chunks > 1 ? emitFunctionsInChunks(emitter, chunks) : null) ?? + emitFunctionsOnWorkers(emitter, emitter.targetOptions.functionWorkers ?? 1) ?? + []; + if (fnDefs.length === 0) + for (const fn of emitter.mod.functions) { + const def = emitter.emitFunction(fn); + fnDefs.push(def); + emitter.fnDefText.set(fn.name, def); + relieveHeapPressure(); + } emitter.selectStacklessAsync(fnDefs); const errorViews: string[] = []; for (const property of ["name", "message"]) diff --git a/packages/compiler/src/backend/llvm/emit-statements.ts b/packages/compiler/src/backend/llvm/emit-statements.ts index dc91da2fb..e3998e782 100644 --- a/packages/compiler/src/backend/llvm/emit-statements.ts +++ b/packages/compiler/src/backend/llvm/emit-statements.ts @@ -2095,8 +2095,7 @@ export function emitLoopCopies( // count. After SPECULATION_BAILS bailing entries in a row, the site runs // the ordinary copy only. The count is a hint: both copies give the same // results, and a race between workers loses at most an update. - const bails = `@scr.spec.bails.${emitter.speculativeLoopSites++}`; - emitter.declare(`${bails} = internal global i8 0`); + const bails = emitter.speculativeLoopSite(); const count = B.tmp(), trying = B.tmp(); B.line(`${count} = load atomic i8, ptr ${bails} unordered, align 1`); diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index bfde59a30..1ca9313b7 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -71,6 +71,13 @@ import { import { scalarizeNumericRecords } from "../../ir/scalar-records.js"; import { addRecordResultEntries } from "./record-results.js"; import { specializeCallbackCalls } from "../../ir/callback-specialization.js"; +import { + type ChunkResult, + EmitJournal, + registerReplay, + renameChunk, + replayJournal, +} from "./emit-journal.js"; import { specializeNumericCalls } from "../../ir/numeric-call-specialization.js"; import { everyExpr, everyExprChild, everyStmtChild, everyStmtList } from "../../ir/traverse.js"; import { INT32_RANGE, type IntegerRange, type IntegerRanges } from "../../ir/integer-ranges.js"; @@ -392,9 +399,30 @@ export interface LlScopeEntry { lazy?: boolean; } +/** Process primitives for forked emission workers. */ +export interface ForkPrimitives { + /** 0 in the worker, the worker's pid in the parent, -1 on failure. */ + fork(): number; + /** Ends the worker without running exit handlers. */ + exit(code: number): void; + /** The worker's exit status, 128 plus a signal number, or -1. */ + wait(pid: number): number; +} + export interface LlvmTargetOptions { /** C ABI target; omitted direct emission uses the host ABI. */ targetTriple?: string; + /** Emit the function bodies in this many contiguous ranges, each by its + * own emitter, and merge them in function order (emit-journal.ts). The + * output is the same as one emitter's. */ + functionChunks?: number; + /** Emit the function bodies of a large module on this many forked worker + * processes at a time (emit-module.ts), forked with `forkPrimitives`. The + * output is the same as one emitter's. */ + functionWorkers?: number; + /** The host's process primitives, where it can fork (the native + * compiler's). */ + forkPrimitives?: ForkPrimitives | undefined; /** Exact frontend sources for dev-build locations and variable metadata. */ debugSources?: ReadonlyMap; /** Pointer width of the target C ABI. Native targets are 64-bit today. */ @@ -660,6 +688,7 @@ export class LlEmitter { tag = 2 + this.fieldAliasTags.size * 2; this.fieldAliasTags.set(field, tag); } + this.journal?.entry("tag", field, tag); this.fieldPointerTags.set(ptr, tag); } @@ -672,6 +701,7 @@ export class LlEmitter { tag = 2 + this.fieldAliasTags.size * 2; this.fieldAliasTags.set(category, tag); } + this.journal?.entry("tag", category, tag); return `, !tbaa !${tag}`; } @@ -1223,13 +1253,27 @@ export class LlEmitter { readonly chainSlots = new Map(); logArgSlots = 0; readonly stackTraces: boolean; + /** The options this emitter was built with (chunk emitters reuse them). */ + readonly targetOptions: LlvmTargetOptions; + /** Records the module-level effects of emitting a range of functions, + * which the emitter that merges the range replays (emit-journal.ts). Null + * when this emitter emits the whole module itself. */ + journal: EmitJournal | null; + /** The emitter's hosts forward to `journal` (journal set, or workers). */ + private readonly journaled: boolean; /** Stack-trace mode: functions no capture can include (no frame push). */ readonly unframedFunctions: ReadonlySet; constructor( readonly mod: IrModule, options: LlvmTargetOptions, + journal: EmitJournal | null = null, ) { + this.targetOptions = options; + this.journal = journal; + this.journaled = + journal !== null || + ((options.functionWorkers ?? 1) > 1 && options.forkPrimitives !== undefined); this.stackTraces = observesStacks(mod); this.nullableUnions = new NullableUnions(mod, options.debugSources === undefined); this.mockTimers = this.runtimeFeatures.mockTimers; @@ -1476,6 +1520,20 @@ export class LlEmitter { } } this.shapeHost = { + // Journal hooks only on emitters that can emit ranges (chunk + // emitters, and emitters whose workers take a journal after the + // fork), so the host's optional calls cost nothing otherwise. + journalAllocated: !this.journaled + ? undefined + : (name, op, args) => this.journal?.allocated(name, op, args), + journalBuilt: !this.journaled ? undefined : () => this.journal?.built(), + journalOpened: !this.journaled ? undefined : (op) => this.journal?.opened(op), + journalUsed: !this.journaled + ? undefined + : (name, op, args) => this.journal?.used(name, op, args), + journalEntry: !this.journaled + ? undefined + : (table, key, value) => this.journal?.entry(table, key, value), sizeType: this.sizeType, cycleBufOffset: this.cycleBufOffset, tracedShapes: this.tracedShapes, @@ -2348,6 +2406,7 @@ export class LlEmitter { }; this.literals.set(text, lit); } + this.journal?.entry("literal", text, lit); const name = `@${lit.sym}`; this.immortalValues.add(name); return name; @@ -2358,6 +2417,7 @@ export class LlEmitter { literalEquals(text: string): string { const literal = this.internLiteral(text); const name = `@sc_streq_${literal.slice("@sc_lit_".length)}`; + this.journal?.entry("literalEquality", text, name); if (!this.literalEqualities.has(text)) { this.literalEqualities.set(text, name); if (Buffer.byteLength(text, "utf8") > INLINE_LITERAL_BYTES) { @@ -2384,6 +2444,7 @@ export class LlEmitter { lit = { sym: this.contentSymbol("sc_bigint_", key), value }; this.bigintLiterals.set(key, lit); } + this.journal?.entry("bigint", key, { sym: lit.sym, value: lit.value.toString() }); const name = `@${lit.sym}`; this.immortalValues.add(name); return name; @@ -2397,6 +2458,7 @@ export class LlEmitter { c = { sym: this.contentSymbol("sc_cs_", text), len: Buffer.byteLength(text, "utf8") }; this.cstrs.set(text, c); } + this.journal?.entry("cstr", text, c); return `@${c.sym}`; } @@ -2413,6 +2475,7 @@ export class LlEmitter { sym = this.contentSymbol("sc_map_ops_", body); this.mapOpsConsts.set(body, sym); } + this.journal?.entry("mapOps", body, sym); return `@${sym}`; } @@ -2439,6 +2502,7 @@ export class LlEmitter { sym = `sc_unit_${this.unitInstances.size}`; this.unitInstances.set(key, sym); } + this.journal?.entry("unit", key, sym); const name = `@${sym}`; this.immortalValues.add(name); return name; @@ -2452,6 +2516,7 @@ export class LlEmitter { sym = `sc_retype_guard_${this.retypeGuards.size}`; this.retypeGuards.set(unionId, sym); } + this.journal?.entry("retype", unionId, sym); return `@${sym}`; } @@ -2473,6 +2538,7 @@ export class LlEmitter { sym = `sc_absent_${this.absentInstances.size}`; this.absentInstances.set(unionId, sym); } + this.journal?.entry("absent", unionId, sym); const name = `@${sym}`; this.immortalValues.add(name); return name; @@ -2520,6 +2586,7 @@ export class LlEmitter { } declare(decl: string): void { + this.journal?.entry("decl", decl, null); this.decls.add(decl); } @@ -3640,6 +3707,7 @@ export class LlEmitter { const meta = this.classMetaOf(className); this.classObjs.set(className, { nameSym: this.internLiteral(classJsName(meta.def)) }); } + this.journal?.entry("classObj", className, this.classObjs.get(className)); return mangleClassObj(className); } @@ -4347,6 +4415,191 @@ export class LlEmitter { return emitFunction(this, fn); } + /** What this range emitter (chunkEmitter) hands the merging emitter, as + * plain data a worker process can write: its journal and the sets and + * flags its functions set. */ + chunkResult(): ChunkResult { + return { + entries: this.journal!.entries, + flags: { + usesCompactRefs: this.usesCompactRefs, + usesCoroutines: this.usesCoroutines, + usesTimers: this.usesTimers, + usesGenSlots: this.usesGenSlots, + needsUnionNone: this.needsUnionNone, + needsNullableAbsent: this.needsNullableAbsent, + needsOom: this.needsOom, + needsBadTag: this.needsBadTag, + needsBadKey: this.needsBadKey, + needsRetainBox: this.needsRetainBox, + }, + publishedNamesSym: this.publishedNamesSym, + rcHelpers: this.rcHelpers === null ? [] : [...this.rcHelpers], + immortalValues: [...this.immortalValues], + fnValues: [...this.fnValues], + arraySearchKinds: [...this.arraySearchKinds], + recordCloneShapes: [...this.recordCloneShapes], + awaitCallees: [...this.awaitCallees], + adoptCallees: [...this.adoptCallees], + directAwaitCallees: [...this.directAwaitCallees], + inlineAwaitCallees: [...this.inlineAwaitCallees], + }; + } + + /** Merges a range emitter's result (chunkResult) into this emitter, in + * the range's order, and answers the rename of the range's text to this + * emitter's numbered helpers. Ranges merge in function order. */ + mergeChunk(chunk: ChunkResult): (text: string) => string { + const names = new Map(); + const tags = new Map(); + // The range's helpers and table entries first, in its order; then the + // sets and flags, which replayed definitions must find as serial + // emission left them (a helper set guards the declarations its first + // request adds). + replayJournal(this, chunk.entries, names, tags); + const flags = chunk.flags; + this.usesCompactRefs ||= flags.usesCompactRefs; + this.usesCoroutines ||= flags.usesCoroutines; + this.usesTimers ||= flags.usesTimers; + this.usesGenSlots ||= flags.usesGenSlots; + this.needsUnionNone ||= flags.needsUnionNone; + this.needsNullableAbsent ||= flags.needsNullableAbsent; + this.needsOom ||= flags.needsOom; + this.needsBadTag ||= flags.needsBadTag; + this.needsBadKey ||= flags.needsBadKey; + this.needsRetainBox ||= flags.needsRetainBox; + this.publishedNamesSym ??= chunk.publishedNamesSym; + if (this.rcHelpers !== null) for (const key of chunk.rcHelpers) this.rcHelpers.add(key); + for (const name of chunk.immortalValues) this.immortalValues.add(name); + for (const name of chunk.fnValues) this.fnValues.add(name); + for (const kind of chunk.arraySearchKinds) this.arraySearchKinds.add(kind as ArraySearchKind); + for (const shapeId of chunk.recordCloneShapes) this.recordCloneShapes.add(shapeId); + for (const name of chunk.awaitCallees) this.awaitCallees.add(name); + for (const name of chunk.adoptCallees) this.adoptCallees.add(name); + for (const name of chunk.directAwaitCallees) this.directAwaitCallees.add(name); + for (const name of chunk.inlineAwaitCallees) this.inlineAwaitCallees.add(name); + return renameChunk(names, tags); + } + + /** Adds a content-named constant a chunk interned. Its symbol is this + * emitter's own: a digest collision retries in first-use order, which a + * chunk alone does not know. */ + private addContentEntry( + into: Map, + key: string, + entry: V, + prefix: string, + names: Map, + ): void { + let own = into.get(key); + if (own === undefined) { + own = { ...entry, sym: this.contentSymbol(prefix, key) }; + into.set(key, own); + } + if (own.sym !== entry.sym) names.set(entry.sym, own.sym); + } + + /** Applies a chunk's table entry (EmitJournal.entry) at its place in the + * chunk's order: a key this emitter lacks is added, numbered after its + * own entries where the table numbers them, and the chunk's name for it + * is renamed to this emitter's. */ + applyJournalEntry( + table: string, + key: string, + value: unknown, + names: Map, + tags: Map, + ): void { + switch (table) { + case "decl": + this.decls.add(key); + return; + case "literal": + this.addContentEntry( + this.literals, + key, + value as { sym: string; len: number }, + "sc_lit_", + names, + ); + return; + case "literalEquality": + addEntry(this.literalEqualities, key, value as string); + return; + case "bigint": + { + const entry = value as { sym: string; value: string }; + this.addContentEntry( + this.bigintLiterals, + key, + { sym: entry.sym, value: BigInt(entry.value) }, + "sc_bigint_", + names, + ); + } + return; + case "cstr": + this.addContentEntry( + this.cstrs, + key, + value as { sym: string; len: number }, + "sc_cs_", + names, + ); + return; + case "mapOps": { + let own = this.mapOpsConsts.get(key); + if (own === undefined) { + own = this.contentSymbol("sc_map_ops_", key); + this.mapOpsConsts.set(key, own); + } + if (own !== value) names.set(value as string, own); + return; + } + case "classObj": + addEntry(this.classObjs, key, value as { nameSym: string }); + return; + case "retainWrapper": + addEntry(this.nullableUnions.retainWrappers, key, value as string); + return; + case "unit": + return addNumbered(this.unitInstances, key, value as string, "sc_unit_", names); + case "absent": + return addNumbered(this.absentInstances, key, value as string, "sc_absent_", names); + case "retype": + return addNumbered(this.retypeGuards, key, value as string, "sc_retype_guard_", names); + case "slots": + return addNumbered(this.threadSlotsBlocks, key, value as string, "@sc_slots_", names); + case "regex": + return addNumberedSymbol(this.regexInstances, key, value as never, "sc_re_", names); + case "templateStrings": + return addNumberedSymbol( + this.templateStringsInstances, + key, + value as never, + "sc_tsa_", + names, + ); + case "tag": { + let own = this.fieldAliasTags.get(key); + if (own === undefined) { + own = 2 + this.fieldAliasTags.size * 2; + this.fieldAliasTags.set(key, own); + } + if (own !== value) tags.set(value as number, own); + return; + } + default: + throw new InternalCompilerError(`llvm emitter bug: no journal table ${table}`); + } + } + + /** An emitter for a range of this module's functions: the same facts, + * empty module-level tables, and a journal of what the range adds. */ + chunkEmitter(): LlEmitter { + return new LlEmitter(this.mod, { ...this.targetOptions, functionChunks: 1 }, new EmitJournal()); + } + /** The double-ABI entry of a function with an i32 entry: every caller * that is not a direct call (closures, dispatch, adapters, the runtime) * enters here. A pending exception passes through to its caller. */ @@ -5429,6 +5682,7 @@ export class LlEmitter { sym = `@sc_slots_${this.threadSlotsBlocks.size}`; this.threadSlotsBlocks.set(name, sym); } + this.journal?.entry("slots", name, sym); return sym; } @@ -5578,6 +5832,20 @@ export class LlEmitter { if (this.publishWalkersCache !== null) return this.publishWalkersCache; const byPre = [...this.classMeta.values()].sort((a, b) => a.pre - b.pre); this.publishWalkersCache = new PublishWalkers({ + // Journal hooks only on emitters that can emit ranges (chunk + // emitters, and emitters whose workers take a journal after the + // fork), so the host's optional calls cost nothing otherwise. + journalAllocated: !this.journaled + ? undefined + : (name, op, args) => this.journal?.allocated(name, op, args), + journalBuilt: !this.journaled ? undefined : () => this.journal?.built(), + journalOpened: !this.journaled ? undefined : (op) => this.journal?.opened(op), + journalUsed: !this.journaled + ? undefined + : (name, op, args) => this.journal?.used(name, op, args), + journalEntry: !this.journaled + ? undefined + : (table, key, value) => this.journal?.entry(table, key, value), declare: (decl) => this.declare(decl), cstr: (text) => this.cstr(text), sizeType: this.sizeType, @@ -6064,12 +6332,27 @@ export class LlEmitter { } jsIcSite(): string { - return `@sc_js_ic_${this.jsIcSiteCount++}`; + const name = `@sc_js_ic_${this.jsIcSiteCount++}`; + this.journal?.site(name, "jsIcSite"); + return name; } /** A fresh read-site cache for one constant-key dyn read. */ dynReadIcSite(): string { - return `@sc_dyn_read_ic_${this.dynReadIcSites++}`; + const name = `@sc_dyn_read_ic_${this.dynReadIcSites++}`; + this.journal?.site(name, "dynReadIcSite"); + return name; + } + + /** A fresh speculative loop site: the global that counts its bailing + * entries in a row. */ + speculativeLoopSite(): string { + const name = `@sc_spec_bails_${this.speculativeLoopSites++}`; + this.journal?.site(name, "speculativeLoopSite"); + this.journal?.opened("speculativeLoopSite"); + this.declare(`${name} = internal global i8 0`); + this.journal?.built(); + return name; } dynKind(d: string): string { @@ -6471,3 +6754,76 @@ export function adoptingAwaitOf(value: IrExpr | null): ExprOf<"awaitExpr"> | nul }); return found; } + +/** Adds `key` to `into` when it lacks it. */ +function addEntry(into: Map, key: string, value: V): void { + if (!into.has(key)) into.set(key, value); +} + +/** Adds `key` to a numbered table when it lacks it, numbering it after the + * table's own entries (`prefix` and its index), and records the rename of + * the chunk's `name` for it. */ +function addNumbered( + into: Map, + key: string, + name: string, + prefix: string, + names: Map, +): void { + let own = into.get(key); + if (own === undefined) { + own = `${prefix}${into.size}`; + into.set(key, own); + } + if (own !== name) names.set(name.replace(/^@/, ""), own.replace(/^@/, "")); +} + +/** addNumbered for entries that keep their numbered name in `sym`. */ +function addNumberedSymbol( + into: Map, + key: string, + entry: V, + prefix: string, + names: Map, +): void { + let own = into.get(key); + if (own === undefined) { + own = { ...entry, sym: `${prefix}${into.size}` }; + into.set(key, own); + } + if (own.sym !== entry.sym) names.set(entry.sym, own.sym); +} + +registerReplay("jsIcSite", (emitter) => emitter.jsIcSite()); +registerReplay("dynReadIcSite", (emitter) => emitter.dynReadIcSite()); +registerReplay("speculativeLoopSite", (emitter) => emitter.speculativeLoopSite()); +for (const op of [ + "dyn.collectionDynOps", + "dyn.dynMatchHelper", + "dyn.jsonSchema", + "dyn.recordNameTable", + "dyn.jsonParseHelper", + "dyn.dynCheckHelper", + "dyn.toDynHelper", + "dyn.promiseDynAdapterHelper", + "dyn.dynFuncThunkHelper", + "dyn.dynFuncBoxHelper", + "dyn.dynFuncAdapterHelper", + "dyn.dynToStrHelper", + "dyn.dynIterNHelper", + "dyn.dynDestrCheckHelper", + "dyn.dynKeyGetHelper", + "dyn.dynComputedKeyGetHelper", + "dyn.caughtToDynHelper", + "dyn.messageListenerHelper", + "dyn.messageReaderHelper", +]) + registerReplay(op, (emitter, args) => emitter.dyn.replayHelper(op, args)); +for (const op of [ + "walkers.jsonWriteHelper", + "walkers.recordFieldWriter", + "walkers.unionJoinHelper", +]) + registerReplay(op, (emitter, args) => emitter.walkers.replayHelper(op, args)); +for (const op of ["publish.walker", "publish.refusal"]) + registerReplay(op, (emitter, args) => emitter.publishWalkers.replayHelper(op, args)); diff --git a/packages/compiler/src/backend/llvm/expr-callbacks.ts b/packages/compiler/src/backend/llvm/expr-callbacks.ts index b91cdc543..398aebb21 100644 --- a/packages/compiler/src/backend/llvm/expr-callbacks.ts +++ b/packages/compiler/src/backend/llvm/expr-callbacks.ts @@ -14,6 +14,7 @@ import { LlvmUnsupportedError } from "./unsupported.js"; import type { LlvmEmitterContext } from "./expr-context.js"; import { unionPeekLines, unionTagLines } from "./union-repr.js"; import { inlineUnionCopyLines } from "./inline-unions.js"; +import { registerReplay } from "./emit-journal.js"; /** Text-template union construction around an owned (+1) reference payload: * a fresh box, or the payload itself for a nullable union. */ @@ -55,6 +56,7 @@ export function raceAdapterFor(host: LlvmEmitterContext, from: IrType, to: IrTyp if (sym) return sym; sym = `sc_race_${host.resolveThunks.size}`; host.resolveThunks.set(key, sym); + host.journal?.allocated(sym, "raceAdapterFor", [from, to]); const toArms = to.kind === "union" ? (host.unionsById.get(to.unionId)?.arms ?? []) : []; if (from.kind === "void" && !toArms.some((a) => a.kind === "undefinedT")) { // A Promise entry (the frontend admits void entries without an @@ -70,6 +72,7 @@ export function raceAdapterFor(host: LlvmEmitterContext, from: IrType, to: IrTyp `}`, ``, ); + host.journal?.built(); return sym; } if (to.kind !== "union") @@ -98,6 +101,7 @@ export function raceAdapterFor(host: LlvmEmitterContext, from: IrType, to: IrTyp const tag = tagOf(UNDEFINED_T); d.push(` ${fulfill(host.unitInstanceRef(to.unionId, tag))}`, ` ret void`, `}`, ``); host.resolveThunkDefs.push(...d); + host.journal?.built(); return sym; } if (from.kind !== "union") { @@ -138,6 +142,7 @@ export function raceAdapterFor(host: LlvmEmitterContext, from: IrType, to: IrTyp } d.push(` ${fulfill("%u")}`, ` ret void`, `}`, ``); host.resolveThunkDefs.push(...d); + host.journal?.built(); return sym; } // Sub-union re-tag: switch over the entry's arms, rebuild under the @@ -200,6 +205,7 @@ export function raceAdapterFor(host: LlvmEmitterContext, from: IrType, to: IrTyp ``, ); host.resolveThunkDefs.push(...d); + host.journal?.built(); return sym; } @@ -213,6 +219,7 @@ export function genResultThunkFor( if (sym) return sym; sym = mangleGenResThunk(host.resolveThunks.size); host.resolveThunks.set(key, sym); + host.journal?.allocated(sym, "genResultThunkFor", [genT, recT]); const shape = host.recordsById.get(recT.shapeId); const base = host.nullableUnions.fieldBase(recT.shapeId); const valueT = shape?.fields.find((f) => f.name === "value")?.type; @@ -299,6 +306,7 @@ export function genResultThunkFor( ``, ]; host.resolveThunkDefs.push(...d); + host.journal?.built(); return sym; } if (valueT.kind !== "union") throw new LlvmUnsupportedError(`genResume:${valueT.kind}`); @@ -456,6 +464,7 @@ export function genResultThunkFor( ``, ); host.resolveThunkDefs.push(...d); + host.journal?.built(); return sym; } @@ -467,6 +476,7 @@ export function childExitThunkFor(host: LlvmEmitterContext, param: IrType): stri if (sym) return sym; sym = `sc_cx_${host.resolveThunks.size}`; host.resolveThunks.set(key, sym); + host.journal?.allocated(sym, "childExitThunkFor", [param]); const def = host.unionsById.get(param.unionId); const f64Tag = def ? def.arms.findIndex((a) => a.kind === "f64") : -1; const nullTag = def ? def.arms.findIndex((a) => a.kind === "nullT") : -1; @@ -494,6 +504,7 @@ export function childExitThunkFor(host: LlvmEmitterContext, param: IrType): stri `}`, ``, ); + host.journal?.built(); return sym; } @@ -510,6 +521,7 @@ export function childExitSignalThunkFor( if (sym) return sym; sym = `sc_cx_${host.resolveThunks.size}`; host.resolveThunks.set(key, sym); + host.journal?.allocated(sym, "childExitSignalThunkFor", [codeParam, sigParam]); const codeDef = host.unionsById.get(codeParam.unionId); const f64Tag = codeDef ? codeDef.arms.findIndex((a) => a.kind === "f64") : -1; const codeNullTag = codeDef ? codeDef.arms.findIndex((a) => a.kind === "nullT") : -1; @@ -560,6 +572,7 @@ export function childExitSignalThunkFor( `}`, ``, ); + host.journal?.built(); return sym; } @@ -569,6 +582,7 @@ export function execFileThunkFor(host: LlvmEmitterContext, cbT: IrType & { kind: if (sym) return sym; sym = `sc_exec_${host.resolveThunks.size}`; host.resolveThunks.set(key, sym); + host.journal?.allocated(sym, "execFileThunkFor", [cbT]); const errorParam = cbT.params[0]; let errorTag = -1; let nullTag = -1; @@ -627,6 +641,7 @@ export function execFileThunkFor(host: LlvmEmitterContext, cbT: IrType & { kind: ``, ); host.resolveThunkDefs.push(...d); + host.journal?.built(); return sym; } @@ -639,6 +654,7 @@ export function ipcMessageThunkFor( if (sym) return sym; sym = `sc_ipc_msg_${host.resolveThunks.size}`; host.resolveThunks.set(key, sym); + host.journal?.allocated(sym, "ipcMessageThunkFor", [cbT]); const param = cbT.params[0]; const d: string[] = [ `define internal void @${sym}(ptr %cb, ptr %message) ${FN_ATTRS} { ; IPC message ${typeKey(cbT)}`, @@ -679,6 +695,7 @@ export function ipcMessageThunkFor( } d.push(` ret void`, `}`, ``); host.resolveThunkDefs.push(...d); + host.journal?.built(); return sym; } @@ -688,6 +705,7 @@ export function ipcSendThunkFor(host: LlvmEmitterContext, cbT: IrType & { kind: if (sym) return sym; sym = `sc_ipc_send_${host.resolveThunks.size}`; host.resolveThunks.set(key, sym); + host.journal?.allocated(sym, "ipcSendThunkFor", [cbT]); const param = cbT.params[0]; const d: string[] = [ `define internal void @${sym}(ptr %cb, ptr %error) ${FN_ATTRS} { ; IPC send callback ${typeKey(cbT)}`, @@ -736,6 +754,7 @@ export function ipcSendThunkFor(host: LlvmEmitterContext, cbT: IrType & { kind: } d.push(` ret void`, `}`, ``); host.resolveThunkDefs.push(...d); + host.journal?.built(); return sym; } @@ -747,6 +766,7 @@ export function childDataThunkFor(host: LlvmEmitterContext, param: IrType): stri if (sym) return sym; sym = `sc_cd_${host.resolveThunks.size}`; host.resolveThunks.set(key, sym); + host.journal?.allocated(sym, "childDataThunkFor", [param]); const def = host.unionsById.get(param.unionId); const bytesTag = def ? def.arms.findIndex((a) => a.kind === "bytes" && a.elem === "u8") : -1; if (bytesTag < 0) @@ -768,6 +788,7 @@ export function childDataThunkFor(host: LlvmEmitterContext, param: IrType): stri `}`, ``, ); + host.journal?.built(); return sym; } @@ -780,6 +801,7 @@ export function childDataDynThunkFor(host: LlvmEmitterContext, chunk: "bytes" | if (sym) return sym; sym = `sc_cdd_${host.resolveThunks.size}`; host.resolveThunks.set(key, sym); + host.journal?.allocated(sym, "childDataDynThunkFor", [chunk]); const box = chunk === "bytes" ? "scr_dyn_new_buffer" : "scr_dyn_new_str"; host.declare(`declare ptr @${box}(ptr)`); host.resolveThunkDefs.push( @@ -793,6 +815,7 @@ export function childDataDynThunkFor(host: LlvmEmitterContext, chunk: "bytes" | `}`, ``, ); + host.journal?.built(); return sym; } @@ -813,6 +836,7 @@ export function emitterFixedAdapter( if (sym) return { fn: sym, shim }; sym = `sc_ee_ad_${host.resolveThunks.size}`; host.resolveThunks.set(key, sym); + host.journal?.allocated(sym, "emitterFixedAdapter", [cbT]); host.declare(`declare ptr @scr_box_get_ref(ptr)`); host.declare(`declare void @scr_closure_release(ptr)`); const params = cursor ? "ptr %args" : cbT.params.map((_, i) => `ptr %a${i}`).join(", "); @@ -857,6 +881,7 @@ export function emitterFixedAdapter( } d.push(` call void @scr_closure_release(ptr %orig)`, ` ret void`, `}`, ``); host.resolveThunkDefs.push(...d); + host.journal?.built(); return { fn: sym, shim }; } @@ -882,6 +907,7 @@ export function emitterOnceAdapter( if (sym) return { fn: sym, shim }; sym = `sc_ee_once_${host.resolveThunks.size}`; host.resolveThunks.set(key, sym); + host.journal?.allocated(sym, "emitterOnceAdapter", [boxerT, fire]); host.declare(`declare ptr @scr_box_get_ref(ptr)`); host.declare(`declare void @scr_closure_release(ptr)`); host.declare(`declare void @${fire}(ptr, ptr)`); @@ -921,6 +947,7 @@ export function emitterOnceAdapter( ``, ); host.resolveThunkDefs.push(...d); + host.journal?.built(); return { fn: sym, shim }; } @@ -990,6 +1017,7 @@ export function closeBindThunkFor( if (sym) return sym; sym = `sc_ncb_${host.resolveThunks.size}`; host.resolveThunks.set(key, sym); + host.journal?.allocated(sym, "closeBindThunkFor", [cbUnion, retServer]); const def = host.unionsById.get(cbUnion.unionId); const funcTag = def ? def.arms.findIndex((a) => a.kind === "func") : -1; const funcArm = funcTag >= 0 ? (def!.arms[funcTag] as IrType & { kind: "func" }) : null; @@ -1079,6 +1107,7 @@ export function closeBindThunkFor( } d.push(`}`, ``); host.resolveThunkDefs.push(...d); + host.journal?.built(); return sym; } @@ -1094,6 +1123,7 @@ export function closeOverrideWrapFor( if (sym) return sym; sym = `sc_ncw_${host.resolveThunks.size}`; host.resolveThunks.set(key, sym); + host.journal?.allocated(sym, "closeOverrideWrapFor", [cbUnion, retServer]); const def = host.unionsById.get(cbUnion.unionId); const undefTag = def ? def.arms.findIndex((a) => a.kind === "undefinedT") : -1; if (undefTag < 0) @@ -1122,6 +1152,7 @@ export function closeOverrideWrapFor( `}`, ``, ); + host.journal?.built(); return sym; } @@ -1140,6 +1171,7 @@ export function settledStoreFor( if (sym) return sym; sym = `sc_settled_${host.resolveThunks.size}`; host.resolveThunks.set(key, sym); + host.journal?.allocated(sym, "settledStoreFor", [from, to]); const def = host.unionsById.get(to.unionId); if (!def) throw new InternalCompilerError("llvm emitter bug: settled store to an unknown union"); const armOf = (field: string): { tag: number; arm: IrType & { kind: "record" } } => { @@ -1230,5 +1262,52 @@ export function settledStoreFor( `}`, ``, ); + host.journal?.built(); return sym; } + +registerReplay("raceAdapterFor", (host, args) => + raceAdapterFor(host, args[0] as IrType, args[1] as IrType), +); +registerReplay("genResultThunkFor", (host, args) => + genResultThunkFor( + host, + args[0] as IrType & { kind: "generator" }, + args[1] as IrType & { kind: "record" }, + ), +); +registerReplay("childExitThunkFor", (host, args) => childExitThunkFor(host, args[0] as IrType)); +registerReplay("childExitSignalThunkFor", (host, args) => + childExitSignalThunkFor(host, args[0] as IrType, args[1] as IrType), +); +registerReplay("execFileThunkFor", (host, args) => + execFileThunkFor(host, args[0] as IrType & { kind: "func" }), +); +registerReplay("ipcMessageThunkFor", (host, args) => + ipcMessageThunkFor(host, args[0] as IrType & { kind: "func" }), +); +registerReplay("ipcSendThunkFor", (host, args) => + ipcSendThunkFor(host, args[0] as IrType & { kind: "func" }), +); +registerReplay("childDataThunkFor", (host, args) => childDataThunkFor(host, args[0] as IrType)); +registerReplay("childDataDynThunkFor", (host, args) => + childDataDynThunkFor(host, args[0] as "bytes" | "str"), +); +registerReplay("closeBindThunkFor", (host, args) => + closeBindThunkFor(host, args[0] as IrType, args[1] as boolean), +); +registerReplay("closeOverrideWrapFor", (host, args) => + closeOverrideWrapFor(host, args[0] as IrType, args[1] as boolean), +); +registerReplay("settledStoreFor", (host, args) => + settledStoreFor(host, args[0] as IrType, args[1] as IrType & { kind: "union" }), +); +registerReplay( + "emitterFixedAdapter", + (host, args) => emitterFixedAdapter(host, args[0] as IrType & { kind: "func" }).fn, +); +registerReplay( + "emitterOnceAdapter", + (host, args) => + emitterOnceAdapter(host, args[0] as IrType & { kind: "func" }, args[1] as string).fn, +); diff --git a/packages/compiler/src/backend/llvm/expr-calls.ts b/packages/compiler/src/backend/llvm/expr-calls.ts index 4bcf2a3d5..a4e7cde2c 100644 --- a/packages/compiler/src/backend/llvm/expr-calls.ts +++ b/packages/compiler/src/backend/llvm/expr-calls.ts @@ -11,6 +11,7 @@ import { type IrExpr, type IrFunction, classValueConstructor, + type IrType, } from "../../ir/ir.js"; import { liveRefView } from "./expr-stream-bridges.js"; import { typedRefConstructor, vAdapters } from "./shapes.js"; @@ -55,6 +56,7 @@ import { import type { Writes } from "./reference-writes.js"; import { emitJsIcCall, jsIcSiteOf } from "./js-ic.js"; import { emitClassMembership, emitDowncastGuard } from "./narrow-guards.js"; +import { registerReplay } from "./emit-journal.js"; export function emitCallExpr( host: LlvmEmitterContext, @@ -815,22 +817,7 @@ export function emitCallExpr( const pushReceiver = (): void => { if (lazyReceiver) { const t = lazyReceiver.type; - const key = typeKey(t); - let recipe = host.typedRefRecipes.get(key); - if (recipe === undefined) { - const adapter = host.liveDynRefAdapter(t); - const rc = vAdapters(host.shapeHost, t); - const ctor = typedRefConstructor(host.shapeHost, t); - const kind = ctor.endsWith("_traced") ? 2 : ctor.endsWith("_observed") ? 1 : 0; - // The box binds the same live view eager boxing attaches, so - // `in`, for-in and Object.keys see the class members. - const view = liveRefView(adapter); - recipe = `@sc_trr_${host.typedRefRecipes.size}`; - host.typedRefRecipes.set(key, recipe); - host.resolveThunkDefs.push( - `${recipe} = internal constant { ptr, ptr, ptr, ${host.sizeType}, ptr, ptr, i32, i32, ptr } { ptr ${rc.retain}, ptr ${rc.release}, ptr ${host.cstr(key)}, ${host.sizeType} ${Buffer.byteLength(key, "utf8")}, ptr @${adapter.snapshot}, ptr ${adapter.commit}, i32 ${kind}, i32 ${view?.code ?? 0}, ptr ${view?.ops ?? "null"} }`, - ); - } + const recipe = typedRefRecipe(host, t); host.declare("declare void @scr_dyn_this_push_typed(ptr, ptr)"); B.line(`call void @scr_dyn_this_push_typed(ptr ${lazyReceiver.name}, ptr ${recipe})`); return; @@ -1493,3 +1480,28 @@ function emitNumericIndexRead( B.line(`${result} = phi ptr [ ${fast}, %${fastLabel} ], [ ${read.name}, %${slowLabel} ]`); return host.own({ name: result, type: e.type }); } + +/** The recipe a lazily boxed receiver of type `t` is built from: its RC + * adapters, type key and live view, one per type. */ +function typedRefRecipe(host: LlvmEmitterContext, t: IrType): string { + const key = typeKey(t); + let recipe = host.typedRefRecipes.get(key); + if (recipe !== undefined) return recipe; + const adapter = host.liveDynRefAdapter(t); + const rc = vAdapters(host.shapeHost, t); + const ctor = typedRefConstructor(host.shapeHost, t); + const kind = ctor.endsWith("_traced") ? 2 : ctor.endsWith("_observed") ? 1 : 0; + // The box binds the same live view eager boxing attaches, so `in`, + // for-in and Object.keys see the class members. + const view = liveRefView(adapter); + recipe = `@sc_trr_${host.typedRefRecipes.size}`; + host.typedRefRecipes.set(key, recipe); + host.journal?.allocated(recipe, "typedRefRecipe", [t]); + host.resolveThunkDefs.push( + `${recipe} = internal constant { ptr, ptr, ptr, ${host.sizeType}, ptr, ptr, i32, i32, ptr } { ptr ${rc.retain}, ptr ${rc.release}, ptr ${host.cstr(key)}, ${host.sizeType} ${Buffer.byteLength(key, "utf8")}, ptr @${adapter.snapshot}, ptr ${adapter.commit}, i32 ${kind}, i32 ${view?.code ?? 0}, ptr ${view?.ops ?? "null"} }`, + ); + host.journal?.built(); + return recipe; +} + +registerReplay("typedRefRecipe", (host, args) => typedRefRecipe(host, args[0] as IrType)); diff --git a/packages/compiler/src/backend/llvm/expr-containers.ts b/packages/compiler/src/backend/llvm/expr-containers.ts index 8970aa3be..1d42c4730 100644 --- a/packages/compiler/src/backend/llvm/expr-containers.ts +++ b/packages/compiler/src/backend/llvm/expr-containers.ts @@ -58,6 +58,7 @@ import { emitDenseArrayPush, } from "./dense-array-access.js"; import { emitUnionWiden } from "./expr-records.js"; +import { registerReplay } from "./emit-journal.js"; export function resolveThunkFor(host: LlvmEmitterContext, inner: IrType): string { const key = typeKey(inner); @@ -65,6 +66,7 @@ export function resolveThunkFor(host: LlvmEmitterContext, inner: IrType): string if (!sym) { sym = mangleResolveThunk(host.resolveThunks.size); host.resolveThunks.set(key, sym); + host.journal?.allocated(sym, "resolveThunkFor", [inner]); const v = vAdapters(host.shapeHost, inner); host.declare(`declare void @scr_resolve_ref_impl(ptr, ptr, ptr, ptr, ptr)`); host.resolveThunkDefs.push( @@ -75,6 +77,7 @@ export function resolveThunkFor(host: LlvmEmitterContext, inner: IrType): string `}`, ``, ); + host.journal?.built(); } return sym; } @@ -1539,3 +1542,5 @@ export function emitSetNew(host: LlvmEmitterContext, e: IrExpr & { kind: "setNew } return out; } + +registerReplay("resolveThunkFor", (host, args) => resolveThunkFor(host, args[0] as IrType)); diff --git a/packages/compiler/src/backend/llvm/expr-primitives.ts b/packages/compiler/src/backend/llvm/expr-primitives.ts index 354004e9e..6f960c11f 100644 --- a/packages/compiler/src/backend/llvm/expr-primitives.ts +++ b/packages/compiler/src/backend/llvm/expr-primitives.ts @@ -563,6 +563,7 @@ export function regexLiteralTemplate( }; host.regexInstances.set(key, re); } + host.journal?.entry("regex", key, re); return `@${re.sym}`; } @@ -823,6 +824,7 @@ export function emitStringExpr( }; host.templateStringsInstances.set(e.key, inst); } + host.journal?.entry("templateStrings", e.key, inst); return host.own({ name: host.retainFrame(`@${inst.sym}`, e.type), type: e.type }); } case "regexIntrinsic": diff --git a/packages/compiler/src/backend/llvm/expr-stream-bridges.ts b/packages/compiler/src/backend/llvm/expr-stream-bridges.ts index a0a0cf24a..7cb0fa772 100644 --- a/packages/compiler/src/backend/llvm/expr-stream-bridges.ts +++ b/packages/compiler/src/backend/llvm/expr-stream-bridges.ts @@ -44,6 +44,7 @@ import type { NullableRefField } from "./nullable-fields.js"; import { emitUnionPeek, emitUnionTag } from "./union-repr.js"; import { inlineUnionCopyLines } from "./inline-unions.js"; import { recordMayKeepSource } from "./record-sources.js"; +import { registerReplay } from "./emit-journal.js"; /** An owned (+1) union value for a nullable-pointer class field slot `p` * (the unit arm's immortal instance, or a fresh box around a retained @@ -91,6 +92,7 @@ export function dynPromiseAdapter(host: LlvmEmitterContext, inner: IrType): stri if (existing) return existing; const sym = `sc_dpa_${host.dynPromiseAdapters.size}`; host.dynPromiseAdapters.set(key, sym); + host.journal?.allocated(sym, "dynPromiseAdapter", [inner]); host.declare(`declare ptr @scr_promise_payload_ref(ptr)`); host.declare(`declare void @scr_dyn_release_v(ptr)`); host.declare(`declare void @scr_promise_reject_pending(ptr)`); @@ -146,6 +148,7 @@ export function dynPromiseAdapter(host: LlvmEmitterContext, inner: IrType): stri `}`, ``, ); + host.journal?.built(); return sym; } @@ -602,6 +605,7 @@ export function liveDynUnionRefAdapter( const sym = `sc_ldu_${host.liveDynUnionRefAdapters.size}`; host.liveDynUnionRefAdapters.set(key, sym); + host.journal?.allocated(sym, "liveDynUnionRefAdapter", [t]); const adapters = new Map(); for (const { arm, tag } of mutableArms) { @@ -684,6 +688,7 @@ export function liveDynUnionRefAdapter( `}`, ``, ); + host.journal?.built(); return sym; } @@ -732,6 +737,11 @@ export function streamTypedRefMaterializeAdapter( const snapshot = `sc_ldr_${host.liveDynRefAdapters.size}_materialize`; const adapter: LlStreamTypedRefAdapter = { snapshot, commit: "null" }; host.liveDynRefAdapters.set(key, adapter); + host.journal?.allocated( + snapshot.replace(/_materialize$/, ""), + "streamTypedRefMaterializeAdapter", + [t], + ); if (isDynTypedRefType(t)) { const fields = host.classMeta.get(t.className)?.def.fields; if ( @@ -744,6 +754,7 @@ export function streamTypedRefMaterializeAdapter( ) { host.declare(`declare ptr @scr_dyn_class_view_unavailable(ptr)`); adapter.snapshot = "scr_dyn_class_view_unavailable"; + host.journal?.built(); return adapter; } } @@ -1185,6 +1196,7 @@ export function streamTypedRefMaterializeAdapter( `}`, ``, ); + host.journal?.built(); return adapter; } @@ -1469,6 +1481,7 @@ export function streamFromArrayAdapter( if (existing) return existing; const sym = `sc_sfa_${host.streamFromArrayAdapters.size}`; host.streamFromArrayAdapters.set(key, sym); + host.journal?.allocated(sym, "streamFromArrayAdapter", [t]); const B = new BlockBuilder(); const unionDef = elem.kind === "union" ? host.unionsById.get(elem.unionId) : undefined; if (elem.kind === "union" && !unionDef) { @@ -1585,5 +1598,17 @@ export function streamFromArrayAdapter( `}`, ``, ); + host.journal?.built(); return sym; } + +registerReplay("dynPromiseAdapter", (host, args) => dynPromiseAdapter(host, args[0] as IrType)); +registerReplay("liveDynUnionRefAdapter", (host, args) => + liveDynUnionRefAdapter(host, args[0] as IrType & { kind: "union" }), +); +registerReplay("streamTypedRefMaterializeAdapter", (host, args) => + streamTypedRefMaterializeAdapter(host, args[0] as IrType).snapshot.replace(/_materialize$/, ""), +); +registerReplay("streamFromArrayAdapter", (host, args) => + streamFromArrayAdapter(host, args[0] as IrType & { kind: "array" }), +); diff --git a/packages/compiler/src/backend/llvm/expr-stream-callbacks.ts b/packages/compiler/src/backend/llvm/expr-stream-callbacks.ts index 0570d0ea7..42c803697 100644 --- a/packages/compiler/src/backend/llvm/expr-stream-callbacks.ts +++ b/packages/compiler/src/backend/llvm/expr-stream-callbacks.ts @@ -4,6 +4,7 @@ import { type IrType, isRefCounted, typeKey } from "../../ir/ir.js"; import { FN_ATTRS, releaseSym, retainSym, traceArg } from "./shapes.js"; import type { LlvmEmitterContext } from "./expr-context.js"; import { unionPeekLines, unionTagLines } from "./union-repr.js"; +import { registerReplay } from "./emit-journal.js"; export function streamDataAdapter( host: LlvmEmitterContext, @@ -14,6 +15,7 @@ export function streamDataAdapter( if (sym) return sym; sym = `sc_ee_dad_${host.resolveThunks.size}`; host.resolveThunks.set(key, sym); + host.journal?.allocated(sym, "streamDataAdapter", [cbT]); const p = cbT.params[0]; if (cbT.params.length > 1) { throw new InternalCompilerError( @@ -207,6 +209,7 @@ export function streamDataAdapter( d.push(`}`, ``); } host.resolveThunkDefs.push(...d); + host.journal?.built(); return sym; } @@ -220,6 +223,7 @@ export function streamDoneFnFor( if (sym) return sym; sym = `sc_sd_${host.resolveThunks.size}`; host.resolveThunks.set(key, sym); + host.journal?.allocated(sym, "streamDoneFnFor", [kind, doneT]); const errT = doneT.params[0]; let errTag = -1; if (errT !== undefined) { @@ -347,6 +351,7 @@ export function streamDoneFnFor( d.push(` call void ${releaseSym(host.shapeHost, dataT)}(ptr %d)`); d.push(` ret void`, `}`, ``); host.resolveThunkDefs.push(...d); + host.journal?.built(); return sym; } @@ -360,6 +365,7 @@ export function fsRenameThunkFor(host: LlvmEmitterContext, cbT: IrType & { kind: if (sym) return sym; sym = `sc_fsren_${host.resolveThunks.size}`; host.resolveThunks.set(key, sym); + host.journal?.allocated(sym, "fsRenameThunkFor", [cbT]); const param = cbT.params[0]!; const d: string[] = [ `define internal void @${sym}(ptr %cb, ptr %err) ${FN_ATTRS} { ; fs.rename callback ${typeKey(cbT)}`, @@ -415,6 +421,7 @@ export function fsRenameThunkFor(host: LlvmEmitterContext, cbT: IrType & { kind: ``, ); host.resolveThunkDefs.push(...d); + host.journal?.built(); return sym; } @@ -447,6 +454,7 @@ export function cryptoIntThunkFor( if (sym) return sym; sym = `sc_cryptoint_${host.resolveThunks.size}`; host.resolveThunks.set(key, sym); + host.journal?.allocated(sym, "cryptoIntThunkFor", [cbT]); const d: string[] = [ `define internal void @${sym}(ptr %cb, double %value) ${FN_ATTRS} { ; randomInt callback ${typeKey(cbT)}`, `entry:`, @@ -489,6 +497,7 @@ export function cryptoIntThunkFor( ``, ); host.resolveThunkDefs.push(...d); + host.journal?.built(); return sym; } @@ -507,6 +516,7 @@ export function zlibBytesThunkFor( if (sym) return sym; sym = `sc_${family}bytes_${host.resolveThunks.size}`; host.resolveThunks.set(key, sym); + host.journal?.allocated(sym, "zlibBytesThunkFor", [cbT, arrayBuffer, family, valueMode]); const d: string[] = [ `define internal void @${sym}(ptr %cb, ptr %err, ptr %value) ${FN_ATTRS} { ; zlib bytes callback ${typeKey(cbT)}`, `entry:`, @@ -618,6 +628,7 @@ export function zlibBytesThunkFor( ``, ); host.resolveThunkDefs.push(...d); + host.journal?.built(); return sym; } @@ -633,6 +644,7 @@ export function streamCbThunkFor( if (sym) return sym; sym = `sc_scb_${host.resolveThunks.size}`; host.resolveThunks.set(key, sym); + host.journal?.allocated(sym, "streamCbThunkFor", [kind, cbT]); const runtimeParams = kind === "r" ? ["ptr %s", "double %size"] @@ -901,5 +913,35 @@ export function streamCbThunkFor( } d.push(` ret void`, `}`, ``); host.resolveThunkDefs.push(...d); + host.journal?.built(); return sym; } + +registerReplay("streamDataAdapter", (host, args) => + streamDataAdapter(host, args[0] as IrType & { kind: "func" }), +); +registerReplay("streamDoneFnFor", (host, args) => + streamDoneFnFor( + host, + args[0] as "w" | "f" | "d" | "t" | "l", + args[1] as IrType & { kind: "func" }, + ), +); +registerReplay("fsRenameThunkFor", (host, args) => + fsRenameThunkFor(host, args[0] as IrType & { kind: "func" }), +); +registerReplay("cryptoIntThunkFor", (host, args) => + cryptoIntThunkFor(host, args[0] as IrType & { kind: "func" }), +); +registerReplay("zlibBytesThunkFor", (host, args) => + zlibBytesThunkFor( + host, + args[0] as IrType & { kind: "func" }, + args[1] as boolean, + args[2] as string, + args[3] as BytesThunkValue, + ), +); +registerReplay("streamCbThunkFor", (host, args) => + streamCbThunkFor(host, args[0] as "r" | "w" | "f" | "d" | "t" | "l" | "e", args[1] as IrType), +); diff --git a/packages/compiler/src/backend/llvm/lib-connect.ts b/packages/compiler/src/backend/llvm/lib-connect.ts index ce9c40e21..4d151fc7e 100644 --- a/packages/compiler/src/backend/llvm/lib-connect.ts +++ b/packages/compiler/src/backend/llvm/lib-connect.ts @@ -3,6 +3,7 @@ import { typeKey } from "../../ir/ir.js"; import type { LibCallExpr, LlValue, LlvmEmitterContext } from "./expr-context.js"; import { abiValue, callbackType, finishRuntimeCall, ptr } from "./lib-abi.js"; import { FN_ATTRS } from "./shapes.js"; +import { registerReplay } from "./emit-journal.js"; /** CONNECT transfers owned request/socket/response values to the listener. */ function connectAdapter(host: LlvmEmitterContext, cb: LlValue, h2: boolean): string { @@ -18,6 +19,7 @@ function connectAdapter(host: LlvmEmitterContext, cb: LlValue, h2: boolean): str if (previous !== undefined) return previous; const symbol = `sc_connect_${host.resolveThunks.size}`; host.resolveThunks.set(key, symbol); + host.journal?.allocated(symbol, "connectAdapter", [cb, h2]); const retain = h2 ? "scr_http_res_retain_v" : "scr_net_sock_retain_v"; const release = h2 ? "scr_http_res_release_v" : "scr_net_sock_release_v"; host.declare(`declare ptr @${retain}(ptr)`); @@ -37,6 +39,7 @@ function connectAdapter(host: LlvmEmitterContext, cb: LlValue, h2: boolean): str "}", "", ); + host.journal?.built(); return symbol; } @@ -70,3 +73,7 @@ export function emitConnectListener(host: LlvmEmitterContext, e: LibCallExpr): L a[2]!, ]); } + +registerReplay("connectAdapter", (host, args) => + connectAdapter(host, args[0] as LlValue, args[1] as boolean), +); diff --git a/packages/compiler/src/backend/llvm/lib-datagram.ts b/packages/compiler/src/backend/llvm/lib-datagram.ts index 184fdb99e..e32fd238d 100644 --- a/packages/compiler/src/backend/llvm/lib-datagram.ts +++ b/packages/compiler/src/backend/llvm/lib-datagram.ts @@ -11,6 +11,7 @@ import { ptr, } from "./lib-abi.js"; import { FN_ATTRS } from "./shapes.js"; +import { registerReplay } from "./emit-journal.js"; function messageAdapter(host: LlvmEmitterContext, type: IrType): string { if (type.kind !== "record") @@ -20,6 +21,7 @@ function messageAdapter(host: LlvmEmitterContext, type: IrType): string { if (previous !== undefined) return previous; const symbol = `sc_dgram_message_${host.resolveThunks.size}`; host.resolveThunks.set(key, symbol); + host.journal?.allocated(symbol, "messageAdapter", [type]); const shape = host.recordsById.get(type.shapeId); if (shape === undefined) throw new InternalCompilerError("missing datagram remote info shape"); const lines: string[] = []; @@ -48,6 +50,7 @@ function messageAdapter(host: LlvmEmitterContext, type: IrType): string { "}", "", ); + host.journal?.built(); return symbol; } @@ -67,6 +70,7 @@ function lookupAdapter(host: LlvmEmitterContext, cb: LlValue): string { if (previous !== undefined) return previous; const symbol = `sc_dns_lookup_${host.resolveThunks.size}`; host.resolveThunks.set(key, symbol); + host.journal?.allocated(symbol, "lookupAdapter", [cb]); host.declare("declare ptr @scr_error_new(i32, ptr)"); host.declare("declare ptr @scr_error_retain_v(ptr)"); host.declare("declare void @scr_error_release_v(ptr)"); @@ -93,6 +97,7 @@ function lookupAdapter(host: LlvmEmitterContext, cb: LlValue): string { "}", "", ); + host.journal?.built(); return symbol; } @@ -250,3 +255,6 @@ export function emitDatagramLibCall(host: LlvmEmitterContext, e: LibCallExpr): L throw new InternalCompilerError(`unhandled datagram call ${e.fn}`); } } + +registerReplay("messageAdapter", (host, args) => messageAdapter(host, args[0] as IrType)); +registerReplay("lookupAdapter", (host, args) => lookupAdapter(host, args[0] as LlValue)); diff --git a/packages/compiler/src/backend/llvm/lib-http2.ts b/packages/compiler/src/backend/llvm/lib-http2.ts index 791987f86..321f1d005 100644 --- a/packages/compiler/src/backend/llvm/lib-http2.ts +++ b/packages/compiler/src/backend/llvm/lib-http2.ts @@ -13,6 +13,7 @@ import { requestHandler, } from "./lib-abi.js"; import { FN_ATTRS } from "./shapes.js"; +import { registerReplay } from "./emit-journal.js"; /** The runtime owns the answer closure; its arguments arrive owned from the caller. */ function sniAnswer(host: LlvmEmitterContext, type: IrType): string { @@ -22,6 +23,7 @@ function sniAnswer(host: LlvmEmitterContext, type: IrType): string { if (previous !== undefined) return previous; const symbol = `sc_sni_answer_${host.resolveThunks.size}`; host.resolveThunks.set(key, symbol); + host.journal?.allocated(symbol, "sniAnswer", [type]); const params = ["ptr %self"], body: string[] = []; let hasError = "false", @@ -82,6 +84,7 @@ function sniAnswer(host: LlvmEmitterContext, type: IrType): string { "}", "", ); + host.journal?.built(); return symbol; } @@ -318,3 +321,5 @@ export function emitHttp2LibCall(host: LlvmEmitterContext, e: LibCallExpr): LlVa const handler = thunk === null ? [a[1]!] : callbackAdapter(host, cb, thunk, params); return finishRuntimeCall(host, e, runtime, [a[0]!, ...handler, ...a.slice(2)]); } + +registerReplay("sniAnswer", (host, args) => sniAnswer(host, args[0] as IrType)); diff --git a/packages/compiler/src/backend/llvm/lib-lookup.ts b/packages/compiler/src/backend/llvm/lib-lookup.ts index c6bc4d97b..56e94461f 100644 --- a/packages/compiler/src/backend/llvm/lib-lookup.ts +++ b/packages/compiler/src/backend/llvm/lib-lookup.ts @@ -1,9 +1,10 @@ import { InternalCompilerError } from "../../errors.js"; -import { typeKey } from "../../ir/ir.js"; +import { type IrType, typeKey } from "../../ir/ir.js"; import { mangleRecordRelease, mangleRecordStruct } from "../mangle.js"; import type { LibCallExpr, LlValue, LlvmEmitterContext } from "./expr-context.js"; import { abiValue, callbackType, finishRuntimeCall, ptr } from "./lib-abi.js"; import { FN_ATTRS } from "./shapes.js"; +import { registerReplay } from "./emit-journal.js"; export function emitLookupConnect(host: LlvmEmitterContext, e: LibCallExpr): LlValue { const args = e.args.map((arg) => host.emitExpr(arg)), @@ -12,6 +13,18 @@ export function emitLookupConnect(host: LlvmEmitterContext, e: LibCallExpr): LlV answer = lookup.params[2]; if (answer?.kind !== "func" || answer.params.length !== 2) throw new InternalCompilerError("lookup answer must have two parameters"); + const symbol = lookupAnswer(host, answer); + host.usesTimers = true; + host.toHeap(cb); + return finishRuntimeCall(host, e, "scr_net_connect_lookup", [ + ...args.map((arg) => abiValue(host, arg)), + ptr("@" + symbol), + ]); +} + +/** The thunk that hands a lookup's answer (error, addresses) to the + * runtime, one per answer signature. */ +function lookupAnswer(host: LlvmEmitterContext, answer: IrType & { kind: "func" }): string { const error = answer.params[0]!, addresses = answer.params[1]!; if (error.kind !== "union" || addresses.kind !== "array" || addresses.elem.kind !== "record") @@ -26,6 +39,7 @@ export function emitLookupConnect(host: LlvmEmitterContext, e: LibCallExpr): LlV if (symbol === undefined) { symbol = `sc_lookup_answer_${host.resolveThunks.size}`; host.resolveThunks.set(key, symbol); + host.journal?.allocated(symbol, "lookupAnswer", [answer]); for (const declaration of [ "declare ptr @scr_str_retain_v(ptr)", "declare void @scr_str_release(ptr)", @@ -84,11 +98,11 @@ export function emitLookupConnect(host: LlvmEmitterContext, e: LibCallExpr): LlV "}", "", ); + host.journal?.built(); } - host.usesTimers = true; - host.toHeap(cb); - return finishRuntimeCall(host, e, "scr_net_connect_lookup", [ - ...args.map((arg) => abiValue(host, arg)), - ptr("@" + symbol), - ]); + return symbol; } + +registerReplay("lookupAnswer", (host, args) => + lookupAnswer(host, args[0] as IrType & { kind: "func" }), +); diff --git a/packages/compiler/src/backend/llvm/publish.ts b/packages/compiler/src/backend/llvm/publish.ts index 2f45c3cd2..60270adda 100644 --- a/packages/compiler/src/backend/llvm/publish.ts +++ b/packages/compiler/src/backend/llvm/publish.ts @@ -31,6 +31,13 @@ import type { InlineUnionFields } from "./inline-unions.js"; import { LlvmUnsupportedError } from "./unsupported.js"; export interface PublishHost { + /** Chunk emitters only (emit-journal.ts): EmitJournal's allocated, built, + * opened, used and entry. */ + journalAllocated?: ((name: string, op: string, args: readonly unknown[]) => void) | undefined; + journalBuilt?: (() => void) | undefined; + journalOpened?: ((op: string) => void) | undefined; + journalUsed?: ((name: string, op: string, args: readonly unknown[]) => void) | undefined; + journalEntry?: ((table: string, key: string, value: unknown) => void) | undefined; declare(decl: string): void; /** An interned NUL-terminated constant. */ cstr(text: string): string; @@ -103,6 +110,14 @@ export class PublishWalkers { constructor(private readonly host: PublishHost) {} + /** Journal replay (emit-journal.ts): builds or finds the walker a chunk + * emitter recorded, and answers its name. */ + replayHelper(op: string, args: readonly unknown[]): string { + if (op === "publish.walker") + return this.walker(args[0] as IrType, args[1] as string, args[2] as SrcLoc)!; + return this.refusal(args[0] as string, args[1] as string); + } + /** The definitions emitted so far (module-level LLVM text). */ defs(): string[] { return this.defsOut; @@ -119,6 +134,7 @@ export class PublishWalkers { } const fn = this.walker(t, "the published value", loc); if (fn === null) return []; + this.host.journalUsed?.(fn, "publish.walker", [t, "the published value", loc]); this.host.declare(`declare void @scr_publish(ptr, ptr)`); return [` call void @scr_publish(ptr ${v}, ptr ${fn})`]; } @@ -156,8 +172,10 @@ export class PublishWalkers { if (known !== undefined) return known; const sym = `@sc_pub_${this.counter++}`; this.walkers.set(key, sym); + this.host.journalOpened?.("publish.walker"); const body = this.body(t, sym, path, loc); this.defsOut.push(...body, ``); + this.host.journalBuilt?.(); return sym; } @@ -172,6 +190,7 @@ export class PublishWalkers { if (known !== undefined) return known; const sym = `@sc_pub_${this.counter++}`; this.refusals.set(key, sym); + this.host.journalOpened?.("publish.refusal"); this.host.declare(`declare void @scr_pub_refuse_at(ptr, ptr, ptr)`); this.defsOut.push( `define internal void ${sym}(ptr %o, ptr %ctx) ${FN_ATTRS} { ; refuse ${llvmComment(detail)}`, @@ -181,6 +200,7 @@ export class PublishWalkers { `}`, ``, ); + this.host.journalBuilt?.(); return sym; } diff --git a/packages/compiler/src/backend/llvm/shapes.ts b/packages/compiler/src/backend/llvm/shapes.ts index 6de92f667..6daa156ce 100644 --- a/packages/compiler/src/backend/llvm/shapes.ts +++ b/packages/compiler/src/backend/llvm/shapes.ts @@ -68,6 +68,13 @@ const CYC_MAX_TYPES = 32768; /** What the tables need from the emitter: the extern-declaration ledger * and the module-wide cycle/shape indexes. */ export interface ShapeHost { + /** Chunk emitters only (emit-journal.ts): EmitJournal's allocated, built, + * opened, used and entry. */ + journalAllocated?: ((name: string, op: string, args: readonly unknown[]) => void) | undefined; + journalBuilt?: (() => void) | undefined; + journalOpened?: ((op: string) => void) | undefined; + journalUsed?: ((name: string, op: string, args: readonly unknown[]) => void) | undefined; + journalEntry?: ((table: string, key: string, value: unknown) => void) | undefined; /** The hierarchy root of a class (a program class or a runtime base), * or undefined for classes outside the class graph. */ classRootOf?(className: string): string | undefined; @@ -386,6 +393,7 @@ function nullableRetainSym(host: ShapeHost, arm: IrType): string { const sym = `@sc_nretain_${arm.kind}`; if (!host.nullableUnions.retainWrappers.has(sym)) host.nullableUnions.retainWrappers.set(sym, retainSym(host, arm)); + host.journalEntry?.("retainWrapper", sym, host.nullableUnions.retainWrappers.get(sym)); return sym; } diff --git a/packages/compiler/src/backend/llvm/walkers.ts b/packages/compiler/src/backend/llvm/walkers.ts index 46dbc86e1..044cb79ae 100644 --- a/packages/compiler/src/backend/llvm/walkers.ts +++ b/packages/compiler/src/backend/llvm/walkers.ts @@ -62,6 +62,21 @@ export class LlWalkers { constructor(private readonly host: WalkerHost) {} + /** Journal replay (emit-journal.ts): builds or finds the helper a chunk + * emitter recorded, and answers its name. */ + replayHelper(op: string, args: readonly unknown[]): string { + switch (op) { + case "walkers.jsonWriteHelper": + return this.jsonWriteHelper(args[0] as IrType); + case "walkers.recordFieldWriter": + return this.recordFieldWriter(args[0] as string); + case "walkers.unionJoinHelper": + return this.unionJoinHelper(args[0] as string); + default: + throw new InternalCompilerError(`llvm emitter bug: no walker replay ${op}`); + } + } + private get S(): "i32" | "i64" { return this.host.sizeType; } @@ -191,6 +206,7 @@ export class LlWalkers { if (existing) return existing; const name = `sc_jw_${this.jsonWriters.size}`; this.jsonWriters.set(key, name); + this.host.journalAllocated?.(name, "walkers.jsonWriteHelper", [t]); const B = new BlockBuilder(); switch (t.kind) { case "f64": @@ -244,6 +260,7 @@ export class LlWalkers { `}`, ``, ); + this.host.journalBuilt?.(); return name; } @@ -457,6 +474,7 @@ export class LlWalkers { throw new InternalCompilerError(`llvm emitter bug: field writer of unknown shape ${shapeId}`); const name = `sc_jwf_${this.recordFieldWriters.size}`; this.recordFieldWriters.set(shapeId, name); + this.host.journalAllocated?.(name, "walkers.recordFieldWriter", [shapeId]); const fieldBase = this.host.nullableUnions.fieldBase(shapeId); const fieldIndex = new Map(shape.fields.map((f, i) => [f.name, i + fieldBase])); const byName = new Map(shape.fields.map((f) => [f.name, f])); @@ -508,6 +526,7 @@ export class LlWalkers { `}`, ``, ); + this.host.journalBuilt?.(); return name; } @@ -852,6 +871,7 @@ export class LlWalkers { if (!def) throw new InternalCompilerError(`llvm emitter bug: join of unknown union ${unionId}`); const name = `sc_uj_${this.unionJoinFns.size}`; this.unionJoinFns.set(unionId, name); + this.host.journalAllocated?.(name, "walkers.unionJoinHelper", [unionId]); const host = this.host; host.declare(`declare void @scr_jb_init(ptr)`); @@ -946,6 +966,7 @@ export class LlWalkers { `}`, ``, ); + this.host.journalBuilt?.(); return name; } } diff --git a/packages/compiler/src/frontend/process-fork.ts b/packages/compiler/src/frontend/process-fork.ts new file mode 100644 index 000000000..744d57aa5 --- /dev/null +++ b/packages/compiler/src/frontend/process-fork.ts @@ -0,0 +1,26 @@ +/** Process primitives for forked lowering workers (unit-journal.ts). The + * native compiler fills these in; under Node they stay unset and reachable + * emit lowers every wave in-process. */ +export interface ProcessFork { + /** fork(): 0 in the worker, the worker's pid in the parent, -1 on failure. */ + fork(): number; + /** Ends the worker without running exit handlers. */ + exit(code: number): void; + /** The worker's exit status, 128 plus a signal number, or -1. */ + wait(pid: number): number; + /** Opens the lock file checker requests from workers take. */ + openLock(path: string): boolean; + lock(): void; + unlock(): void; +} + +export const processFork: { + primitives: ProcessFork | null; + /** True in a worker: checker requests take the lock (rpc-client.ts). */ + worker: boolean; + /** A worker's checker requests and the time it waited for the lock and + * for responses (SCRIPTC_TIMING). */ + requests: number; + lockWaitMs: number; + requestMs: number; +} = { primitives: null, worker: false, requests: 0, lockWaitMs: 0, requestMs: 0 }; diff --git a/packages/compiler/src/native/cli.ts b/packages/compiler/src/native/cli.ts index e2c3a895b..a2335444c 100644 --- a/packages/compiler/src/native/cli.ts +++ b/packages/compiler/src/native/cli.ts @@ -20,12 +20,28 @@ import { openNativeCache } from "./cache.js"; import { loadNativeToolchain } from "./toolchain.js"; import { buildSanitizedRuntime } from "./sanitizer.js"; import { runCompilerTask } from "./task.js"; +import { processFork } from "../frontend/process-fork.js"; declare function compilerNativePrivateDirectory(path: string, harden: boolean): boolean; declare function compilerNativeCollectCycles(): void; +declare function compilerNativeFork(): number; +declare function compilerNativeExit(code: number): void; +declare function compilerNativeWait(pid: number): number; +declare function compilerNativeLockOpen(path: string): boolean; +declare function compilerNativeLock(): void; +declare function compilerNativeUnlock(): void; async function main(): Promise { setFrontendDisposedHook(() => compilerNativeCollectCycles()); + if (process.platform !== "win32") + processFork.primitives = { + fork: () => compilerNativeFork(), + exit: (code) => compilerNativeExit(code), + wait: (pid) => compilerNativeWait(pid), + openLock: (path) => compilerNativeLockOpen(path), + lock: () => compilerNativeLock(), + unlock: () => compilerNativeUnlock(), + }; const manifestPath = process.env["SCRIPTC_TOOLCHAIN"] ?? process.execPath + ".json"; let compiler: NativeCompiler | null = null; let cache: ReturnType = null; diff --git a/packages/compiler/src/native/prepare.ts b/packages/compiler/src/native/prepare.ts index e6e36c08f..6b21b2230 100644 --- a/packages/compiler/src/native/prepare.ts +++ b/packages/compiler/src/native/prepare.ts @@ -7,7 +7,7 @@ import type { CompileLibraryOptions, CompileRequestOptions, } from "../compile-types.js"; -import { emitLlvmModule, lowRegionTarget } from "../backend/llvm/emitter.js"; +import { emitLlvmModule, type ForkPrimitives, lowRegionTarget } from "../backend/llvm/emitter.js"; import { inlineRcOption } from "../backend/llvm/shapes.js"; import { executableLinkFeatures } from "../backend/executable-features.js"; import type { NativeLinkFeatures } from "../backend/native-link-info-core.js"; @@ -23,6 +23,7 @@ import { trackedReadFile, } from "../frontend/input-tracker.js"; import { provenanceSources } from "../frontend/provenance-registry.js"; +import { processFork } from "../frontend/process-fork.js"; import { prepareLibrary } from "../library/prepare.js"; import type { LibraryProfile } from "../library/library-profile.js"; import type { IrModule } from "../ir/ir.js"; @@ -297,6 +298,8 @@ export function prepareNativeExecutable( runtimeAbiMarker: outputKind === "obj" || outputKind === "exe", objectAudit: options.sanitize === true, lowRegion: lowRegionTarget(toolchain.target.platform, options.sanitize === true), + functionWorkers: emitWorkers(), + forkPrimitives: emitForkPrimitives(), ...inlineRcOption(options.optimization, options.sanitize === true), ...(options.detachedExit === true && outputKind === "exe" ? { detachedExit: true } : {}), ...(options.optimization === "dev" && !options.strip @@ -433,3 +436,22 @@ export function prepareNativeLibrary( timing("frontend-cache-publish"); return input; } + +/** Worker processes that emit function bodies at a time + * (SCRIPTC_EMIT_WORKERS; 0 or 1 emits in this process). */ +function emitWorkers(): number { + const value = Number(process.env["SCRIPTC_EMIT_WORKERS"] ?? "4"); + return Number.isFinite(value) && value > 1 ? Math.floor(value) : 1; +} + +/** The process primitives emission workers fork with, unless the host + * cannot fork or this process is a worker itself. */ +function emitForkPrimitives(): ForkPrimitives | undefined { + const primitives = processFork.primitives; + if (primitives === null || processFork.worker) return undefined; + return { + fork: () => primitives.fork(), + exit: (code) => primitives.exit(code), + wait: (pid) => primitives.wait(pid), + }; +} diff --git a/packages/compiler/test/chunked-emission.test.ts b/packages/compiler/test/chunked-emission.test.ts new file mode 100644 index 000000000..9902569f7 --- /dev/null +++ b/packages/compiler/test/chunked-emission.test.ts @@ -0,0 +1,69 @@ +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { compile, deserializeModule } from "../src/index.js"; +import type { IrModule } from "../src/ir/ir.js"; +import { emitLlvmModuleSource, type LlvmTargetOptions } from "../src/backend/llvm/emitter.js"; +import { inlineRcOption } from "../src/backend/llvm/shapes.js"; + +// Programs whose function bodies request module-level helpers and tables +// that the chunk journal (emit-journal.ts) carries: checked-dynamic +// conversions, inspection, publication walkers, live record views, stream +// adapters, regex instances, colliding literal digests, interned C strings, +// speculative loop sites, typed message listeners and readers. +const PROGRAMS = [ + "1003-json-parse-unions.ts", + "114-live-record-delete.ts", + "1200-regex-test-basics.ts", + "1631-inspect-arrays.ts", + "4598-publish-constructor-stores.ts", + "4900-console-instances.ts", + "5612-readable-from-iterables.cjs", + "5702-publish-candidate-buffers.ts", + "7321-speculative-loop-bail-sites.ts", + "7360-worker-typed-records.ts", + "node24-text-utilities-coercion.js", +]; + +const OPTIONS: Record = { + release: {}, + speed: { + targetTriple: "x86_64-unknown-linux-gnu", + pointerBits: 64, + runtimeAbiMarker: true, + lowRegion: true, + ...inlineRcOption("speed", false), + }, +}; + +const text = (out: string | readonly string[]): string => + typeof out === "string" ? out : out.join(""); + +async function lower(program: string, dir: string): Promise { + const entry = join(import.meta.dirname, "../../../tests/corpus", program); + const outPath = join(dir, `${program}.ir.json`); + const result = await compile(entry, { outDir: dir, outPath, outputKind: "ir" }); + if (!result.ok) + throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); + return readFile(outPath, "utf8"); +} + +test("function bodies emitted in chunks merge into the text one emitter gives", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-chunked-emission-")); + try { + for (const program of PROGRAMS) { + const ir = await lower(program, dir); + const fresh = (): IrModule => deserializeModule(ir); + for (const [name, options] of Object.entries(OPTIONS)) { + const serial = text(emitLlvmModuleSource(fresh(), options)); + for (const functionChunks of [3, 7]) { + const chunked = text(emitLlvmModuleSource(fresh(), { ...options, functionChunks })); + expect(chunked === serial, `${program}, ${name}, ${functionChunks} chunks`).toBe(true); + } + } + } + } finally { + await rm(dir, { recursive: true, force: true }); + } +}, 600_000); From eace8e1a1766f76fef29bcb1f8ef790f680ce147 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 11:31:36 -0500 Subject: [PATCH 056/197] Give the zod 3 npm case the long per-case timeout --- tests/harness/npm.test.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/harness/npm.test.ts b/tests/harness/npm.test.ts index 2df0d87ef..795e42846 100644 --- a/tests/harness/npm.test.ts +++ b/tests/harness/npm.test.ts @@ -112,9 +112,10 @@ const KNOWN: Record = { const cases = shardSelect(npmCases(fixturesRoot), (c) => c.name); /** Per-case timeouts past the default 120 s: the undici cases compile the - * built-in undici implementation's module graph, which a sanitized build - * on a CI runner takes minutes to compile. */ -const TIMEOUTS: Record = { undici5: 600_000, undici7: 600_000 }; + * built-in undici implementation's module graph, and zod3 compiles zod 3's + * 3,700-line types module. A sanitized build of either takes minutes on a + * CI runner. */ +const TIMEOUTS: Record = { undici5: 600_000, undici7: 600_000, zod3: 600_000 }; interface RunResult { stdout: Buffer; From 972e44a11a530321cdaf38aeeffc68b9639297e3 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 11:40:01 -0500 Subject: [PATCH 057/197] Hold any later value in JavaScript properties initialized to undefined or null A JavaScript object literal's property whose initializer is `undefined` or `null` (`[{ oldPos: -1, lastComponent: undefined }]`, `{ user: null }`) mapped to the unit-only union, so a later write of another value threw: the diff package's diffLines failed with "a '{ added: boolean; count: number; previousComponent: unknown; removed: boolean }' value is not representable in the target union" when it stored its first path component. Nothing checks a JavaScript file's writes, so such a property is now checked-dynamic and holds whatever the program stores. TypeScript properties keep the unit-only union, and `__proto__: null` still sets the prototype. --- packages/compiler/src/frontend/type-mapper.ts | 23 ++++++++++ .../main.ts.json | 6 +++ .../main.ts | 8 ++++ .../node_modules/edit-paths/index.d.ts | 2 + .../node_modules/edit-paths/index.js | 43 +++++++++++++++++++ .../node_modules/edit-paths/package.json | 1 + .../main.ts.json | 1 + 7 files changed, 84 insertions(+) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7008-js-unit-initialized-properties/main.ts.json create mode 100644 tests/corpus/7008-js-unit-initialized-properties/main.ts create mode 100644 tests/corpus/7008-js-unit-initialized-properties/node_modules/edit-paths/index.d.ts create mode 100644 tests/corpus/7008-js-unit-initialized-properties/node_modules/edit-paths/index.js create mode 100644 tests/corpus/7008-js-unit-initialized-properties/node_modules/edit-paths/package.json create mode 100644 tests/harness/corpus-costs/7008-js-unit-initialized-properties/main.ts.json diff --git a/packages/compiler/src/frontend/type-mapper.ts b/packages/compiler/src/frontend/type-mapper.ts index 93349054a..fa8897266 100644 --- a/packages/compiler/src/frontend/type-mapper.ts +++ b/packages/compiler/src/frontend/type-mapper.ts @@ -5744,6 +5744,28 @@ function emptyNeverArrayProp(p: ts.Symbol, fieldTs: ts.Type, checker: ts.TypeChe ); } +/** A JavaScript object literal's property (`{ lastComponent: undefined }`, + * `{ user: null }`). Nothing checks a JavaScript file's later writes, which + * can store any value in it (`path.lastComponent = { ... }`), so a + * property whose only initializer is a unit holds the checked-dynamic value + * rather than the unit-only union. */ +function isUnitOnlyIr(t: IrType, ctx: TypeMapperCtx): boolean { + if (t.kind !== "union") return false; + const def = ctx.unions.get(t.unionId); + return def !== undefined && def.arms.every((arm) => isUnitType(arm)); +} + +function jsObjectLiteralProperty(p: ts.Symbol): boolean { + const d = p.valueDeclaration; + // `__proto__: null` sets the literal's prototype; it declares no property. + return ( + d !== undefined && + (ts.isPropertyAssignment(d) || ts.isShorthandPropertyAssignment(d)) && + ts.symbolName(p) !== "__proto__" && + isJsSourceFile(ts.getSourceFileOfNode(d)) + ); +} + function mapGenericAwaitedAlias(type: ts.Type, ctx: TypeMapperCtx): IrType | null { const { resolveTypeParam } = ctx; if (!resolveTypeParam) return null; @@ -7811,6 +7833,7 @@ function mapRecordTypeInner( // omits the undefined arm exactly like an omitted optional. if (pt?.kind === "void" && isUnitOnlyTsType(fieldTs, ctx.resolveTypeParam)) pt = unitOnlyUnion(ctx.unions); + if (pt !== null && isUnitOnlyIr(pt, ctx) && jsObjectLiteralProperty(p)) pt = DYN; // dyn FIELDS map now (`{ v: unknown }`, `[string, unknown]` tuples): // the slot carries a dyn value exactly like an `unknown`-valued // overflow entry — same RC adapters, same dynFrom conversion on the diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7008-js-unit-initialized-properties/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7008-js-unit-initialized-properties/main.ts.json new file mode 100644 index 000000000..85f5a6e12 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7008-js-unit-initialized-properties/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7008-js-unit-initialized-properties/main.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7008-js-unit-initialized-properties/main.ts b/tests/corpus/7008-js-unit-initialized-properties/main.ts new file mode 100644 index 000000000..3ac113d6a --- /dev/null +++ b/tests/corpus/7008-js-unit-initialized-properties/main.ts @@ -0,0 +1,8 @@ +// A JavaScript package whose object literals start a property as +// `undefined` or `null` and later store records, numbers and arrays in it, +// as the diff package's path list does (`{ oldPos: -1, lastComponent: +// undefined }`). Node runs the later writes; each property holds them. +import { describe, session } from "edit-paths"; + +console.log(describe()); +console.log(session()); diff --git a/tests/corpus/7008-js-unit-initialized-properties/node_modules/edit-paths/index.d.ts b/tests/corpus/7008-js-unit-initialized-properties/node_modules/edit-paths/index.d.ts new file mode 100644 index 000000000..68e2b409b --- /dev/null +++ b/tests/corpus/7008-js-unit-initialized-properties/node_modules/edit-paths/index.d.ts @@ -0,0 +1,2 @@ +export declare function describe(): string; +export declare function session(): string; diff --git a/tests/corpus/7008-js-unit-initialized-properties/node_modules/edit-paths/index.js b/tests/corpus/7008-js-unit-initialized-properties/node_modules/edit-paths/index.js new file mode 100644 index 000000000..ec3e7c4d0 --- /dev/null +++ b/tests/corpus/7008-js-unit-initialized-properties/node_modules/edit-paths/index.js @@ -0,0 +1,43 @@ +"use strict"; +// A Myers-style path list, as the diff package keeps it: each path starts +// with `lastComponent: undefined` and later holds a linked list of records, +// and a state object starts with `null` fields that later hold other values. +function addToPath(path, added, removed, oldPosInc) { + var last = path.lastComponent; + if (last && last.added === added && last.removed === removed) { + return { oldPos: path.oldPos + oldPosInc, lastComponent: { count: last.count + 1, added: added, removed: removed, previousComponent: last.previousComponent } }; + } + return { oldPos: path.oldPos + oldPosInc, lastComponent: { count: 1, added: added, removed: removed, previousComponent: last } }; +} +function extractCommon(basePath, common) { + if (common) basePath.lastComponent = { count: common, previousComponent: basePath.lastComponent, added: false, removed: false }; + basePath.oldPos += common; +} +function components(lastComponent) { + var list = []; + while (lastComponent) { + list.push(lastComponent); + var next = lastComponent.previousComponent; + delete lastComponent.previousComponent; + lastComponent = next; + } + return list.reverse(); +} +function describe() { + var bestPath = [{ oldPos: -1, lastComponent: undefined }]; + extractCommon(bestPath[0], 1); + var path = addToPath(bestPath[0], false, true, 1); + path = addToPath(path, true, false, 0); + path = addToPath(path, true, false, 0); + return components(path.lastComponent).map(function (c) { + return (c.added ? "+" : c.removed ? "-" : "=") + c.count + ("previousComponent" in c ? "!" : ""); + }).join(" "); +} +function session() { + var state = { user: null, retries: null, tags: undefined }; + state.user = { name: "ada" }; + state.retries = 3; + state.tags = ["a", "b"]; + return state.user.name + " " + state.retries + " " + state.tags.join("+") + " " + JSON.stringify(state); +} +module.exports = { describe: describe, session: session }; diff --git a/tests/corpus/7008-js-unit-initialized-properties/node_modules/edit-paths/package.json b/tests/corpus/7008-js-unit-initialized-properties/node_modules/edit-paths/package.json new file mode 100644 index 000000000..98648c289 --- /dev/null +++ b/tests/corpus/7008-js-unit-initialized-properties/node_modules/edit-paths/package.json @@ -0,0 +1 @@ +{ "name": "edit-paths", "version": "1.0.0", "main": "./index.js", "types": "./index.d.ts" } diff --git a/tests/harness/corpus-costs/7008-js-unit-initialized-properties/main.ts.json b/tests/harness/corpus-costs/7008-js-unit-initialized-properties/main.ts.json new file mode 100644 index 000000000..3a6df58d2 --- /dev/null +++ b/tests/harness/corpus-costs/7008-js-unit-initialized-properties/main.ts.json @@ -0,0 +1 @@ +{ "plain": 3065, "san": 2623 } From 7ed4597407f8aa8ab7496a4003c0fc517af71a04 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 11:59:49 -0500 Subject: [PATCH 058/197] Match the speculative loop site global by its numbered-site name in the bail test --- packages/compiler/test/speculative-loop-emission.test.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/compiler/test/speculative-loop-emission.test.ts b/packages/compiler/test/speculative-loop-emission.test.ts index a691d856d..a81348102 100644 --- a/packages/compiler/test/speculative-loop-emission.test.ts +++ b/packages/compiler/test/speculative-loop-emission.test.ts @@ -78,15 +78,14 @@ test("a failed check finishes its iteration outside the ordinary copy", async () test("a loop site stops entering its fast copy after repeated bails", async () => { const llvm = await emitLlvm(source); - const site = /^(@scr\.spec\.bails\.\d+) = internal global i8 0$/m.exec(llvm)?.[1]; + const site = /^(@sc_spec_bails_\d+) = internal global i8 0$/m.exec(llvm)?.[1]; expect(site).toBeDefined(); const fn = /^define internal double @sc_bf_scaled\([^]*?^}/m.exec(llvm)?.[0] ?? ""; - const escaped = site!.replace(/\./g, "\\."); // The entry test reads the count, a bail adds one, and a completed fast // copy clears it. expect(fn).toMatch( new RegExp( - `(%\\S+) = load atomic i8, ptr ${escaped} unordered, align 1\\n\\s*%\\S+ = icmp ult i8 \\1, 8`, + `(%\\S+) = load atomic i8, ptr ${site} unordered, align 1\\n\\s*%\\S+ = icmp ult i8 \\1, 8`, ), ); expect(fn).toMatch(new RegExp(`call i8 @llvm\\.uadd\\.sat\\.i8\\(i8 %\\S+, i8 1\\)`)); From a6427a53c0524d5c28e3872abe6c6d28a1b13cc3 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 12:10:51 -0500 Subject: [PATCH 059/197] Find own toString assignments in the syntax tree, not the file text --- .../src/frontend/lowering/lower-calls.ts | 82 +++++++++++++++++-- .../compiler/src/frontend/lowering/lowerer.ts | 3 + .../npm/cases/js-own-tostring/main.ts | 4 +- .../node_modules/nsutil/dist/index.d.ts | 3 + .../node_modules/nsutil/dist/index.js | 14 ++++ 5 files changed, 99 insertions(+), 7 deletions(-) diff --git a/packages/compiler/src/frontend/lowering/lower-calls.ts b/packages/compiler/src/frontend/lowering/lower-calls.ts index 7e104c2f1..252e85c5a 100644 --- a/packages/compiler/src/frontend/lowering/lower-calls.ts +++ b/packages/compiler/src/frontend/lowering/lower-calls.ts @@ -10977,13 +10977,83 @@ function fenceClassCopyIntoFreshObject( * that function returns, not Object.prototype.toString's string. */ function jsAssignsOwnToString(lowerer: Lowerer, receiver: ts.Expression): boolean { if (!ts.isIdentifier(receiver)) return false; - let sym = lowerer.checker.getSymbolAtLocation(receiver); - if (sym !== undefined && (sym.flags & ts.SymbolFlags.Alias) !== 0) - sym = lowerer.checker.getAliasedSymbol(sym); - const decl = sym?.valueDeclaration ?? sym?.declarations[0]; + const decl = bindingDeclaration(lowerer, receiver); if (decl === undefined) return false; const sf = ts.getSourceFileOfNode(decl); if (!isJsSourceFile(sf)) return false; - const name = ts.symbolName(sym!); - return new RegExp(`\\b${name.replace(/[$]/g, "\\$")}\\.toString\\s*=[^=]`).test(sf.text); + let targets = lowerer.ownToStringTargets.get(sf); + if (targets === undefined) { + targets = ownToStringTargets(lowerer, sf); + lowerer.ownToStringTargets.set(sf, targets); + } + return targets.has(decl); +} + +const unparenthesized = (e: ts.Expression): ts.Expression => { + let out = e; + while (ts.isParenthesizedExpression(out)) out = out.expression; + return out; +}; + +/** The declaration of the binding an identifier names, through imports. */ +function bindingDeclaration(lowerer: Lowerer, id: ts.Identifier): ts.Node | undefined { + let sym = lowerer.checker.getSymbolAtLocation(id); + if (sym !== undefined && (sym.flags & ts.SymbolFlags.Alias) !== 0) + sym = lowerer.checker.getAliasedSymbol(sym); + return sym?.valueDeclaration ?? sym?.declarations?.[0]; +} + +/** The declarations of the bindings a file's assignments `x.toString = ...` + * (or `x["toString"] = ...`) give an own `toString`. An assignment through + * the parameter of an immediately called function counts for the binding + * the call passes (`(function (ns) { ns.toString = f; })(ns || (ns = {}))`). */ +function ownToStringTargets(lowerer: Lowerer, sf: ts.SourceFile): Set { + const targets = new Set(); + const visit = (node: ts.Node): void => { + if (ts.isBinaryExpression(node) && node.operatorToken.kind === ts.SyntaxKind.EqualsToken) { + const left = unparenthesized(node.left); + const named = + (ts.isPropertyAccessExpression(left) && left.name.text === "toString") || + (ts.isElementAccessExpression(left) && + ts.isStringLiteralLike(left.argumentExpression) && + left.argumentExpression.text === "toString"); + const target = named + ? unparenthesized( + (left as ts.PropertyAccessExpression | ts.ElementAccessExpression).expression, + ) + : undefined; + if (target !== undefined && ts.isIdentifier(target)) { + const decl = bindingDeclaration(lowerer, target); + if (decl !== undefined) { + targets.add(decl); + const passed = iifeArgumentDeclaration(lowerer, decl); + if (passed !== undefined) targets.add(passed); + } + } + } + ts.forEachChild(node, visit); + }; + visit(sf); + return targets; +} + +/** For a parameter of an immediately called function, the declaration of + * the binding its argument names: `x`, `x || (x = {})` or `(x = {})`. */ +function iifeArgumentDeclaration(lowerer: Lowerer, param: ts.Node): ts.Node | undefined { + if (!ts.isParameter(param)) return undefined; + const fn = param.parent; + if (!ts.isFunctionExpression(fn) && !ts.isArrowFunction(fn)) return undefined; + let callee: ts.Node = fn; + while (ts.isParenthesizedExpression(callee.parent)) callee = callee.parent; + const call = callee.parent; + if (!ts.isCallExpression(call) || call.expression !== callee) return undefined; + const index = fn.parameters.nodes.indexOf(param); + let arg = index < 0 ? undefined : call.arguments.nodes[index]; + if (arg === undefined) return undefined; + arg = unparenthesized(arg); + if (ts.isBinaryExpression(arg) && arg.operatorToken.kind === ts.SyntaxKind.BarBarToken) + arg = unparenthesized(arg.left); + if (ts.isBinaryExpression(arg) && arg.operatorToken.kind === ts.SyntaxKind.EqualsToken) + arg = unparenthesized(arg.left); + return ts.isIdentifier(arg) ? bindingDeclaration(lowerer, arg) : undefined; } diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index b02a0a843..7e5e93e51 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -1918,6 +1918,9 @@ export class Lowerer { /** Identifiers by spelling, per file, for the generic bindings' use * ordering (runsAfterDeclaration). */ readonly bindingIdentifiers = new Map>(); + /** Per JavaScript file, the declarations of the bindings it assigns an + * own `toString` (jsAssignsOwnToString in lower-calls.ts). */ + readonly ownToStringTargets = new Map>(); /** Per-symbol result of the never-reassigned file scan * (bindingNeverReassigned — object-literal generic-method receivers). */ readonly neverReassignedCache = new Map(); diff --git a/tests/fixtures/npm/cases/js-own-tostring/main.ts b/tests/fixtures/npm/cases/js-own-tostring/main.ts index 279e2fd66..9793b2baa 100644 --- a/tests/fixtures/npm/cases/js-own-tostring/main.ts +++ b/tests/fixtures/npm/cases/js-own-tostring/main.ts @@ -2,11 +2,13 @@ // namespace object assigns its own `toString` returning string | undefined; // Number.parseInt with one argument; and a `then` whose handler result the // declared union types (zod's ParseStatus.mergeArray pattern). -import { check, direct, mergeLater, wholePart } from "nsutil"; +import { check, direct, mergeLater, note, show, showRadix, wholePart } from "nsutil"; console.log(String(direct("a")), String(direct({ message: "b" })), String(direct(undefined))); console.log(JSON.stringify(check(undefined)), JSON.stringify(check("m"))); console.log(wholePart(1.23), wholePart(0.35), wholePart(-4.5), wholePart(12)); +const values: unknown[] = [12, "s", true, 1.5]; +console.log(values.map((value) => show(value)).join(" "), showRadix(5, 2), note.length); async function main(): Promise { console.log(JSON.stringify(await mergeLater([Promise.resolve(1), Promise.resolve("two")], { ok: true }))); console.log(JSON.stringify(await mergeLater([Promise.resolve(3), Promise.resolve("four")], { ok: false }))); diff --git a/tests/fixtures/npm/cases/js-own-tostring/node_modules/nsutil/dist/index.d.ts b/tests/fixtures/npm/cases/js-own-tostring/node_modules/nsutil/dist/index.d.ts index f4eea833a..18f135cad 100644 --- a/tests/fixtures/npm/cases/js-own-tostring/node_modules/nsutil/dist/index.d.ts +++ b/tests/fixtures/npm/cases/js-own-tostring/node_modules/nsutil/dist/index.d.ts @@ -4,3 +4,6 @@ export declare function direct(message?: errorUtil.ErrMessage): string | undefin export declare function wholePart(value: number): number; export type Merged = { status: "valid"; value: unknown[] } | { status: "aborted" }; export declare function mergeLater(items: Promise[], status: { ok: boolean }): Promise; +export declare const note: string; +export declare function show(value: unknown): string; +export declare function showRadix(value: number, radix: number): string; diff --git a/tests/fixtures/npm/cases/js-own-tostring/node_modules/nsutil/dist/index.js b/tests/fixtures/npm/cases/js-own-tostring/node_modules/nsutil/dist/index.js index 272a4631c..105187f49 100644 --- a/tests/fixtures/npm/cases/js-own-tostring/node_modules/nsutil/dist/index.js +++ b/tests/fixtures/npm/cases/js-own-tostring/node_modules/nsutil/dist/index.js @@ -12,3 +12,17 @@ export function mergeLater(items, status) { return Promise.all(items).then((results) => status.ok ? { status: "valid", value: results } : INVALID); } const INVALID = Object.freeze({ status: "aborted" }); +// Only an assignment to this binding's own toString counts: not this +// comment's value.toString = (x) => x, a string, or another function's +// parameter of the same name. +export const note = "value.toString = (x) => x"; +export function show(value) { + return value.toString(); +} +export function showRadix(value, radix) { + return value.toString(radix); +} +export function patch(value) { + value.toString = () => "patched"; + return value; +} From 83d426f9be4c4d45fd3131a7d3756ba8b8e3acb3 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 12:14:23 -0500 Subject: [PATCH 060/197] Support FinalizationRegistry with bookkeeping and no cleanup callbacks wasm-bindgen output constructs a FinalizationRegistry per exported class when its module loads, so every wasm-bindgen package stopped at load with SC2020. `new FinalizationRegistry(cleanup)` now creates a native registry: - register(target, held, token) and unregister(token) check their arguments and answer as Node does, with Node's TypeError messages; - the unregister tokens are keys of an internal weak map, so the registry keeps no token alive; - the cleanup callback never runs, which ECMAScript allows, so the registry keeps neither the callback nor the held values. The global answers typeof "function", instanceof, Object.prototype.toString and util.inspect as Node does. Symbol targets and tokens are refused with SC2020. `"encodeInto" in encoder` on a TextEncoder, which wasm-bindgen's loaders test at load, answered false because the encoder is a brand record without fields for its methods. `in` over such a builtin brand record now answers from the members the standard library declares, plus Object.prototype's. The limitations page says that memory a package frees only from a cleanup callback, such as the WebAssembly memory behind wasm-bindgen wrappers the program drops without free(), is not reclaimed until the program exits. Fixes #1678. --- docs/content/docs/contracts.mdx | 2 +- .../weak-collections-and-memory.mdx | 9 ++ .../compiler/src/backend/llvm/lib-dispatch.ts | 1 + .../compiler/src/backend/llvm/lib-shared.ts | 2 + packages/compiler/src/contracts/check.ts | 7 +- .../lowering/expressions/identifiers.ts | 2 - .../lowering/expressions/property-access.ts | 1 + .../lowering/expressions/relational.ts | 30 ++++ .../src/frontend/lowering/lower-calls.ts | 3 + .../src/frontend/lowering/lower-classes.ts | 41 ++++- packages/compiler/src/frontend/type-mapper.ts | 2 + packages/compiler/src/ir/builtin-effects.ts | 2 + .../compiler/src/ir/builtin-signatures.ts | 2 + .../main.ts.json | 6 + packages/runtime/src/scr_dyn.h | 3 + packages/runtime/src/scr_dyn_invoke.c | 7 +- packages/runtime/src/scr_json.c | 145 +++++++++++++++++- .../main.ts | 37 +++++ .../counter-wasm/counter_wasm.d.ts | 9 ++ .../node_modules/counter-wasm/counter_wasm.js | 61 ++++++++ .../node_modules/counter-wasm/package.json | 1 + .../main.ts.json | 1 + 22 files changed, 362 insertions(+), 12 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7009-finalization-registry-wasm-loader/main.ts.json create mode 100644 tests/corpus/7009-finalization-registry-wasm-loader/main.ts create mode 100644 tests/corpus/7009-finalization-registry-wasm-loader/node_modules/counter-wasm/counter_wasm.d.ts create mode 100644 tests/corpus/7009-finalization-registry-wasm-loader/node_modules/counter-wasm/counter_wasm.js create mode 100644 tests/corpus/7009-finalization-registry-wasm-loader/node_modules/counter-wasm/package.json create mode 100644 tests/harness/corpus-costs/7009-finalization-registry-wasm-loader/main.ts.json diff --git a/docs/content/docs/contracts.mdx b/docs/content/docs/contracts.mdx index 5af1cad2f..268f5592b 100644 --- a/docs/content/docs/contracts.mdx +++ b/docs/content/docs/contracts.mdx @@ -360,7 +360,7 @@ A class can be both `ProgramLifetime` and `Acyclic`. `Acyclic` is still checked Instances can be published to other threads with `@scriptc/threads` like other class instances: `publish()` makes them immutable and shares them by reference, writes to a published instance throw the same `TypeError` as under Node.js, and instances that were not published stay writable and are cloned by `postMessage` and `workerData`. -Static code has no `WeakRef` or `FinalizationRegistry`, and a `WeakMap` or `WeakSet` entry whose key is never freed is never observable as removed, so nothing else can tell the difference. +Static code has no `WeakRef`, a `FinalizationRegistry` never runs its cleanup callbacks, and a `WeakMap` or `WeakSet` entry whose key is never freed is never observable as removed, so nothing else can tell the difference. ### Lifetime regions diff --git a/docs/content/limitations/weak-collections-and-memory.mdx b/docs/content/limitations/weak-collections-and-memory.mdx index 97ee20d7b..21c6128d9 100644 --- a/docs/content/limitations/weak-collections-and-memory.mdx +++ b/docs/content/limitations/weak-collections-and-memory.mdx @@ -3,6 +3,15 @@ Copied values, other native handles, and symbol keys are unsupported. Constructor seeds support arrays; custom iterators and detached methods are unsupported. Weak collections do not retain their keys, but values that refer back to a key can keep it alive until deletion or collection release. +`FinalizationRegistry` keeps its registrations: `register` and `unregister` check their arguments and answer as in Node.js. +Its cleanup callbacks never run, which the ECMAScript specification allows. +Program output matches Node.js, which runs cleanup callbacks only when its garbage collector decides to. +Memory that a package frees only from a cleanup callback is not reclaimed until the program exits. +For example, a wasm-bindgen package frees the WebAssembly memory behind a wrapper object from its cleanup callback, so a long-running server that drops such wrappers without calling their `free()` method keeps that memory. +Call `free()` on wrappers the program no longer needs. +Symbol targets and unregister tokens are unsupported. +`WeakRef` is unsupported. + Memory uses reference counting. Acyclic values are freed deterministically, and cycles are collected at defined collection points. Native checked objects, arrays, Sets, and captured closures participate in cycle collection. diff --git a/packages/compiler/src/backend/llvm/lib-dispatch.ts b/packages/compiler/src/backend/llvm/lib-dispatch.ts index 24f2bf019..5e2b0217c 100644 --- a/packages/compiler/src/backend/llvm/lib-dispatch.ts +++ b/packages/compiler/src/backend/llvm/lib-dispatch.ts @@ -858,6 +858,7 @@ export function emitLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlValue { return host.emitGenericLibCall(e); case "weakMap": case "weakSet": + case "finalizationRegistry": case "ffi": case "sharedArrayBuffer": case "arrayBuffer": diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index 9a65d9a24..76d7b690c 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -44,6 +44,8 @@ export const LIB_FN_SYMS: Record = { "weakSet.is": "scr_weak_set_is", "weakMap.new": "scr_weak_map_new", "weakSet.new": "scr_weak_set_new", + "finalizationRegistry.new": "scr_finalization_registry_new", + "finalizationRegistry.is": "scr_finalization_registry_is", "dyn.fromEntries": "scr_dyn_from_entries", "bytes.constructor": "scr_bytes_constructor", "bytes.construct": "scr_bytes_construct", diff --git a/packages/compiler/src/contracts/check.ts b/packages/compiler/src/contracts/check.ts index c80468961..fe5dc171e 100644 --- a/packages/compiler/src/contracts/check.ts +++ b/packages/compiler/src/contracts/check.ts @@ -285,9 +285,10 @@ class ContractChecker { * the lifetimes of the objects they receive). Publication to other * threads needs no check: an instance carries a reference count of its * own until publication marks it like any other object it reaches - * (classes.ts programLifetimeRc). Static code has no WeakRef or - * FinalizationRegistry; weak collections stay correct because an entry - * keyed by an object that is never freed is never observable as removed. */ + * (classes.ts programLifetimeRc). Static code has no WeakRef, and a + * FinalizationRegistry never runs its cleanup callbacks; weak collections + * stay correct because an entry keyed by an object that is never freed is + * never observable as removed. */ private checkProgramLifetime(): void { if (this.notes.classes.size === 0) return; const roots: string[] = []; diff --git a/packages/compiler/src/frontend/lowering/expressions/identifiers.ts b/packages/compiler/src/frontend/lowering/expressions/identifiers.ts index 1f74cea42..fc432b503 100644 --- a/packages/compiler/src/frontend/lowering/expressions/identifiers.ts +++ b/packages/compiler/src/frontend/lowering/expressions/identifiers.ts @@ -789,8 +789,6 @@ export function lowerIdentifierExpr(lowerer: Lowerer, expr: ts.Identifier, loc: "ArrayBuffer constructor values have no native lowering; use new ArrayBuffer(length) for fixed-length shared storage", WeakRef: "deref()-after-collect exposes GC timing — genuinely dynamic; hold a strong reference instead", - FinalizationRegistry: - "finalization callbacks expose GC timing — genuinely dynamic; release resources explicitly instead", eval: "runtime code evaluation cannot be compiled ahead of time", }; lowerer.noLowering(expr.text, expr, globalHints[expr.text], sym ?? undefined); diff --git a/packages/compiler/src/frontend/lowering/expressions/property-access.ts b/packages/compiler/src/frontend/lowering/expressions/property-access.ts index 7f3910c15..6a32c0b99 100644 --- a/packages/compiler/src/frontend/lowering/expressions/property-access.ts +++ b/packages/compiler/src/frontend/lowering/expressions/property-access.ts @@ -4120,6 +4120,7 @@ const GLOBAL_CONSTRUCTOR_VALUES: ReadonlySet = new Set([ "Promise", "WeakMap", "WeakSet", + "FinalizationRegistry", ...ERROR_CONSTRUCTOR_VALUES, ]); diff --git a/packages/compiler/src/frontend/lowering/expressions/relational.ts b/packages/compiler/src/frontend/lowering/expressions/relational.ts index 61558f008..7ab328120 100644 --- a/packages/compiler/src/frontend/lowering/expressions/relational.ts +++ b/packages/compiler/src/frontend/lowering/expressions/relational.ts @@ -147,6 +147,10 @@ export function lowerInstanceOf(lowerer: Lowerer, expr: ts.BinaryExpression, loc loc, }; } + if (lowerer.isStdlibGlobal(expr.right, "FinalizationRegistry")) { + const value = lowerer.coerceInto(expr.left, lowerer.lowerExpr(expr.left), DYN); + return { kind: "libCall", fn: "finalizationRegistry.is", args: [value], type: BOOL, loc }; + } const rhsBuiltin = ts.isIdentifier(expr.right) ? lowerer.builtinImportOf(expr.right) : ts.isPropertyAccessExpression(expr.right) @@ -1493,6 +1497,32 @@ export function lowerInExpression( ); if (helper) recv = { kind: "call", callee: helper, args: [recv], type: siteType, loc }; } + // A builtin class held as a brand record (TextEncoder, TextDecoder: only + // `%`-prefixed internal fields): its members are the class's prototype + // members, which the standard library declares, so `"encodeInto" in + // encoder` answers from the checker's type, plus Object.prototype's keys. + // The receiver evaluates for its effects. + if (recv.type.kind === "record") { + const shape = lowerer.shapes.get(recv.type.shapeId); + if (shape && shape.fields.length > 0 && shape.fields.every((f) => f.name.startsWith("%"))) { + const declared = + lowerer.checker.getPropertyOfType(lowerer.typeOf(expr.right), key) !== undefined; + const answer: IrExpr = { + kind: "boolLit", + value: declared || OBJECT_PROTOTYPE_KEYS.has(key), + type: BOOL, + loc, + }; + if (recv.kind === "varRef") return answer; + return { + kind: "seqExpr", + stmts: [{ kind: "exprStmt", expr: recv, loc }], + result: answer, + type: BOOL, + loc, + }; + } + } // Error-rooted receivers (builtin or user subclass — the isErrnoException // predicate's `err instanceof Error && "code" in err` shape): `code` // answers from the runtime error's code slot (stamped by fs/system/ diff --git a/packages/compiler/src/frontend/lowering/lower-calls.ts b/packages/compiler/src/frontend/lowering/lower-calls.ts index 39b7207c0..9378ccbb6 100644 --- a/packages/compiler/src/frontend/lowering/lower-calls.ts +++ b/packages/compiler/src/frontend/lowering/lower-calls.ts @@ -4318,6 +4318,9 @@ const DYN_DISPATCH_METHODS = new Set([ "additionalHeaders", "altsvc", "origin", + // FinalizationRegistry's methods (scr_finalization_registry_new). + "register", + "unregister", // Worker and MessagePort methods use the owning context's handle adapter. "postMessage", "terminate", diff --git a/packages/compiler/src/frontend/lowering/lower-classes.ts b/packages/compiler/src/frontend/lowering/lower-classes.ts index d8ccc15b7..307b3d508 100644 --- a/packages/compiler/src/frontend/lowering/lower-classes.ts +++ b/packages/compiler/src/frontend/lowering/lower-classes.ts @@ -10394,6 +10394,45 @@ export function lowerNew(lowerer: Lowerer, expr: ts.NewExpression): IrExpr { loc, }; } + // A FinalizationRegistry construction: register and unregister keep + // their bookkeeping, and the cleanup callback never runs, which + // ECMAScript allows (scr_finalization_registry_new). + if ( + symbol !== null && + ts.symbolName(symbol) === "FinalizationRegistry" && + lowerer.isStdlibSymbol(symbol) + ) { + const args = expr.arguments?.nodes ?? []; + if (args.some(ts.isSpreadElement)) { + lowerer.unsupported( + "SC1090", + expr, + "FinalizationRegistry construction with spread arguments", + ); + } + const cleanup = args[0] ? lowerer.lowerExprExpecting(args[0], DYN) : dynUndefinedExpr(loc); + const surplus = args.slice(1).map((arg) => lowerer.lowerExpr(arg)); + const created: IrExpr = { + kind: "libCall", + fn: "finalizationRegistry.new", + args: [cleanup], + type: DYN, + loc, + }; + if (surplus.length === 0) return created; + // Surplus arguments evaluate after the cleanup argument and are ignored. + const local = lowerer.declareHiddenLocal("%finalizationCleanup", DYN); + return { + kind: "seqExpr", + stmts: [ + { kind: "varDecl", localId: local.id, init: cleanup, loc }, + ...surplus.map((value): IrStmt => ({ kind: "exprStmt", expr: value, loc })), + ], + result: { ...created, args: [varRef(local.id, DYN, loc)] }, + type: DYN, + loc, + }; + } // `new Set()`: construct + bulk add from supported values, // including live collection cursors. Duplicates collapse in insertion order. if (ts.isIdentifier(identifier) && lowerer.isStdlibGlobal(identifier, "Set")) { @@ -10605,8 +10644,6 @@ export function lowerNew(lowerer: Lowerer, expr: ts.NewExpression): IrExpr { Boolean: "boxed wrapper objects have no lowering — use the boolean primitive", WeakRef: "deref()-after-collect exposes GC timing — genuinely dynamic; hold a strong reference instead", - FinalizationRegistry: - "finalization callbacks expose GC timing — genuinely dynamic; release resources explicitly instead", SharedArrayBuffer: "no shared-memory threads exist in a compiled program — Uint8Array is the byte storage", ArrayBuffer: diff --git a/packages/compiler/src/frontend/type-mapper.ts b/packages/compiler/src/frontend/type-mapper.ts index bd2a32c37..25ecac04a 100644 --- a/packages/compiler/src/frontend/type-mapper.ts +++ b/packages/compiler/src/frontend/type-mapper.ts @@ -3155,6 +3155,8 @@ function mapTypeInner(type: ts.Type, ctx: TypeMapperCtx): IrType | null { // Native WeakMap keeps reference identities in checked-dynamic storage. // The runtime rejects reference kinds without a weak lifetime contract. if (isStdlibInterface("WeakMap") || isStdlibInterface("WeakSet")) return DYN; + // FinalizationRegistry is a native handle with register and unregister. + if (isStdlibInterface("FinalizationRegistry")) return DYN; // Set: Map's sibling — same provenance rule and key domain. // Anything else stays unmapped; // the `new Set` lowering names the offending element type specifically. diff --git a/packages/compiler/src/ir/builtin-effects.ts b/packages/compiler/src/ir/builtin-effects.ts index 16db45b6c..b44d73622 100644 --- a/packages/compiler/src/ir/builtin-effects.ts +++ b/packages/compiler/src/ir/builtin-effects.ts @@ -88,6 +88,8 @@ export const MAY_THROW_LIB_FNS: ReadonlySet = new Set([ "stream.unpipeDyn", "weakMap.new", "weakSet.new", + // Node's "cleanup must be callable" TypeError. + "finalizationRegistry.new", "dyn.fromEntries", "bytes.constructor", "bytes.instanceOf", diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index fcdbe8694..dd1aec3aa 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -3591,6 +3591,8 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ "weakSet.is": { argTypes: [DYN], result: BOOL }, "weakMap.new": { argTypes: [DYN], result: DYN }, "weakSet.new": { argTypes: [DYN], result: DYN }, + "finalizationRegistry.new": { argTypes: [DYN], result: DYN }, + "finalizationRegistry.is": { argTypes: [DYN], result: BOOL }, "dyn.fromEntries": { argTypes: [DYN], result: DYN }, "bytes.constructor": { argTypes: [STRING], result: DYN }, "bytes.instanceOf": { argTypes: [DYN, DYN], result: BOOL }, diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7009-finalization-registry-wasm-loader/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7009-finalization-registry-wasm-loader/main.ts.json new file mode 100644 index 000000000..89ee292b3 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7009-finalization-registry-wasm-loader/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7009-finalization-registry-wasm-loader/main.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_dyn.h b/packages/runtime/src/scr_dyn.h index bb3b06847..97db18618 100644 --- a/packages/runtime/src/scr_dyn.h +++ b/packages/runtime/src/scr_dyn.h @@ -169,6 +169,7 @@ typedef enum { X(FETCH_RESPONSE, 12) /* native static-fetch Response */ \ X(FILE, 42) /* ScrBlob with a name — new File() */ \ X(FILE_HANDLE, 17) /* fs/promises.FileHandle */ \ + X(FINALIZATION_REGISTRY, 72) /* FinalizationRegistry (bookkeeping only) */ \ X(GLOBAL, 22) /* native global object identity */ \ X(H2_SESSION, 4) /* ScrH2Session — Http2Session (client & server) */ \ X(H2_STREAM, 5) /* ScrH2Stream — Http2Stream (client & server) */ \ @@ -1066,6 +1067,8 @@ ScrDyn *scr_dyn_map_seed_entry(const ScrDyn *entry); bool scr_weak_map_is(const ScrDyn *value); bool scr_weak_set_is(const ScrDyn *value); ScrDyn *scr_weak_map_new(ScrDyn *entries); +ScrDyn *scr_finalization_registry_new(ScrDyn *cleanup); +bool scr_finalization_registry_is(const ScrDyn *value); ScrDyn *scr_weak_set_new(ScrDyn *values); ScrDyn *scr_dyn_from_entries(ScrDyn *entries); ScrDyn *scr_dyn_mark_snapshot(ScrDyn *value); /* consumes and returns +1 */ diff --git a/packages/runtime/src/scr_dyn_invoke.c b/packages/runtime/src/scr_dyn_invoke.c index 80e01b08c..bfe4fc9f7 100644 --- a/packages/runtime/src/scr_dyn_invoke.c +++ b/packages/runtime/src/scr_dyn_invoke.c @@ -1854,9 +1854,10 @@ static const char dyn_s29[] SCR_DYN_COMPAT_STR = "Date"; static const char dyn_s30[] SCR_DYN_COMPAT_STR = "Map"; static const char dyn_s31[] SCR_DYN_COMPAT_STR = "Set"; static const char dyn_s32[] SCR_DYN_COMPAT_STR = ""; +static const char dyn_s33[] SCR_DYN_COMPAT_STR = "FinalizationRegistry"; static const char *const dyn_global_ctor_names[] = { - dyn_s26, dyn_s29, dyn_s25, dyn_s30, dyn_s31, dyn_s21, dyn_s22, dyn_s24, + dyn_s26, dyn_s29, dyn_s25, dyn_s30, dyn_s31, dyn_s21, dyn_s22, dyn_s24, dyn_s33, }; #define DYN_GLOBAL_CTORS (sizeof dyn_global_ctor_names / sizeof dyn_global_ctor_names[0]) static SCR_TL ScrDyn *dyn_global_ctors[DYN_GLOBAL_CTORS]; @@ -1896,8 +1897,8 @@ static ScrDyn *dyn_global_ctor(const char *name) { } if (!dyn_global_ctors[i]) { /* Each constructor's own length: Object 1, Date 7, RegExp 2, Promise 1, - * the collections 0. */ - uint32_t length = i == 0 || i == 5 ? 1 : i == 1 ? 7 : i == 2 ? 2 : 0; + * the collections 0, FinalizationRegistry 1. */ + uint32_t length = i == 0 || i == 5 || i == 8 ? 1 : i == 1 ? 7 : i == 2 ? 2 : 0; dyn_global_ctors[i] = scr_dyn_new_func(scr_closure_new((void *)dyn_global_ctor_names[i], 0), dyn_global_ctor_call, length, dyn_s6, dyn_global_ctor_names[i]); diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index 1c85a928a..44416efc6 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -3964,6 +3964,7 @@ static int scr_dyn_global_constructor_instanceof(const ScrDyn *value, const ScrD static const struct { const char *name; int tag; } kinds[] = { {"Date", SCR_DYNH_DATE}, {"RegExp", SCR_DYNH_REGEXP}, {"Map", SCR_DYNH_MAP}, {"Set", SCR_DYNH_SET}, {"WeakMap", SCR_DYNH_WEAK_MAP}, {"WeakSet", SCR_DYNH_WEAK_SET}, + {"FinalizationRegistry", SCR_DYNH_FINALIZATION_REGISTRY}, }; if (strcmp(callee->v.fn.name, "Promise") == 0) return value->kind == SCR_DYN_PROMISE; for (size_t i = 0; i < sizeof kinds / sizeof kinds[0]; i++) { @@ -7030,6 +7031,7 @@ ScrStr *scr_dyn_object_tag(const ScrDyn *d) { d->v.handle.tag <= SCR_DYNH_ABORT_CONTROLLER) || d->v.handle.tag == SCR_DYNH_EVENT_TARGET || d->v.handle.tag == SCR_DYNH_CUSTOM_EVENT || d->v.handle.tag == SCR_DYNH_ARRAY_BUFFER || d->v.handle.tag == SCR_DYNH_SHARED_ARRAY_BUFFER || d->v.handle.tag == SCR_DYNH_WEAK_MAP || d->v.handle.tag == SCR_DYNH_WEAK_SET || + d->v.handle.tag == SCR_DYNH_FINALIZATION_REGISTRY || d->v.handle.tag == SCR_DYNH_SET || d->v.handle.tag == SCR_DYNH_MAP || d->v.handle.tag == SCR_DYNH_REGEXP || d->v.handle.tag == SCR_DYNH_DATE || d->v.handle.tag == SCR_DYNH_BLOB || d->v.handle.tag == SCR_DYNH_FILE || d->v.handle.tag == SCR_DYNH_DISPOSABLE_STACK) { @@ -14417,7 +14419,8 @@ static int scr_weak_key(const ScrDyn *key, void **ptr, unsigned *kind) { *kind = key->v.fn.class_obj ? 3 : 2; return 1; case SCR_DYN_HANDLE: if (key->v.handle.tag != SCR_DYNH_ARRAY_BUFFER && key->v.handle.tag != SCR_DYNH_SHARED_ARRAY_BUFFER && key->v.handle.tag != SCR_DYNH_WEAK_MAP && - key->v.handle.tag != SCR_DYNH_WEAK_SET && key->v.handle.tag != SCR_DYNH_STDIO && + key->v.handle.tag != SCR_DYNH_WEAK_SET && key->v.handle.tag != SCR_DYNH_FINALIZATION_REGISTRY && + key->v.handle.tag != SCR_DYNH_STDIO && key->v.handle.tag != SCR_DYNH_FETCH_REQUEST && key->v.handle.tag != SCR_DYNH_FETCH_RESPONSE && key->v.handle.tag != SCR_DYNH_MAP && key->v.handle.tag != SCR_DYNH_SET && key->v.handle.tag != SCR_DYNH_NET_SOCKET && key->v.handle.tag != SCR_DYNH_NET_SERVER) return -1; @@ -14596,6 +14599,146 @@ bool scr_weak_set_is(const ScrDyn *value) { ScrDyn *scr_weak_map_new(ScrDyn *seed) { return scr_weak_collection_new(seed, false); } ScrDyn *scr_weak_set_new(ScrDyn *seed) { return scr_weak_collection_new(seed, true); } +/* FinalizationRegistry. register() and unregister() validate and keep their + * bookkeeping as Node does; the cleanup callback never runs, which + * ECMAScript allows (an implementation need not collect a target or call + * its cleanup), so the registry keeps neither the callback nor the held + * values. The unregister tokens are keys of an internal WeakMap counting + * each token's registrations, so the registry keeps no token alive. */ +typedef struct { + size_t rc; + ScrDyn *tokens; /* WeakMap: token -> registration count */ +} ScrFinalizationRegistry; + +static void *scr_finalization_registry_retain(void *ptr) { + ((ScrFinalizationRegistry *)ptr)->rc++; + return ptr; +} + +static void scr_finalization_registry_release(void *ptr) { + ScrFinalizationRegistry *registry = ptr; + if (--registry->rc) return; + /* Weak collections keyed by the registry drop their entries. */ + scr_weak_dispose(registry); + scr_dyn_release(registry->tokens); + free(registry); +} + +static void scr_finalization_type_error(const char *message) { + scr_throw_error_msg(SCR_ERR_TYPE, message, strlen(message)); +} + +/* The weak-key identity of a registration target or token, or false with + * the TypeError (or the SC2020 refusal) pending. */ +static bool scr_finalization_key(const ScrDyn *value, void **ptr, unsigned *kind, + const char *invalid) { + if (value->kind == SCR_DYN_SYMBOL) { + static const char message[] = "FinalizationRegistry with symbol targets or tokens is not supported yet"; + scr_throw_error_msg_code(SCR_ERR_ERROR, message, sizeof message - 1, "SC2020"); + return false; + } + int supported = scr_weak_key(value, ptr, kind); + if (supported < 0) { + static const char message[] = "FinalizationRegistry targets of this native reference type have no weak lifetime lowering"; + scr_throw_error_msg_code(SCR_ERR_ERROR, message, sizeof message - 1, "SC2020"); + return false; + } + if (!supported) { + if (invalid) { + scr_finalization_type_error(invalid); + } else { + /* Node's unregister message quotes the token as String() renders it. */ + ScrStr *text = scr_dyn_string_coerce(value); + ScrJsonBuf b; + scr_jb_init(&b); + scr_jb_puts(&b, "Invalid unregisterToken ('"); + if (text) scr_jb_write(&b, text->data, text->len); + scr_jb_puts(&b, "')"); + scr_str_release(text); + ScrStr *message = scr_jb_finish(&b); + scr_throw_error_msg(SCR_ERR_TYPE, message->data, message->len); + scr_str_release(message); + } + return false; + } + return true; +} + +static ScrDyn *scr_finalization_registry_invoke(void *ptr, ScrDyn *self, const char *method, + ScrDyn *const *args, size_t argc, const char *what) { + (void)self; + ScrFinalizationRegistry *registry = ptr; + ScrWeakMap *tokens = registry->tokens->v.handle.ptr; + ScrDyn *first = argc > 0 ? args[0] : scr_dyn_undefined(); + void *identity = NULL; + unsigned kind = 0; + if (strcmp(method, "register") == 0) { + ScrDyn *held = argc > 1 ? args[1] : scr_dyn_undefined(); + ScrDyn *token = argc > 2 ? args[2] : scr_dyn_undefined(); + if (!scr_finalization_key(first, &identity, &kind, "FinalizationRegistry.prototype.register: invalid target")) + return NULL; + if (scr_dyn_strict_eq(first, held)) { + scr_finalization_type_error("FinalizationRegistry.prototype.register: target and holdings must not be same"); + return NULL; + } + if (token->kind != SCR_DYN_UNDEF) { + if (!scr_finalization_key(token, &identity, &kind, NULL)) return NULL; + ScrWeakEntry *entry = scr_weak_find(tokens, identity, kind); + double count = entry && entry->value->kind == SCR_DYN_NUM ? entry->value->v.num : 0; + ScrDyn *next = scr_dyn_new_num(count + 1); + bool ok = scr_weak_set(tokens, token, next); + scr_dyn_release(next); + if (!ok) return NULL; + } + return scr_dyn_retain(scr_dyn_undefined()); + } + if (strcmp(method, "unregister") == 0) { + if (!scr_finalization_key(first, &identity, &kind, NULL)) return NULL; + ScrWeakEntry *entry = scr_weak_find(tokens, identity, kind); + if (entry) scr_weak_remove(entry); + return scr_dyn_new_bool(entry != NULL); + } + ScrJsonBuf b; + scr_jb_init(&b); + scr_jb_puts(&b, what ? what : method); + scr_jb_puts(&b, " is not a function"); + ScrStr *message = scr_jb_finish(&b); + scr_throw_error_msg(SCR_ERR_TYPE, message->data, message->len); + scr_str_release(message); + return NULL; +} + +static ScrDyn *scr_finalization_registry_view(void *ptr) { + (void)ptr; + return scr_dyn_new_obj(); +} + +ScrDyn *scr_finalization_registry_new(ScrDyn *cleanup) { + if (!scr_dyn_is_callable(cleanup)) { + scr_finalization_type_error("FinalizationRegistry: cleanup must be callable"); + return NULL; + } + static const ScrDynHandleOps ops = { + .cls = "FinalizationRegistry", + .retain = &scr_finalization_registry_retain, + .release = &scr_finalization_registry_release, + .invoke = &scr_finalization_registry_invoke, + .inspect_view = &scr_finalization_registry_view, + .inspect_view_empty = true, + }; + scr_dyn_handle_install(SCR_DYNH_FINALIZATION_REGISTRY, &ops); + ScrFinalizationRegistry *registry = calloc(1, sizeof *registry); + if (!registry) scr_json_oom(); + registry->tokens = scr_weak_collection_new(scr_dyn_undefined(), false); + if (!registry->tokens) { free(registry); return NULL; } + ScrDyn *result = scr_dyn_new_handle(registry, SCR_DYNH_FINALIZATION_REGISTRY); + return result; +} + +bool scr_finalization_registry_is(const ScrDyn *value) { + return value->kind == SCR_DYN_HANDLE && value->v.handle.tag == SCR_DYNH_FINALIZATION_REGISTRY; +} + ScrDyn *scr_dyn_from_entries(ScrDyn *seed) { if (seed->kind == SCR_DYN_UNDEF || seed->kind == SCR_DYN_NULL) { static const char message[] = "undefined is not iterable"; diff --git a/tests/corpus/7009-finalization-registry-wasm-loader/main.ts b/tests/corpus/7009-finalization-registry-wasm-loader/main.ts new file mode 100644 index 000000000..d792f72d4 --- /dev/null +++ b/tests/corpus/7009-finalization-registry-wasm-loader/main.ts @@ -0,0 +1,37 @@ +// A package loaded the way wasm-bindgen's web target loads: FinalizationRegistry +// guards and registrations run at load and in constructors, and free() +// unregisters. Registrations keep their bookkeeping; cleanup +// callbacks run at times only the garbage collector decides, so this program +// prints nothing from them, as under Node. +import { Counter, initSync, released_total } from "counter-wasm"; + +// (module (func (export "add") (param i32 i32) (result i32) local.get 0 local.get 1 i32.add)) +const bytes = new Uint8Array([ + 0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00, 0x01, 0x07, 0x01, 0x60, 0x02, 0x7f, 0x7f, 0x01, + 0x7f, 0x03, 0x02, 0x01, 0x00, 0x07, 0x07, 0x01, 0x03, 0x61, 0x64, 0x64, 0x00, 0x00, 0x0a, 0x09, + 0x01, 0x07, 0x00, 0x20, 0x00, 0x20, 0x01, 0x6a, 0x0b, +]); +initSync(bytes); + +const a = new Counter(10); +a.increment(5); +const b = new Counter(2); +b.increment(1); +console.log(a.value, b.value); +b.free(); +a.free(); +console.log(a.value, released_total()); + +const registry = new FinalizationRegistry((held) => console.log("cleanup", held)); +const token = {}; +registry.register({}, "first", token); +registry.register({ id: 2 }, "second", token); +registry.register({ id: 3 }, "third"); +console.log(registry.unregister(token), registry.unregister(token), registry.unregister({})); +try { + new FinalizationRegistry(() => {}).register(token, token); +} catch (error) { + console.log((error as Error).name, (error as Error).message); +} +console.log(typeof FinalizationRegistry, registry instanceof FinalizationRegistry); +console.log(Object.prototype.toString.call(registry), registry); diff --git a/tests/corpus/7009-finalization-registry-wasm-loader/node_modules/counter-wasm/counter_wasm.d.ts b/tests/corpus/7009-finalization-registry-wasm-loader/node_modules/counter-wasm/counter_wasm.d.ts new file mode 100644 index 000000000..525dec86f --- /dev/null +++ b/tests/corpus/7009-finalization-registry-wasm-loader/node_modules/counter-wasm/counter_wasm.d.ts @@ -0,0 +1,9 @@ +export declare class Counter { + constructor(start: number); + free(): void; + increment(by: number): number; + readonly value: number; + [Symbol.dispose](): void; +} +export declare function released_total(): number; +export declare function initSync(bytes: Uint8Array): unknown; diff --git a/tests/corpus/7009-finalization-registry-wasm-loader/node_modules/counter-wasm/counter_wasm.js b/tests/corpus/7009-finalization-registry-wasm-loader/node_modules/counter-wasm/counter_wasm.js new file mode 100644 index 000000000..92715f743 --- /dev/null +++ b/tests/corpus/7009-finalization-registry-wasm-loader/node_modules/counter-wasm/counter_wasm.js @@ -0,0 +1,61 @@ +// A loader in the shape wasm-bindgen's web target generates: the module's +// wrapper classes register with a FinalizationRegistry when they are +// created and unregister when free() releases them, and a TextEncoder +// polyfill guard runs at load. +let wasm; +const cachedTextEncoder = new TextEncoder(); +if (!("encodeInto" in cachedTextEncoder)) { + cachedTextEncoder.encodeInto = function (arg, view) { + const buf = cachedTextEncoder.encode(arg); + view.set(buf); + return { read: arg.length, written: buf.length }; + }; +} +let released = 0; +const CounterFinalization = + typeof FinalizationRegistry === "undefined" + ? { register: () => {}, unregister: () => {} } + : new FinalizationRegistry((ptr) => wasm.add(ptr >>> 0, 1)); + +export class Counter { + static __wrap(ptr) { + const obj = Object.create(Counter.prototype); + obj.__wbg_ptr = ptr; + CounterFinalization.register(obj, obj.__wbg_ptr, obj); + return obj; + } + constructor(start) { + this.__wbg_ptr = wasm.add(start, 0); + CounterFinalization.register(this, this.__wbg_ptr, this); + } + __destroy_into_raw() { + const ptr = this.__wbg_ptr; + this.__wbg_ptr = 0; + CounterFinalization.unregister(this); + return ptr; + } + free() { + const ptr = this.__destroy_into_raw(); + released += wasm.add(ptr, 0); + } + increment(by) { + this.__wbg_ptr = wasm.add(this.__wbg_ptr, by); + return this.__wbg_ptr; + } + get value() { + return this.__wbg_ptr; + } +} +if (Symbol.dispose) Counter.prototype[Symbol.dispose] = Counter.prototype.free; + +export function released_total() { + return released; +} + +export function initSync(bytes) { + if (wasm !== undefined) return wasm; + const module = new WebAssembly.Module(bytes); + const instance = new WebAssembly.Instance(module, {}); + wasm = instance.exports; + return wasm; +} diff --git a/tests/corpus/7009-finalization-registry-wasm-loader/node_modules/counter-wasm/package.json b/tests/corpus/7009-finalization-registry-wasm-loader/node_modules/counter-wasm/package.json new file mode 100644 index 000000000..d86e34137 --- /dev/null +++ b/tests/corpus/7009-finalization-registry-wasm-loader/node_modules/counter-wasm/package.json @@ -0,0 +1 @@ +{ "name": "counter-wasm", "version": "1.0.0", "type": "module", "main": "./counter_wasm.js", "types": "./counter_wasm.d.ts" } diff --git a/tests/harness/corpus-costs/7009-finalization-registry-wasm-loader/main.ts.json b/tests/harness/corpus-costs/7009-finalization-registry-wasm-loader/main.ts.json new file mode 100644 index 000000000..db9fc4168 --- /dev/null +++ b/tests/harness/corpus-costs/7009-finalization-registry-wasm-loader/main.ts.json @@ -0,0 +1 @@ +{ "plain": 4351, "san": 2333 } From 317ec98fa5669cbd5ebd4ed8bb5f3512195d55b7 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 12:31:21 -0500 Subject: [PATCH 061/197] Close async generators on early for await exits and iterate untyped native streams --- packages/cli/test/native-link-info.test.ts | 4 +- packages/compiler/src/backend/llvm/emitter.ts | 5 + .../compiler/src/backend/llvm/lib-dispatch.ts | 6 +- .../compiler/src/backend/native/executable.ts | 5 + packages/compiler/src/backend/runtime-abi.ts | 4 +- .../src/backend/runtime-pack-targets.test.ts | 2 +- .../compiler/src/backend/runtime-pack.test.ts | 2 +- .../lowering/lower-async-iteration.ts | 4 +- .../src/frontend/lowering/lower-calls.ts | 15 +++ .../src/frontend/lowering/lower-generators.ts | 72 +++++++------ .../src/frontend/lowering/lower-stream.ts | 71 ++++++++----- .../test/native-codegen-integration.test.ts | 4 +- .../tests/corpus/7420-for-await-close.ts.json | 6 ++ .../corpus/7421-for-await-close.mjs.json | 6 ++ .../7422-for-await-untyped-streams.mjs.json | 6 ++ .../7423-stream-untyped-bindings.mjs.json | 6 ++ .../runtime-darwin-arm64/scripts/build.mjs | 2 +- .../runtime-pack-matrix.mjs | 1 + .../runtime-pack-common/scripts/build.mjs | 2 +- packages/runtime/src/scr_console.c | 2 +- packages/runtime/src/scr_core.h | 2 +- packages/runtime/src/scr_dyn.h | 1 + packages/runtime/src/scr_dyn_for_await.c | 5 + .../runtime/src/scr_dyn_for_await_stream.c | 61 +++++++++++ packages/runtime/src/scr_stream.c | 39 ++++++- packages/runtime/src/scr_stream.h | 8 ++ tests/corpus/7420-for-await-close.ts | 100 ++++++++++++++++++ tests/corpus/7421-for-await-close.mjs | 53 ++++++++++ .../corpus/7422-for-await-untyped-streams.mjs | 38 +++++++ tests/corpus/7423-stream-untyped-bindings.mjs | 33 ++++++ .../corpus-costs/7420-for-await-close.ts.json | 1 + .../7421-for-await-close.mjs.json | 1 + .../7422-for-await-untyped-streams.mjs.json | 1 + .../7423-stream-untyped-bindings.mjs.json | 1 + .../self-hosting-native-toolchain.test.ts | 2 +- 35 files changed, 496 insertions(+), 75 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7420-for-await-close.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7421-for-await-close.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7422-for-await-untyped-streams.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7423-stream-untyped-bindings.mjs.json create mode 100644 packages/runtime/src/scr_dyn_for_await_stream.c create mode 100644 tests/corpus/7420-for-await-close.ts create mode 100644 tests/corpus/7421-for-await-close.mjs create mode 100644 tests/corpus/7422-for-await-untyped-streams.mjs create mode 100644 tests/corpus/7423-stream-untyped-bindings.mjs create mode 100644 tests/harness/corpus-costs/7420-for-await-close.ts.json create mode 100644 tests/harness/corpus-costs/7421-for-await-close.mjs.json create mode 100644 tests/harness/corpus-costs/7422-for-await-untyped-streams.mjs.json create mode 100644 tests/harness/corpus-costs/7423-stream-untyped-bindings.mjs.json diff --git a/packages/cli/test/native-link-info.test.ts b/packages/cli/test/native-link-info.test.ts index a1b673630..484b03e8b 100644 --- a/packages/cli/test/native-link-info.test.ts +++ b/packages/cli/test/native-link-info.test.ts @@ -72,7 +72,7 @@ describe.runIf(supported)("macOS arm64 native link info", () => { minimum_os: "14.0", }, program: { object, entry_symbol: "main" }, - runtime_abi: { version: 28, marker: "scr_runtime_abi_v28" }, + runtime_abi: { version: 29, marker: "scr_runtime_abi_v29" }, runtime_pack: { kind: "precompiled", package: "@scriptc/runtime-darwin-arm64", @@ -165,6 +165,6 @@ describe.runIf(supported)("macOS arm64 native link info", () => { () => null, (failure: { stderr?: string }) => failure, ); - expect(error?.stderr).toContain("scr_runtime_abi_v28"); + expect(error?.stderr).toContain("scr_runtime_abi_v29"); }); }); diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index be9423442..9f21791d6 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -972,6 +972,11 @@ export class LlEmitter { get usesDynBuiltinStream(): boolean { return this.runtimeFeatures.dynBuiltinStream; } + /** A checked-dynamic for-await may hold a native stream: the program + * links the stream unit (and the dyn-async one the loop needs). */ + get dynForAwaitStreams(): boolean { + return this.runtimeFeatures.stream && this.runtimeFeatures.dynAsync; + } /** Runtime-arm dispatchers for live union values. */ readonly liveDynUnionRefAdapters = new Map(); readonly dynPromiseAdapters = new Map(); diff --git a/packages/compiler/src/backend/llvm/lib-dispatch.ts b/packages/compiler/src/backend/llvm/lib-dispatch.ts index fd5b5d681..10d83057f 100644 --- a/packages/compiler/src/backend/llvm/lib-dispatch.ts +++ b/packages/compiler/src/backend/llvm/lib-dispatch.ts @@ -553,7 +553,11 @@ export function emitGenericLibCall(host: LlvmEmitterContext, e: LibCallExpr): Ll // which also takes those streams. e.fn === "stream.unboxDyn" && host.usesDynBuiltinStream ? "scr_dyn_builtin_stream_unbox" - : LIB_FN_SYMS[e.fn]); + : // for await over a checked-dynamic value iterates a native + // stream behind it (scr_dyn_for_await_stream.c) + e.fn === "dyn.asyncIteratorOpen" && host.dynForAwaitStreams + ? "scr_stream_dyn_async_iterator_open" + : LIB_FN_SYMS[e.fn]); if (sym === undefined) throw new LlvmUnsupportedError(`libCall:${e.fn}`, e.loc); const args = READ_ONLY_STRING_LIB_FNS.has(e.fn) ? emitBorrowedInputs(host, e.args) diff --git a/packages/compiler/src/backend/native/executable.ts b/packages/compiler/src/backend/native/executable.ts index 714bc09a5..ad318e623 100644 --- a/packages/compiler/src/backend/native/executable.ts +++ b/packages/compiler/src/backend/native/executable.ts @@ -281,6 +281,11 @@ const RUNTIME_UNITS: readonly RuntimeUnit[] = [ unit((u) => u.opts.handleProps === true, "scr_dyn_handle_props.c"), // for await over checked-dynamic values. unit((u) => u.opts.dynForAwait === true, "scr_dyn_for_await.c"), + // for await over checked-dynamic values that may hold native streams. + unit( + (u) => u.opts.dynForAwait === true && u.opts.stream && u.opts.dynAsync, + "scr_dyn_for_await_stream.c", + ), // FileHandle streams reference both the stream unit and FileHandle. unit((u) => u.opts.stream && u.opts.fileHandle, "scr_fs_stream_fh.c"), // The readiness-poller backends (scr_platform.h): kqueue on macOS/BSD, diff --git a/packages/compiler/src/backend/runtime-abi.ts b/packages/compiler/src/backend/runtime-abi.ts index a616dbec2..948fa6359 100644 --- a/packages/compiler/src/backend/runtime-abi.ts +++ b/packages/compiler/src/backend/runtime-abi.ts @@ -2,8 +2,8 @@ * of the link contract: a program object keeps this symbol undefined and the * matching runtime defines it, so incompatible manual links fail before the * program can start. */ -export const RUNTIME_ABI_VERSION = 28 as const; -export const RUNTIME_ABI_MARKER = "scr_runtime_abi_v28" as const; +export const RUNTIME_ABI_VERSION = 29 as const; +export const RUNTIME_ABI_MARKER = "scr_runtime_abi_v29" as const; /** Object consumption is useful today, but the complete scr_* surface may * still change before 1.0. The versioned marker prevents accidental mixing; diff --git a/packages/compiler/src/backend/runtime-pack-targets.test.ts b/packages/compiler/src/backend/runtime-pack-targets.test.ts index c2536ede8..a97769328 100644 --- a/packages/compiler/src/backend/runtime-pack-targets.test.ts +++ b/packages/compiler/src/backend/runtime-pack-targets.test.ts @@ -52,7 +52,7 @@ async function fixture(target: NativeTargetSpec, workers = false) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 28, marker: "scr_runtime_abi_v28" }, + runtime_abi: { version: 29, marker: "scr_runtime_abi_v29" }, compiler: { command: "fixture", identity: "fixture", target: target.llvmTriple }, macros: { executable: [], excluded: ["SCR_LIB"], sanitizer: "external-toolchain-required" }, flavors: { diff --git a/packages/compiler/src/backend/runtime-pack.test.ts b/packages/compiler/src/backend/runtime-pack.test.ts index 33b1ca38e..85038d8c5 100644 --- a/packages/compiler/src/backend/runtime-pack.test.ts +++ b/packages/compiler/src/backend/runtime-pack.test.ts @@ -127,7 +127,7 @@ async function fixture(target: NativeTargetSpec = MACOS_ARM64_TARGET) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 28, marker: "scr_runtime_abi_v28" }, + runtime_abi: { version: 29, marker: "scr_runtime_abi_v29" }, compiler: { command: "clang", identity: "fixture clang", diff --git a/packages/compiler/src/frontend/lowering/lower-async-iteration.ts b/packages/compiler/src/frontend/lowering/lower-async-iteration.ts index 5b427a750..95916ee9a 100644 --- a/packages/compiler/src/frontend/lowering/lower-async-iteration.ts +++ b/packages/compiler/src/frontend/lowering/lower-async-iteration.ts @@ -359,13 +359,15 @@ function boolRef(localId: string, loc: ReturnType): IrExpr { /** The loop's stream: a JavaScript value typed as a stream can arrive as a * checked-dynamic value (returned through an untyped function), which - * checks back into the stream class. */ + * checks back into the stream class, or as a union with it (an element + * read), which narrows to it. */ function streamSource( lowerer: Lowerer, plan: NodeReadablePlan, loc: ReturnType, ): IrExpr { const v = lowerer.lowerExpr(plan.source); + if (v.type.kind === "union") return lowerer.coerceToExpected(v, plan.type); if (v.type.kind !== "dyn") return v; return { kind: "dynCheck", value: v, type: plan.type, loc }; } diff --git a/packages/compiler/src/frontend/lowering/lower-calls.ts b/packages/compiler/src/frontend/lowering/lower-calls.ts index 875fb9e7a..0ffaab400 100644 --- a/packages/compiler/src/frontend/lowering/lower-calls.ts +++ b/packages/compiler/src/frontend/lowering/lower-calls.ts @@ -4338,6 +4338,21 @@ export function lowerDynDispatchMethodCall( lowerer.chainBlocked(call, access) ) return null; + // readable.compose lowers its stages from their syntax, so it needs the + // stream's class: a stream a JavaScript function returns untyped has none. + if (method === "compose") { + const recvT = lowerer.mapTypeOf(lowerer.typeOf(access.expression)); + if ( + recvT?.kind === "object" && + streamSidesOf(lowerer, lowerer.classes.get(recvT.className)) !== null + ) { + lowerer.noLowering( + "readable.compose on a stream held as a checked-dynamic value", + call, + "type the stream where it comes from, such as a JSDoc @returns {Readable} on the function that returns it", + ); + } + } if (call.arguments.nodes.some((arg) => ts.isSpreadElement(arg))) { const loc = locOf(call); const local = lowerer.declareHiddenLocal("%spreadReceiver", DYN); diff --git a/packages/compiler/src/frontend/lowering/lower-generators.ts b/packages/compiler/src/frontend/lowering/lower-generators.ts index b4e3ac102..80463fa6c 100644 --- a/packages/compiler/src/frontend/lowering/lower-generators.ts +++ b/packages/compiler/src/frontend/lowering/lower-generators.ts @@ -687,8 +687,12 @@ export function lowerForAwaitGenerator( lowerer.scopes.push(new Map()); try { const g = lowerer.declareHiddenLocal("%fag", genT); - const done = lowerer.declareHiddenLocal("%fagdone", BOOL); - done.mutable = true; + // Whether a value has arrived and the body holds the generator open: + // leaving the loop then (break, return, a throw, or a return() at a + // yield in the body) closes it, as AsyncIteratorClose does; an + // exhausted generator and a rejected next() are not closed. + const active = lowerer.declareHiddenLocal("%fagactive", BOOL); + active.mutable = true; const p = lowerer.declareHiddenLocal("%fagpromise", promiseT); const r = lowerer.declareHiddenLocal("%fagresult", recT); const gRef = (): IrExpr => ({ kind: "varRef", localId: g.id, type: genT, loc }); @@ -709,7 +713,14 @@ export function lowerForAwaitGenerator( `for-await over an async generator yielding '${lowerer.fmt(genT.yieldT)}' (no per-element extraction exists)`, ); } + const setActive = (value: boolean): IrStmt => ({ + kind: "assign", + localId: active.id, + value: { kind: "boolLit", value, type: BOOL, loc }, + loc, + }); const head: IrStmt[] = [ + setActive(false), { kind: "varDecl", localId: p.id, @@ -737,19 +748,12 @@ export function lowerForAwaitGenerator( type: BOOL, loc, }, - then: [ - { - kind: "assign", - localId: done.id, - value: { kind: "boolLit", value: true, type: BOOL, loc }, - loc, - }, - { kind: "break", loc }, - ], + then: [{ kind: "break", loc }], else_: null, loc, }, ...generatorLoopBinding(lowerer, stmt, extracted, genT.yieldT), + setActive(true), ]; const body = lowerer.inCtl("loop", () => lowerer.lowerScopedBlock(stmt.statement), labels); const closePromise: IrExpr = { @@ -766,34 +770,40 @@ export function lowerForAwaitGenerator( { kind: "varDecl", localId: g.id, init: iterable, loc }, { kind: "varDecl", - localId: done.id, + localId: active.id, init: { kind: "boolLit", value: false, type: BOOL, loc }, loc, }, { - kind: "while", - cond: { kind: "boolLit", value: true, type: BOOL, loc }, - body: [...head, ...body], - ...(labels && { labels }), - loc, - }, - { - kind: "if", - cond: { - kind: "unary", - op: "!", - operand: { kind: "varRef", localId: done.id, type: BOOL, loc }, - type: BOOL, - loc, - }, - then: [ + kind: "tryCatch", + tryBody: [ { - kind: "exprStmt", - expr: { kind: "awaitExpr", value: closePromise, type: recT, loc }, + kind: "while", + cond: { kind: "boolLit", value: true, type: BOOL, loc }, + body: [...head, ...body], + ...(labels && { labels }), + loc, + }, + ], + catchBody: null, + catchLocalId: null, + // After a throw, an error from closing the generator is dropped. + dropFinallyErrors: true, + finallyBody: [ + { + kind: "if", + cond: { kind: "varRef", localId: active.id, type: BOOL, loc }, + then: [ + { + kind: "exprStmt", + expr: { kind: "awaitExpr", value: closePromise, type: recT, loc }, + loc, + }, + ], + else_: null, loc, }, ], - else_: null, loc, }, ], diff --git a/packages/compiler/src/frontend/lowering/lower-stream.ts b/packages/compiler/src/frontend/lowering/lower-stream.ts index 214a2402e..6aa8aef96 100644 --- a/packages/compiler/src/frontend/lowering/lower-stream.ts +++ b/packages/compiler/src/frontend/lowering/lower-stream.ts @@ -3944,6 +3944,29 @@ function lowerStreamCompose(lowerer: Lowerer, args: readonly ts.Expression[], lo /* ── the Readable operators ──────────────────────────────────────────── */ /** The Readable operators (scr_stream_ops.c — Node's operators.js). */ +/** A stream member's receiver, lowered: a value the checker types as a + * stream class whose lowering is checked-dynamic (a JavaScript binding + * that holds an untyped call's result) or a union (an element read) + * converts to that class, checked where the member runs. */ +function streamReceiverExpr(lowerer: Lowerer, node: ts.Expression): IrExpr { + const value = lowerer.lowerExpr(node); + if (value.type.kind !== "dyn" && value.type.kind !== "union") return value; + let classT: IrType | null; + try { + classT = lowerer.mapTypeOf(lowerer.typeOf(node)); + } catch { + return value; + } + if ( + classT?.kind !== "object" || + streamSidesOf(lowerer, lowerer.classes.get(classT.className)) === null + ) + return value; + return value.type.kind === "dyn" + ? { kind: "dynCheck", value, type: classT, loc: value.loc } + : lowerer.coerceToExpected(value, classT); +} + export const STREAM_OPERATORS: ReadonlySet = new Set([ "map", "filter", @@ -4066,7 +4089,7 @@ function lowerStreamOperator( ): IrExpr { const loc = locOf(call); const args = call.arguments.nodes; - const recv = lowerer.lowerExpr(access.expression); + const recv = streamReceiverExpr(lowerer, access.expression); const readable: IrType = { kind: "object", className: "%Readable" }; const opts = (i: number): IrExpr => lowerOperatorValue(lowerer, args[i], `${member} options`, loc, true); @@ -4210,7 +4233,7 @@ function pipeIntoStdio( ): IrExpr { const loc = locOf(call); // JS order: the receiver, then the destination, then the options. - const receiverValue = lowerer.lowerExpr(access.expression); + const receiverValue = streamReceiverExpr(lowerer, access.expression); const source = lowerer.declareHiddenLocal("%pipeSource", receiverValue.type); const stdio = lowerer.declareHiddenLocal("%pipeStdio", target.type); const options = call.arguments.nodes[1]; @@ -4332,7 +4355,7 @@ function pipeIntoDyn( ): IrExpr { const loc = locOf(call); noteUntypedStreamFlow(lowerer, null); - const receiverValue = lowerer.lowerExpr(access.expression); + const receiverValue = streamReceiverExpr(lowerer, access.expression); const source = lowerer.declareHiddenLocal("%pipeSource", receiverValue.type); const destination = lowerer.declareHiddenLocal("%pipeDestination", target.type); const stmts: IrStmt[] = [ @@ -4621,7 +4644,7 @@ export function lowerStreamMethodCall( loc, ); } - const receiver = lowerer.lowerExpr(access.expression); + const receiver = streamReceiverExpr(lowerer, access.expression); const chunk = lowerer.lowerExprExpecting(args[0]!, STRING); return { kind: "libCall", @@ -4637,7 +4660,7 @@ export function lowerStreamMethodCall( if ((kind === "null" || kind === "union" || kind === "dyn") && member === "unshift") { // unshift(value): an object-mode stream's value at the front, null // the end of the stream, as Node's readableAddChunk takes them - const receiver = lowerer.lowerExpr(access.expression); + const receiver = streamReceiverExpr(lowerer, access.expression); const chunk = lowerer.lowerExprExpecting(args[0]!, DYN); return { kind: "libCall", @@ -4647,7 +4670,7 @@ export function lowerStreamMethodCall( loc, }; } - const receiver = lowerer.lowerExpr(access.expression); + const receiver = streamReceiverExpr(lowerer, access.expression); if (kind === "null") { return { kind: "libCall", fn: "readable.pushNull", args: [receiver], type: BOOL, loc }; } @@ -4700,7 +4723,7 @@ export function lowerStreamMethodCall( if (member === "read") { requireSide(readableSide, "read"); if (args.length > 1) lowerer.noLowering(`read with ${args.length} arguments`, call); - const receiver = lowerer.lowerExpr(access.expression); + const receiver = streamReceiverExpr(lowerer, access.expression); const size = args[0] ? lowerer.lowerExprExpecting(args[0], F64) : numLit(-1, loc); const type: IrType = unionOf(lowerer, [BYTES, { kind: "nullT" }]); const read: IrExpr = { @@ -4716,7 +4739,7 @@ export function lowerStreamMethodCall( if (member === "pause" || member === "resume") { requireSide(readableSide, member); if (args.length !== 0) lowerer.noLowering(`${member} with arguments`, call); - const receiver = lowerer.lowerExpr(access.expression); + const receiver = streamReceiverExpr(lowerer, access.expression); return { kind: "libCall", fn: member === "pause" ? "readable.pause" : "readable.resume", @@ -4729,7 +4752,7 @@ export function lowerStreamMethodCall( if (member === "isPaused") { requireSide(readableSide, "isPaused"); if (args.length !== 0) lowerer.noLowering(`isPaused with arguments`, call); - const receiver = lowerer.lowerExpr(access.expression); + const receiver = streamReceiverExpr(lowerer, access.expression); return { kind: "libCall", fn: "readable.isPaused", args: [receiver], type: BOOL, loc }; } @@ -4746,7 +4769,7 @@ export function lowerStreamMethodCall( // Transform, and pipe() answers the value itself (the receiver // evaluates first, as in JS). if (isValueStream(lowerer, args[0]!)) { - const receiverValue = lowerer.lowerExpr(access.expression); + const receiverValue = streamReceiverExpr(lowerer, access.expression); const source = lowerer.declareHiddenLocal("%pipeSource", receiverValue.type); const hashValue = lowerer.lowerExpr(args[0]!); const hash = lowerer.declareHiddenLocal("%pipeHash", hashValue.type); @@ -4796,7 +4819,7 @@ export function lowerStreamMethodCall( ); } const end = pipeEndOption(lowerer, args[1], loc); - const receiver = lowerer.lowerExpr(access.expression); + const receiver = streamReceiverExpr(lowerer, access.expression); return { kind: "libCall", fn: generalPipe(lowerer, dst.type), @@ -4809,7 +4832,7 @@ export function lowerStreamMethodCall( if (member === "unpipe") { requireSide(readableSide, "unpipe"); if (args.length > 1) lowerer.noLowering(`unpipe with ${args.length} arguments`, call); - const receiver = lowerer.lowerExpr(access.expression); + const receiver = streamReceiverExpr(lowerer, access.expression); if (args[0] !== undefined) { const dst = lowerer.lowerExpr(args[0]); if (dst.type.kind === "dyn") return unpipeDyn(lowerer, receiver, dst, loc); @@ -4870,7 +4893,7 @@ export function lowerStreamMethodCall( if (member === "destroy") { if (args.length > 1) lowerer.noLowering(`destroy with ${args.length} arguments`, call); - const receiver = lowerer.lowerExpr(access.expression); + const receiver = streamReceiverExpr(lowerer, access.expression); if (args[0] !== undefined) { const err = lowerer.lowerExpr(args[0]); const rootsAtError = (t: IrType): boolean => { @@ -4923,7 +4946,7 @@ export function lowerStreamMethodCall( // the callback took its place (a byte-mode side encodes a string // chunk with it). const writeEnc = (encArg: ts.Expression | undefined, cbArg: ts.Expression | undefined) => { - const receiver = lowerer.lowerExpr(access.expression); + const receiver = streamReceiverExpr(lowerer, access.expression); const chunk = lowerer.lowerExprExpecting(args[0]!, DYN); const tail: IrExpr[] = []; if (encArg !== undefined) tail.push(lowerer.lowerExprExpecting(encArg, DYN)); @@ -4953,7 +4976,7 @@ export function lowerStreamMethodCall( } return writeEnc(args[1], undefined); } - const receiver = lowerer.lowerExpr(access.expression); + const receiver = streamReceiverExpr(lowerer, access.expression); const chunk = lowerer.lowerExprExpecting(args[0]!, DYN); return { kind: "libCall", fn: "writable.writeDyn", args: [receiver, chunk], type: BOOL, loc }; } @@ -4963,7 +4986,7 @@ export function lowerStreamMethodCall( // lowered value may have collapsed to one arm (checker narrowing) — // re-dispatch on its IR type. if (args.length > 1) lowerer.noLowering("write with a union chunk and more arguments", call); - const receiver = lowerer.lowerExpr(access.expression); + const receiver = streamReceiverExpr(lowerer, access.expression); const chunk = lowerer.lowerExpr(args[0]!); const fn = chunk.type.kind === "union" @@ -4986,7 +5009,7 @@ export function lowerStreamMethodCall( return writeEnc(args[1], undefined); cbArg = args[1]; } - const receiver = lowerer.lowerExpr(access.expression); + const receiver = streamReceiverExpr(lowerer, access.expression); const chunk = kind === "bytes" ? lowerer.lowerExprExpecting(args[0]!, BYTES) @@ -5021,7 +5044,7 @@ export function lowerStreamMethodCall( encNode = args[1]; cbArg = args[2]; } - const receiver = lowerer.lowerExpr(access.expression); + const receiver = streamReceiverExpr(lowerer, access.expression); let flags = 0; const tail: IrExpr[] = []; if (chunkNode !== undefined) { @@ -5068,7 +5091,7 @@ export function lowerStreamMethodCall( if (member === "cork" || member === "uncork") { requireSide(writableSide, member); if (args.length !== 0) lowerer.noLowering(`${member} with arguments`, call); - const receiver = lowerer.lowerExpr(access.expression); + const receiver = streamReceiverExpr(lowerer, access.expression); return { kind: "libCall", fn: member === "cork" ? "writable.cork" : "writable.uncork", @@ -5088,7 +5111,7 @@ export function lowerStreamMethodCall( ); } const enc = bufEncoding(lowerer, "setEncoding", args[0]!); - const receiver = lowerer.lowerExpr(access.expression); + const receiver = streamReceiverExpr(lowerer, access.expression); return { kind: "libCall", fn: "readable.setEncoding", @@ -5104,7 +5127,7 @@ export function lowerStreamMethodCall( requireSide(writableSide, "setDefaultEncoding"); if (args.length > 1) lowerer.noLowering(`setDefaultEncoding with ${args.length} arguments`, call); - const receiver = lowerer.lowerExpr(access.expression); + const receiver = streamReceiverExpr(lowerer, access.expression); const enc = args[0] ? lowerer.lowerExprExpecting(args[0], DYN) : dynUndefinedExpr(loc); return { kind: "libCall", @@ -5257,7 +5280,7 @@ export function lowerStreamProperty( const propCall = (type: IrType): IrExpr => ({ kind: "libCall", fn: "stream.prop", - args: [lowerer.lowerExpr(expr.expression), strLit(name, loc)], + args: [streamReceiverExpr(lowerer, expr.expression), strLit(name, loc)], type, loc, }); @@ -5280,7 +5303,7 @@ export function lowerStreamProperty( const read: IrExpr = { kind: "libCall", fn: "readable.flowing", - args: [lowerer.lowerExpr(expr.expression)], + args: [streamReceiverExpr(lowerer, expr.expression)], type, loc, }; @@ -5294,7 +5317,7 @@ export function lowerStreamProperty( const read: IrExpr = { kind: "libCall", fn: "stream.errored", - args: [lowerer.lowerExpr(expr.expression)], + args: [streamReceiverExpr(lowerer, expr.expression)], type, loc, }; diff --git a/packages/compiler/test/native-codegen-integration.test.ts b/packages/compiler/test/native-codegen-integration.test.ts index 71b9570b4..bb6ccf874 100644 --- a/packages/compiler/test/native-codegen-integration.test.ts +++ b/packages/compiler/test/native-codegen-integration.test.ts @@ -378,7 +378,7 @@ done: expect(await symbols(helperObject, ["-u"])).toEqual(await symbols(clangObject, ["-u"])); expect(await symbols(helperObject, ["-gU"])).toEqual(await symbols(clangObject, ["-gU"])); expect(await symbols(helperObject, ["-gU"])).toEqual(["0000000000000000 T _main"]); - expect(await symbols(helperObject, ["-u"])).toContain("_scr_runtime_abi_v28"); + expect(await symbols(helperObject, ["-u"])).toContain("_scr_runtime_abi_v29"); const clangExe = join(dir, "clang-program"); const linkDriver = join(dir, "clang-driver.c"); @@ -712,7 +712,7 @@ done: .split("\n"); expect(undefinedSymbols).toContain("_sf_scale"); expect(undefinedSymbols).toContain("_sf_callback_mix"); - expect(undefinedSymbols).toContain("_scr_runtime_abi_v28"); + expect(undefinedSymbols).toContain("_scr_runtime_abi_v29"); }); test("helper and clang object paths retain the same outbound FFI C ABI", async () => { diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7420-for-await-close.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7420-for-await-close.ts.json new file mode 100644 index 000000000..6e48bbc1c --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7420-for-await-close.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7420-for-await-close.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7421-for-await-close.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7421-for-await-close.mjs.json new file mode 100644 index 000000000..dea911d1b --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7421-for-await-close.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7421-for-await-close.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7422-for-await-untyped-streams.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7422-for-await-untyped-streams.mjs.json new file mode 100644 index 000000000..c2677d0b9 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7422-for-await-untyped-streams.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7422-for-await-untyped-streams.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7423-stream-untyped-bindings.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7423-stream-untyped-bindings.mjs.json new file mode 100644 index 000000000..180721bd6 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7423-stream-untyped-bindings.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7423-stream-untyped-bindings.mjs" + ], + "diags": [] +} diff --git a/packages/runtime-darwin-arm64/scripts/build.mjs b/packages/runtime-darwin-arm64/scripts/build.mjs index 3d22b2360..71126f806 100644 --- a/packages/runtime-darwin-arm64/scripts/build.mjs +++ b/packages/runtime-darwin-arm64/scripts/build.mjs @@ -340,7 +340,7 @@ async function build() { package: packageManifest.name, version: packageManifest.version, target: RUNTIME_PACK_MATRIX.target, - runtime_abi: { version: 28, marker: "scr_runtime_abi_v28" }, + runtime_abi: { version: 29, marker: "scr_runtime_abi_v29" }, compiler: { command: compiler, identity: compilerVersion, diff --git a/packages/runtime-pack-common/runtime-pack-matrix.mjs b/packages/runtime-pack-common/runtime-pack-matrix.mjs index 322832f9f..1c0458d7e 100644 --- a/packages/runtime-pack-common/runtime-pack-matrix.mjs +++ b/packages/runtime-pack-common/runtime-pack-matrix.mjs @@ -129,6 +129,7 @@ export const OPTIONAL_RUNTIME_SOURCES = [ ["scr_dyn_handle_props.c", "handleProps"], // for await over checked-dynamic values. ["scr_dyn_for_await.c", "dynForAwait"], + ["scr_dyn_for_await_stream.c", all("dynForAwait", "stream", "dynAsync")], // fs.ReadStream / fs.WriteStream reference only the stream unit and base units. ["scr_fs_stream.c", "stream"], // FileHandle streams reference both the stream unit and FileHandle. diff --git a/packages/runtime-pack-common/scripts/build.mjs b/packages/runtime-pack-common/scripts/build.mjs index 909412bc7..b0ec0005b 100644 --- a/packages/runtime-pack-common/scripts/build.mjs +++ b/packages/runtime-pack-common/scripts/build.mjs @@ -437,7 +437,7 @@ async function build() { package: packageManifest.name, version: packageManifest.version, target: matrix.target, - runtime_abi: { version: 28, marker: "scr_runtime_abi_v28" }, + runtime_abi: { version: 29, marker: "scr_runtime_abi_v29" }, compiler: { command: compiler, identity: compilerVersion, diff --git a/packages/runtime/src/scr_console.c b/packages/runtime/src/scr_console.c index 9bd5abf8c..317ee124b 100644 --- a/packages/runtime/src/scr_console.c +++ b/packages/runtime/src/scr_console.c @@ -148,7 +148,7 @@ static void scr_enable_utf8_console_output(void) { } #endif -void scr_runtime_abi_v28(void) {} +void scr_runtime_abi_v29(void) {} void scr_init(void) { #ifdef SCR_WORKERS diff --git a/packages/runtime/src/scr_core.h b/packages/runtime/src/scr_core.h index 35332d0ca..6f3aae93a 100644 --- a/packages/runtime/src/scr_core.h +++ b/packages/runtime/src/scr_core.h @@ -81,7 +81,7 @@ void scr_init(void); /* Program objects emitted by the bundled LLVM helper reference this symbol. * Its versioned spelling makes a mismatched manual runtime link fail before * the program can start. */ -void scr_runtime_abi_v28(void); +void scr_runtime_abi_v29(void); void scr_runtime_workers_v8(void); /* ── the trap funnel (scr_console.c; scr_library.c under -DSCR_LIB) ────── diff --git a/packages/runtime/src/scr_dyn.h b/packages/runtime/src/scr_dyn.h index bb3b06847..92bac77ae 100644 --- a/packages/runtime/src/scr_dyn.h +++ b/packages/runtime/src/scr_dyn.h @@ -839,6 +839,7 @@ ScrDyn *scr_dyn_iterator(const ScrDyn *value, const ScrStr *spell); /* for await over a checked-dynamic value (scr_dyn_for_await.c). */ ScrDyn *scr_dyn_async_iterator_open(ScrDyn *value, ScrStr *spell); bool scr_dyn_async_iterator_was_sync(void); +void scr_dyn_async_iterator_set_sync(bool sync); ScrDyn *scr_dyn_iterator_optional(const ScrDyn *value); /* from.js's probe: 2 when value[Symbol.asyncIterator] is set, 1 when * value[Symbol.iterator] is, 0 for neither, -1 with an error pending. */ diff --git a/packages/runtime/src/scr_dyn_for_await.c b/packages/runtime/src/scr_dyn_for_await.c index 25183c4ee..3a1a6458d 100644 --- a/packages/runtime/src/scr_dyn_for_await.c +++ b/packages/runtime/src/scr_dyn_for_await.c @@ -38,6 +38,11 @@ ScrDyn *scr_dyn_async_iterator_open(ScrDyn *value, ScrStr *spell) { return iterator; } +/* For an iterator another open path made (scr_dyn_for_await_stream.c). */ +void scr_dyn_async_iterator_set_sync(bool sync) { + scr_for_await_sync = sync; +} + /* Whether the iterator the last scr_dyn_async_iterator_open returned is a * synchronous one. Read immediately after the open. */ bool scr_dyn_async_iterator_was_sync(void) { diff --git a/packages/runtime/src/scr_dyn_for_await_stream.c b/packages/runtime/src/scr_dyn_for_await_stream.c new file mode 100644 index 000000000..6facf916c --- /dev/null +++ b/packages/runtime/src/scr_dyn_for_await_stream.c @@ -0,0 +1,61 @@ +/* `for await` over a checked-dynamic value in a program that links the + * stream unit: a native readable stream behind the value (a boxed stream, + * or a Cipheriv/Decipheriv handle's Transform) iterates through an + * iterator object whose next() answers the readable iterator's step as + * { value, done } and whose return() closes it as a typed loop does + * (destroy with an AbortError). Any other value takes + * scr_dyn_async_iterator_open. Linked only by programs with such a loop + * and streams. */ +#include "scr_runtime.h" +#include "scr_stream.h" + +/* caps[0]: the stream */ +static ScrStream *iter_stream(ScrClosure *clo) { + return scr_box_get_ref(clo->caps[0]); /* +1 */ +} + +static ScrDyn *iter_next_thunk(ScrClosure *clo, ScrDyn *const *args, size_t argc) { + (void)args; + (void)argc; + ScrStream *s = iter_stream(clo); + ScrPromise *p = scr_stream_next_result_dyn(s); + scr_stream_release(s); + ScrDyn *d = scr_dyn_new_promise(p); + scr_promise_release(p); + return d; +} + +static ScrDyn *iter_return_thunk(ScrClosure *clo, ScrDyn *const *args, size_t argc) { + (void)args; + (void)argc; + ScrStream *s = iter_stream(clo); + scr_stream_release(scr_stream_iterator_close(s)); + scr_stream_release(s); + ScrDyn *result = scr_dyn_new_obj(); + scr_dyn_obj_set(result, "value", 5, scr_dyn_retain(scr_dyn_undefined())); + scr_dyn_obj_set(result, "done", 4, scr_dyn_new_bool(true)); + ScrPromise *p = scr_promise_new(); + scr_promise_fulfill_ref(p, result, scr_dyn_retain_v, scr_dyn_release_v, NULL); + ScrDyn *d = scr_dyn_new_promise(p); + scr_promise_release(p); + return d; +} + +static ScrDyn *iter_method(ScrStream *s, ScrDynThunk thunk, const char *name) { + ScrClosure *clo = scr_closure_new((void *)thunk, 1); + clo->caps[0] = scr_box_new_obj(&scr_stream_retain_v, &scr_stream_release_v, &scr_stream_trace); + scr_box_set_ref(clo->caps[0], scr_stream_retain(s)); + return scr_dyn_new_func(clo, thunk, 0, "()", name); +} + +ScrDyn *scr_stream_dyn_async_iterator_open(ScrDyn *value, ScrStr *spell) { + ScrStream *s = scr_stream_native_readable_of_dyn(value); + if (!s) return scr_dyn_async_iterator_open(value, spell); + /* the loop awaits its results with the async protocol */ + scr_dyn_async_iterator_set_sync(false); + ScrDyn *it = scr_dyn_new_obj(); + scr_dyn_obj_set(it, "next", 4, iter_method(s, &iter_next_thunk, "next")); + scr_dyn_obj_set(it, "return", 6, iter_method(s, &iter_return_thunk, "return")); + scr_stream_release(s); + return it; +} diff --git a/packages/runtime/src/scr_stream.c b/packages/runtime/src/scr_stream.c index 75debe615..4a53da4c0 100644 --- a/packages/runtime/src/scr_stream.c +++ b/packages/runtime/src/scr_stream.c @@ -145,9 +145,12 @@ struct ScrStreamState { bool object_mode_opt; /* The parked for-await next() promise (at most one — the loop awaits * each chunk before asking again). next_dyn picks the resolution - * shape: a dyn boxed by tag (the JS lane), or bytes. */ + * shape: a dyn boxed by tag (the JS lane), or bytes; next_result + * wraps the dyn in an iterator result ({ value, done }) for the + * checked-dynamic for-await protocol. */ ScrPromise *next_waiter; bool next_dyn; + bool next_result; /* The iterator's own read() is calling _read: synchronous pushes * buffer, and the waiter takes the whole buffer afterwards (Node's * read() returns everything _read pushed). */ @@ -1226,7 +1229,15 @@ static void scr_stream_next_fulfill(ScrStreamState *st, ScrPromise *w /*moves*/, scr_promise_release(w); return; } - if (dyn) { + if (dyn && st->r.next_result) { + /* { value, done }: the value, or undefined at the end */ + ScrDyn *result = scr_dyn_new_obj(); + scr_dyn_obj_set(result, "value", 5, + chunk == NULL ? scr_dyn_retain(scr_dyn_undefined()) + : scr_stream_entry_box(chunk, kind)); + scr_dyn_obj_set(result, "done", 4, scr_dyn_new_bool(chunk == NULL)); + payload = result; + } else if (dyn) { payload = chunk == NULL ? scr_stream_from_done() : scr_stream_entry_box(chunk, kind); } else { payload = chunk ? (ScrBytes *)chunk : scr_bytes_new(SCR_BYTES_U8, 0); @@ -1352,11 +1363,12 @@ static void scr_stream_settle_next(ScrStream *s) { } } -static ScrPromise *scr_stream_next_chunk_impl(ScrStream *s, bool dyn) { +static ScrPromise *scr_stream_next_chunk_impl(ScrStream *s, bool dyn, bool result) { ScrStreamState *st = s->st; ScrPromise *p = scr_promise_new(); if (!st->has_r) { st->r.next_dyn = dyn; + st->r.next_result = result; scr_stream_next_fulfill(st, scr_promise_retain(p), NULL, SCR_ENTRY_BYTES); return p; } @@ -1368,6 +1380,7 @@ static ScrPromise *scr_stream_next_chunk_impl(ScrStream *s, bool dyn) { return p; } st->r.next_dyn = dyn; + st->r.next_result = result; st->r.next_waiter = scr_promise_retain(p); st->r.next_parked = false; if (!st->r.next_started) { @@ -1391,11 +1404,15 @@ ScrPromise *scr_stream_next_chunk(ScrStream *s) { scr_promise_reject_pending(p); return p; } - return scr_stream_next_chunk_impl(s, false); + return scr_stream_next_chunk_impl(s, false, false); } ScrPromise *scr_stream_next_chunk_dyn(ScrStream *s) { - return scr_stream_next_chunk_impl(s, true); + return scr_stream_next_chunk_impl(s, true, false); +} + +ScrPromise *scr_stream_next_result_dyn(ScrStream *s) { + return scr_stream_next_chunk_impl(s, true, true); } /* ── Readable.from (array, string and Buffer sources) ─────────────────── */ @@ -7524,6 +7541,18 @@ static ScrStream *scr_stream_boxed(const ScrDyn *value) { return NULL; } +/* The native readable stream behind a checked-dynamic value (+1): a + * boxed stream or a Cipheriv/Decipheriv handle's Transform, or NULL. */ +ScrStream *scr_stream_native_readable_of_dyn(const ScrDyn *value) { + ScrStream *s = scr_stream_of_handle(value); + if (!s) s = scr_stream_boxed(value); + if (s && !s->st->has_r) { + scr_stream_release(s); + return NULL; + } + return s; +} + /* stream.handleStream: the stream of a value typed as a stream handle * class (a Cipheriv or Decipheriv), +1. A value of that type can also be * the stream itself, boxed (the stream addAbortSignal answers). */ diff --git a/packages/runtime/src/scr_stream.h b/packages/runtime/src/scr_stream.h index 0e0416e05..623f60465 100644 --- a/packages/runtime/src/scr_stream.h +++ b/packages/runtime/src/scr_stream.h @@ -301,6 +301,14 @@ bool scr_stream_write_dyn(ScrStream *s, const ScrDyn *d, ScrClosure *cb); /* cb bool scr_stream_write_enc(ScrStream *s, const ScrDyn *chunk, ScrDyn *enc, ScrClosure *cb); ScrPromise *scr_stream_next_chunk(ScrStream *s); ScrPromise *scr_stream_next_chunk_dyn(ScrStream *s); +/* The same step resolving with an iterator result ({ value, done }). */ +ScrPromise *scr_stream_next_result_dyn(ScrStream *s); +/* The native readable stream behind a checked-dynamic value (+1), or NULL. */ +ScrStream *scr_stream_native_readable_of_dyn(const struct ScrDyn *value); +/* for await over a checked-dynamic value in a program with streams + * (scr_stream_ops.c): a native readable stream iterates as a typed loop + * does; any other value takes scr_dyn_async_iterator_open. */ +struct ScrDyn *scr_stream_dyn_async_iterator_open(struct ScrDyn *value, ScrStr *spell); /* node:stream/promises — the promise forms over the finished/pipeline * machinery above: a pending void promise the terminal watcher settles * (fulfilled on a clean finish, rejected with the finish status diff --git a/tests/corpus/7420-for-await-close.ts b/tests/corpus/7420-for-await-close.ts new file mode 100644 index 000000000..52930a02c --- /dev/null +++ b/tests/corpus/7420-for-await-close.ts @@ -0,0 +1,100 @@ +// Leaving a for await loop over an async generator early closes it, as +// AsyncIteratorClose does: return, throw and break inside the body, a +// labeled continue to an outer loop, and a return() at a yield in the +// body all await the generator's return(), so its finally runs before the +// loop's caller continues. An exhausted generator and a rejected next() +// are not closed. After a throw, an error from closing is dropped; after +// a break or return it propagates. +const log: string[] = []; + +async function* numbers(tag: string, failClose = false): AsyncGenerator { + try { + yield 1; + yield 2; + yield 3; + } finally { + log.push(`${tag} finally`); + if (failClose) throw new Error(`${tag} close failed`); + } +} + +async function* slowClose(tag: string): AsyncGenerator { + try { + yield 1; + } finally { + await null; + log.push(`${tag} finally after await`); + } +} + +async function firstOf(tag: string): Promise { + for await (const n of numbers(tag)) return n * 10; + return -1; +} + +async function throwsInside(tag: string): Promise { + for await (const n of numbers(tag)) throw new Error(`${tag} body ${n}`); +} + +async function* takeOne(tag: string): AsyncGenerator { + for await (const n of numbers(tag)) { + yield n; + return; + } +} + +async function* rethrowing(tag: string): AsyncGenerator { + for await (const n of numbers(tag)) { + if (n === 2) throw new Error(`${tag} at ${n}`); + yield `${tag} ${n}`; + } +} + +async function main(): Promise { + log.push(`first ${await firstOf("return")}`); + try { + await throwsInside("throw"); + } catch (e) { + log.push(`caught ${(e as Error).message}`); + } + for await (const n of takeOne("generator-return")) log.push(`took ${n}`); + try { + for await (const s of rethrowing("generator-throw")) log.push(s); + } catch (e) { + log.push(`caught ${(e as Error).message}`); + } + const early = takeOne("consumer-return"); + for await (const n of early) { + log.push(`consumer got ${n}`); + break; + } + try { + for await (const n of numbers("break-close-fails", true)) { + log.push(`break at ${n}`); + break; + } + } catch (e) { + log.push(`caught ${(e as Error).message}`); + } + try { + for await (const n of numbers("throw-close-fails", true)) throw new Error(`body ${n}`); + } catch (e) { + log.push(`caught ${(e as Error).message}`); + } + void Promise.resolve().then(() => log.push("microtask")); + for await (const n of slowClose("slow")) { + log.push(`slow ${n}`); + break; + } + log.push("after slow"); + outer: for (const k of [1, 2]) { + for await (const n of numbers(`labeled ${k}`)) { + log.push(`labeled ${k} got ${n}`); + continue outer; + } + } + for await (const n of numbers("exhausted")) log.push(`exhausted ${n}`); + for (const line of log) console.log(line); +} + +void main(); diff --git a/tests/corpus/7421-for-await-close.mjs b/tests/corpus/7421-for-await-close.mjs new file mode 100644 index 000000000..c891a2084 --- /dev/null +++ b/tests/corpus/7421-for-await-close.mjs @@ -0,0 +1,53 @@ +// The same early exits in JavaScript: a for await loop that returns, +// throws or breaks closes its async generator, including one that reaches +// the loop as an untyped parameter, and a source whose return() is still +// running finishes before the code after the loop. +const log = []; + +async function* letters(tag) { + try { + yield "a"; + yield "b"; + } finally { + log.push(`${tag} finally`); + } +} + +async function firstLetter(source) { + for await (const c of source) return c; +} + +async function failOn(source, target) { + for await (const c of source) { + if (c === target) throw new Error(`failed on ${c}`); + log.push(`kept ${c}`); + } +} + +async function* upperUntil(source, stop) { + for await (const c of source) { + if (c === stop) return; + yield c.toUpperCase(); + } +} + +log.push(`first ${await firstLetter(letters("param-return"))}`); +try { + await failOn(letters("param-throw"), "b"); +} catch (e) { + log.push(`caught ${e.message}`); +} +for await (const c of upperUntil(letters("stage-return"), "b")) log.push(`upper ${c}`); +let ticks = 0; +const tick = () => { + log.push(`tick ${ticks}`); + if (++ticks < 4) process.nextTick(tick); +}; +process.nextTick(tick); +for await (const c of letters("break")) { + log.push(`break ${c}`); + break; +} +log.push("after break"); +await new Promise((resolve) => setTimeout(resolve, 1)); +console.log(log.join("\n")); diff --git a/tests/corpus/7422-for-await-untyped-streams.mjs b/tests/corpus/7422-for-await-untyped-streams.mjs new file mode 100644 index 000000000..b7dd21072 --- /dev/null +++ b/tests/corpus/7422-for-await-untyped-streams.mjs @@ -0,0 +1,38 @@ +// for await over a native stream that reaches the loop as an untyped +// value iterates it as a typed loop does, in the same tick order as Node: +// whole reads, an early return (the stream is destroyed), a break, an +// object-mode Transform and a stream error. +import { Readable, PassThrough, Transform } from "node:stream"; +const log = []; +async function readAll(s, tag) { + for await (const c of s) log.push(`${tag} ${typeof c} ${String(c)}`); + log.push(`${tag} done`); +} +async function first(s, tag) { + for await (const c of s) return `${tag} first ${String(c)}`; +} +async function breaks(s, tag) { + for await (const c of s) { log.push(`${tag} got ${String(c)}`); break; } +} +async function failing(s, tag) { + try { for await (const c of s) log.push(`${tag} ${c}`); } catch (e) { log.push(`${tag} caught ${e.message}`); } +} +let n = 0; +const ticker = () => { log.push(`tick ${n}`); if (++n < 6) process.nextTick(ticker); }; +process.nextTick(ticker); +await readAll(Readable.from(["a", "b"]), "arr"); +const r = Readable.from(["x", "y", "z"]); +r.on("close", () => log.push(`r close ${r.destroyed}`)); +log.push(await first(r, "r")); +const pt = new PassThrough(); +pt.on("close", () => log.push("pt close")); +pt.end("bytes"); +await breaks(pt, "pt"); +const t = new Transform({ objectMode: true, transform(c, e, cb) { cb(null, { v: c }); } }); +t.end(1); +await (async (s) => { for await (const c of s) log.push(`t ${JSON.stringify(c)}`); })(t); +const bad = new Readable({ read() { this.destroy(new Error("boom")); } }); +await failing(bad, "bad"); +await readAll(["plain", "array"], "dyn"); +await new Promise((res) => setTimeout(res, 5)); +console.log(log.join("\n")); diff --git a/tests/corpus/7423-stream-untyped-bindings.mjs b/tests/corpus/7423-stream-untyped-bindings.mjs new file mode 100644 index 000000000..23b9e1411 --- /dev/null +++ b/tests/corpus/7423-stream-untyped-bindings.mjs @@ -0,0 +1,33 @@ +// Streams held where the lowering keeps a checked-dynamic value: a +// module-level binding of a stream method's result on a value that a +// JavaScript function returns, an element read of an array of streams, +// and readable.compose on a stream whose function is typed with JSDoc. +import { PassThrough, Readable, Transform } from "node:stream"; + +const words = () => Readable.from(["alpha", "beta"]); +const shouted = words().map((w) => w.toUpperCase()); +console.log("shouted", shouted.readable, shouted.readableObjectMode); +for await (const w of shouted) console.log("shouted", w); + +const counted = words().filter((w) => w.length > 4); +counted.on("data", (w) => console.log("counted", w)); +await new Promise((resolve) => counted.on("end", resolve)); +console.log("counted ended", counted.readableEnded); + +const streams = [new PassThrough({ objectMode: true }), new Transform({ + objectMode: true, + transform(chunk, _enc, cb) { + cb(null, { wrapped: chunk }); + }, +})]; +streams[1].end(7); +for await (const v of streams[1]) console.log("element", JSON.stringify(v)); +streams[0].end("done"); +for await (const v of streams[0]) console.log("element", v); + +/** @returns {Readable} */ +const typedWords = () => Readable.from(["x", "yy"]); +const lengths = typedWords().compose(async function* (source) { + for await (const w of source) yield `${w}:${w.length}`; +}); +console.log("composed", (await lengths.toArray()).join(",")); diff --git a/tests/harness/corpus-costs/7420-for-await-close.ts.json b/tests/harness/corpus-costs/7420-for-await-close.ts.json new file mode 100644 index 000000000..2df20a0ed --- /dev/null +++ b/tests/harness/corpus-costs/7420-for-await-close.ts.json @@ -0,0 +1 @@ +{ "plain": 2100, "san": 2800 } diff --git a/tests/harness/corpus-costs/7421-for-await-close.mjs.json b/tests/harness/corpus-costs/7421-for-await-close.mjs.json new file mode 100644 index 000000000..cb701dc93 --- /dev/null +++ b/tests/harness/corpus-costs/7421-for-await-close.mjs.json @@ -0,0 +1 @@ +{ "plain": 1900, "san": 2600 } diff --git a/tests/harness/corpus-costs/7422-for-await-untyped-streams.mjs.json b/tests/harness/corpus-costs/7422-for-await-untyped-streams.mjs.json new file mode 100644 index 000000000..cb701dc93 --- /dev/null +++ b/tests/harness/corpus-costs/7422-for-await-untyped-streams.mjs.json @@ -0,0 +1 @@ +{ "plain": 1900, "san": 2600 } diff --git a/tests/harness/corpus-costs/7423-stream-untyped-bindings.mjs.json b/tests/harness/corpus-costs/7423-stream-untyped-bindings.mjs.json new file mode 100644 index 000000000..d2ac5db99 --- /dev/null +++ b/tests/harness/corpus-costs/7423-stream-untyped-bindings.mjs.json @@ -0,0 +1 @@ +{ "plain": 4900, "san": 6454 } diff --git a/tests/harness/self-hosting-native-toolchain.test.ts b/tests/harness/self-hosting-native-toolchain.test.ts index c4013d288..f143419dc 100644 --- a/tests/harness/self-hosting-native-toolchain.test.ts +++ b/tests/harness/self-hosting-native-toolchain.test.ts @@ -108,7 +108,7 @@ for (const backend of ["llvm"] as const) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 28, marker: "scr_runtime_abi_v28" }, + runtime_abi: { version: 29, marker: "scr_runtime_abi_v29" }, compiler: { command: "fixture", identity: "fixture", target: target.llvmTriple }, macros: { executable: [], excluded: [], sanitizer: "external-toolchain-required" }, flavors: { From ef8ee2b3939b342b83be6a169ac256b7ba77cf84 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 13:03:33 -0500 Subject: [PATCH 062/197] Emit a debug build's function bodies in one emitter Debug metadata is numbered per emitter (debug-info.ts) and the range journal does not carry it, so a debug build emitted in ranges or on workers referred to metadata the merged module did not define. Debug builds now emit every function body in the emitter itself. --- .../compiler/src/backend/llvm/emit-module.ts | 7 ++- packages/compiler/src/backend/llvm/emitter.ts | 10 ++-- .../compiler/test/chunked-emission.test.ts | 53 +++++++++++++++++++ 3 files changed, 65 insertions(+), 5 deletions(-) diff --git a/packages/compiler/src/backend/llvm/emit-module.ts b/packages/compiler/src/backend/llvm/emit-module.ts index 75e6ad50f..1f66e566f 100644 --- a/packages/compiler/src/backend/llvm/emit-module.ts +++ b/packages/compiler/src/backend/llvm/emit-module.ts @@ -192,10 +192,13 @@ function readRange(dir: string, range: number): { result: ChunkResult; texts: st export function emitModuleParts(emitter: LlEmitter): string[] { // Function bodies first (the literal/unit/fn-value tables fill as they // emit), then the file assembles around them — the runtime ABI’s order. + // Debug metadata is numbered per emitter and not journaled, so a debug + // build emits every body here. + const ranges = emitter.debug === null; const chunks = emitter.targetOptions.functionChunks ?? 1; const fnDefs = - (chunks > 1 ? emitFunctionsInChunks(emitter, chunks) : null) ?? - emitFunctionsOnWorkers(emitter, emitter.targetOptions.functionWorkers ?? 1) ?? + (ranges && chunks > 1 ? emitFunctionsInChunks(emitter, chunks) : null) ?? + (ranges ? emitFunctionsOnWorkers(emitter, emitter.targetOptions.functionWorkers ?? 1) : null) ?? []; if (fnDefs.length === 0) for (const fn of emitter.mod.functions) { diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 1ca9313b7..2daabdb92 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -414,11 +414,13 @@ export interface LlvmTargetOptions { targetTriple?: string; /** Emit the function bodies in this many contiguous ranges, each by its * own emitter, and merge them in function order (emit-journal.ts). The - * output is the same as one emitter's. */ + * output is the same as one emitter's. Debug builds (`debugSources`) + * emit in one emitter. */ functionChunks?: number; /** Emit the function bodies of a large module on this many forked worker * processes at a time (emit-module.ts), forked with `forkPrimitives`. The - * output is the same as one emitter's. */ + * output is the same as one emitter's. Debug builds (`debugSources`) + * emit in one emitter. */ functionWorkers?: number; /** The host's process primitives, where it can fork (the native * compiler's). */ @@ -1273,7 +1275,9 @@ export class LlEmitter { this.journal = journal; this.journaled = journal !== null || - ((options.functionWorkers ?? 1) > 1 && options.forkPrimitives !== undefined); + ((options.functionWorkers ?? 1) > 1 && + options.forkPrimitives !== undefined && + options.debugSources === undefined); this.stackTraces = observesStacks(mod); this.nullableUnions = new NullableUnions(mod, options.debugSources === undefined); this.mockTimers = this.runtimeFeatures.mockTimers; diff --git a/packages/compiler/test/chunked-emission.test.ts b/packages/compiler/test/chunked-emission.test.ts index 9902569f7..b4912c9f6 100644 --- a/packages/compiler/test/chunked-emission.test.ts +++ b/packages/compiler/test/chunked-emission.test.ts @@ -67,3 +67,56 @@ test("function bodies emitted in chunks merge into the text one emitter gives", await rm(dir, { recursive: true, force: true }); } }, 600_000); + +// Debug metadata is numbered per emitter (debug-info.ts) and not journaled: +// a debug build emits its function bodies in one emitter. +test("debug builds emit function bodies serially, with every metadata reference defined", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-chunked-debug-")); + const minFunctions = process.env["SCRIPTC_EMIT_WORKER_FUNCTIONS"]; + try { + const corpus = join(import.meta.dirname, "../../../tests/corpus/3066-source-debugging"); + const entry = join(corpus, "main.ts"); + const outPath = join(dir, "main.ir.json"); + const result = await compile(entry, { + outDir: dir, + outPath, + outputKind: "ir", + optimization: "dev", + }); + if (!result.ok) + throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); + const ir = await readFile(outPath, "utf8"); + const debugSources = new Map(); + for (const name of ["main.ts", "helper.ts"]) + debugSources.set(join(corpus, name), await readFile(join(corpus, name), "utf8")); + const options: LlvmTargetOptions = { debugSources }; + const serial = text(emitLlvmModuleSource(deserializeModule(ir), options)); + expect(serial).toContain("!DILocation"); + + let forks = 0; + process.env["SCRIPTC_EMIT_WORKER_FUNCTIONS"] = "1"; + const forkPrimitives = { + fork: (): number => { + forks++; + return -1; + }, + exit: (): void => {}, + wait: (): number => -1, + }; + for (const variant of [ + { functionChunks: 3 }, + { functionWorkers: 4, forkPrimitives }, + ] satisfies LlvmTargetOptions[]) { + const emitted = text(emitLlvmModuleSource(deserializeModule(ir), { ...options, ...variant })); + expect(emitted === serial, JSON.stringify(Object.keys(variant))).toBe(true); + const defined = new Set([...emitted.matchAll(/^(![0-9]+) = /gm)].map((m) => m[1])); + const referenced = new Set([...emitted.matchAll(/(![0-9]+)\b/g)].map((m) => m[1])); + expect([...referenced].filter((id) => !defined.has(id))).toEqual([]); + } + expect(forks).toBe(0); + } finally { + if (minFunctions === undefined) delete process.env["SCRIPTC_EMIT_WORKER_FUNCTIONS"]; + else process.env["SCRIPTC_EMIT_WORKER_FUNCTIONS"] = minFunctions; + await rm(dir, { recursive: true, force: true }); + } +}, 120_000); From 8f08cbbec594caa07af7164bb57882209f81fbcd Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 13:09:01 -0500 Subject: [PATCH 063/197] Build Sets, Maps and weak collections from checked-dynamic iterables `new Set(x)`, `new Map(x)`, `new WeakSet(x)`, `new WeakMap(x)` and `Object.fromEntries(x)` now accept any checked-dynamic iterable, as in Node: a native iterator (`map.keys()`, `set.values()`), a compiled generator, an object with a Symbol.iterator method, or a Set. Before, a Set seed that was not an array or a string threw "is not iterable", and a Map, weak collection or Object.fromEntries seed that was a generator or a collection other than a Map refused with SC2020. The runtime drains native iterators, packs arrays, strings and collections, and steps every other iterable through its iterator (scr_dyn_seed_values). A program that builds these from an untyped seed registers the compiled generators' next and return methods with the runtime, as destructuring rest already did, so a generator seed can be stepped. The seed's values are all taken before the first one is added, so when an entry of a Map seed is not an object the TypeError comes after the iterable is exhausted; Node closes the iterator at that entry. The limitations page says so. --- .../limitations/builtin-apis/map-and-set.mdx | 2 ++ .../src/frontend/lowering/dispatch-sites.ts | 14 +++++++- .../6926-dynamic-collection-seeds.mjs.json | 6 ++++ .../6927-dynamic-collection-seeds.ts.json | 6 ++++ packages/runtime/src/scr_json.c | 32 +++++++++++-------- .../corpus/6926-dynamic-collection-seeds.mjs | 28 ++++++++++++++++ tests/corpus/6927-dynamic-collection-seeds.ts | 9 ++++++ .../6926-dynamic-collection-seeds.mjs.json | 1 + .../6927-dynamic-collection-seeds.ts.json | 1 + 9 files changed, 85 insertions(+), 14 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6926-dynamic-collection-seeds.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6927-dynamic-collection-seeds.ts.json create mode 100644 tests/corpus/6926-dynamic-collection-seeds.mjs create mode 100644 tests/corpus/6927-dynamic-collection-seeds.ts create mode 100644 tests/harness/corpus-costs/6926-dynamic-collection-seeds.mjs.json create mode 100644 tests/harness/corpus-costs/6927-dynamic-collection-seeds.ts.json diff --git a/docs/content/limitations/builtin-apis/map-and-set.mdx b/docs/content/limitations/builtin-apis/map-and-set.mdx index bf77001ee..ee7435b2d 100644 --- a/docs/content/limitations/builtin-apis/map-and-set.mdx +++ b/docs/content/limitations/builtin-apis/map-and-set.mdx @@ -7,4 +7,6 @@ They support `.next()`, `for...of`, array spreads, and mapper-less `Array.from`. They also support the iterator helper methods, such as `.map()` and `.take()`. Detached collection methods are unsupported. + Constructing a Map, Set, WeakMap or WeakSet, or calling `Object.fromEntries`, with a checked-dynamic iterable (a native iterator, a generator, or an object with a `Symbol.iterator` method) takes all of its values before adding the first one. + When an entry of a Map seed is not an object, the TypeError comes after the iterable is exhausted; Node closes the iterator at that entry. The Set composition methods (`union`, `intersection`, `difference`, `symmetricDifference`, `isSubsetOf`, `isSupersetOf`, `isDisjointFrom`) accept a Set of the same element type or a Map keyed by it; other set-like arguments are unsupported. diff --git a/packages/compiler/src/frontend/lowering/dispatch-sites.ts b/packages/compiler/src/frontend/lowering/dispatch-sites.ts index 2618dacf9..c8a8d6ba6 100644 --- a/packages/compiler/src/frontend/lowering/dispatch-sites.ts +++ b/packages/compiler/src/frontend/lowering/dispatch-sites.ts @@ -11,6 +11,18 @@ import { everyStmtList } from "../../ir/traverse.js"; import { toPrimitiveHint } from "./class-dynamic-dispatch.js"; import { GENERATOR_STEP_METHODS, HELPER_METHODS } from "./generator-dynamic-dispatch.js"; +/** Runtime entry points that step a checked-dynamic iterable's iterator: + * destructuring rest, and the collection constructors' and + * Object.fromEntries' seeds. */ +const RUNTIME_STEP_LIB_FNS: ReadonlySet = new Set([ + "dyn.destructureRest", + "dyn.nativeSetNew", + "dyn.mapSeedEntries", + "dyn.fromEntries", + "weakMap.new", + "weakSet.new", +]); + /** What one function's body holds for the class dispatch pass and its * caught-value, generator and class construction sub-passes, collected in * one walk. Lists keep the order the walk meets their entries. */ @@ -186,7 +198,7 @@ function scanSites(statements: IrStmt[]): FunctionSites { expr.key.kind === "strLit" && HELPER_METHODS.has(expr.key.value)) || expr.kind === "dynIterN" || - (expr.kind === "libCall" && expr.fn === "dyn.destructureRest") + (expr.kind === "libCall" && RUNTIME_STEP_LIB_FNS.has(expr.fn)) ) sites.runtimeSteps = true; switch (expr.kind) { diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6926-dynamic-collection-seeds.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6926-dynamic-collection-seeds.mjs.json new file mode 100644 index 000000000..32edfc402 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6926-dynamic-collection-seeds.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6926-dynamic-collection-seeds.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6927-dynamic-collection-seeds.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6927-dynamic-collection-seeds.ts.json new file mode 100644 index 000000000..89c591dd7 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6927-dynamic-collection-seeds.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6927-dynamic-collection-seeds.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index 1c85a928a..e4b66990d 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -1577,6 +1577,20 @@ ScrDyn *scr_dyn_iter_pack(const ScrDyn *src, const ScrStr *msg) { static ScrDyn *scr_native_iterator_drain(const ScrDyn *src); +/* The values a collection constructor's iterable seed gives, in order: +1 + * array, or NULL with the exception pending. A native iterator (a Map's + * keys(), a Set's values()) drains, which runs no program code; arrays, + * strings, typed arrays and collections pack; other iterables (a + * generator, an object with a Symbol.iterator method) step their + * iterator to the end, so their values are all taken before the + * constructor adds the first one. */ +static ScrDyn *scr_dyn_seed_values(const ScrDyn *src) { + if (src->kind == SCR_DYN_HANDLE && + (src->v.handle.tag == SCR_DYNH_ITERATOR || scr_dyn_handle_ops_of(src)->iter_step)) + return scr_native_iterator_drain(src); + return scr_dyn_destructure_rest(src, NULL); +} + /* The for-of-over-dyn pack accessors: the emitted index loop drives them * over a scr_dyn_iter_pack result (ARR by construction — the defensive * arms cover nothing reachable from that lowering). Never throw. */ @@ -1592,18 +1606,10 @@ ScrDyn *scr_dyn_map_seed_entries(const ScrDyn *src) { if (scr_exc_pending()) return NULL; } /* A native iterator (sp.entries(), a Map's entries()) or a handle that - * steps through live entries (URLSearchParams): producing its entries - * runs no program code, so draining them first is what the consumer - * observes. */ - if (src->kind == SCR_DYN_HANDLE && - (src->v.handle.tag == SCR_DYNH_ITERATOR || scr_dyn_handle_ops_of(src)->iter_step)) - return scr_native_iterator_drain(src); - if (src->kind == SCR_DYN_HANDLE || src->kind == SCR_DYN_TYPED_REF) { - static const char message[] = "new Map(entries) over native non-array iterables has no lowering"; - scr_throw_error_msg_code(SCR_ERR_ERROR, message, sizeof message - 1, "SC2020"); - return NULL; - } - return scr_dyn_iter_pack(src, NULL); + * steps through live entries (URLSearchParams) drains; a Set, a + * generator or another iterable gives its values in order + * (scr_dyn_seed_values). */ + return scr_dyn_seed_values(src); } ScrDyn *scr_dyn_map_seed_entry(const ScrDyn *entry) { @@ -14836,7 +14842,7 @@ ScrDyn *scr_dyn_native_set(ScrMap *value) { ScrDyn *scr_dyn_native_set_new(const ScrDyn *seed) { ScrDyn *items = NULL; if (seed->kind != SCR_DYN_NULL && seed->kind != SCR_DYN_UNDEF) { - items = scr_dyn_iter_pack(seed, NULL); + items = scr_dyn_seed_values(seed); if (!items) return NULL; } ScrMap *set = scr_map_new(SCR_MAP_KEY_DYN, SCR_MAP_VAL_F64, &scr_map_dyn_set_ops); diff --git a/tests/corpus/6926-dynamic-collection-seeds.mjs b/tests/corpus/6926-dynamic-collection-seeds.mjs new file mode 100644 index 000000000..a48fab894 --- /dev/null +++ b/tests/corpus/6926-dynamic-collection-seeds.mjs @@ -0,0 +1,28 @@ +// Set, Map, WeakSet and Object.fromEntries over checked-dynamic iterables: +// native iterators, generators, objects with a Symbol.iterator method and +// collections, as seeds that reach the runtime untyped. +function* pairs() { yield ["g", 1]; yield ["h", 2]; } +function* values() { yield 1; yield 2; yield 2; } +const iterable = { *[Symbol.iterator]() { yield ["i", 3]; } }; +const valueIterable = { *[Symbol.iterator]() { yield "a"; yield "b"; } }; +function attempt(label, make) { + try { console.log(label, make()); } catch (e) { console.log(label, "threw", e.name, e.message); } +} +function viaSet(x) { return new Set(x).size; } +function viaMap(x) { return [...new Map(x).keys()].join(); } +function viaWeakSet(x) { return new WeakSet(x).has(key); } +const key = {}; +const m = new Map([["a", 1], ["b", 2]]); +attempt("set keys()", () => viaSet(m.keys())); +attempt("set values()", () => viaSet(new Set([1, 2]).values())); +attempt("set generator", () => viaSet(values())); +attempt("set iterable", () => viaSet(valueIterable)); +attempt("set string", () => viaSet("abca")); +attempt("set number", () => viaSet(5)); +attempt("map generator", () => viaMap(pairs())); +attempt("map iterable", () => viaMap(iterable)); +attempt("map set", () => viaMap(new Set([["s", 1]]))); +attempt("map entries()", () => viaMap(m.entries())); +attempt("map bad entry", () => viaMap([1])); +attempt("weakset generator", () => viaWeakSet((function* () { yield key; })())); +attempt("fromEntries generator", () => JSON.stringify(Object.fromEntries(pairs()))); diff --git a/tests/corpus/6927-dynamic-collection-seeds.ts b/tests/corpus/6927-dynamic-collection-seeds.ts new file mode 100644 index 000000000..c57433d57 --- /dev/null +++ b/tests/corpus/6927-dynamic-collection-seeds.ts @@ -0,0 +1,9 @@ +// Set and Map seeds held as any: native iterators and generators. +function* values(): Generator { yield 1; yield 2; yield 2; } +function* pairs(): Generator<[string, number]> { yield ["a", 1]; yield ["b", 2]; } +const m = new Map([["x", 1], ["y", 2]]); +const anyKeys: any = m.keys(); +const anyGen: any = values(); +const anyPairs: any = pairs(); +console.log(new Set(anyKeys).size, new Set(anyGen).size, new Map(anyPairs).size); +console.log(new Set(values()).size, new Map(pairs()).get("b"), new Set(m.values()).size); diff --git a/tests/harness/corpus-costs/6926-dynamic-collection-seeds.mjs.json b/tests/harness/corpus-costs/6926-dynamic-collection-seeds.mjs.json new file mode 100644 index 000000000..b4a56f5c5 --- /dev/null +++ b/tests/harness/corpus-costs/6926-dynamic-collection-seeds.mjs.json @@ -0,0 +1 @@ +{ "plain": 3813, "san": 3374 } diff --git a/tests/harness/corpus-costs/6927-dynamic-collection-seeds.ts.json b/tests/harness/corpus-costs/6927-dynamic-collection-seeds.ts.json new file mode 100644 index 000000000..93da087d2 --- /dev/null +++ b/tests/harness/corpus-costs/6927-dynamic-collection-seeds.ts.json @@ -0,0 +1 @@ +{ "plain": 2436, "san": 2273 } From dc9b60656d010567bb47c6c4c4f4ba14a3254f4e Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 13:11:34 -0500 Subject: [PATCH 064/197] Index arrays with exact i64 products of int32 values `a[i * n + k]` with int32 operands kept the index in doubles whenever the product's range could pass 2^53: integer arithmetic needs a proven safe range, and an int32 product can reach 2^62. Every access then converted the double back with an exactness test. Such arithmetic now also carries a wide view: the exact integer as an i64, the magnitude it is bounded by, and a threshold below which the double equals it. A product of two exact integers equals its double below 2^53 in magnitude, and a sum or difference of a wide view exact below T and an exact integer of magnitude at most B equals its double below min(T - B, 2^53). Every i64 result stays within 2^62 + 2^53, so none wraps. An array index below a length or capacity is below 2^32, under every threshold kept, so the dense array guards and local array reads compare the wide view, unsigned, with the array's counts and skip the round trip. Any index that fails those comparisons, including every index whose double rounds, takes the double's path as before. Typed arrays and strings are unchanged. --- .../src/backend/llvm/dense-array-access.ts | 15 ++-- .../compiler/src/backend/llvm/expr-context.ts | 12 +++ .../src/backend/llvm/expr-primitives.ts | 10 ++- .../src/backend/llvm/integer-values.ts | 85 ++++++++++++++++++- .../src/backend/llvm/local-array-reads.ts | 22 +++-- .../test/array-index-product-emission.test.ts | 55 ++++++++++++ ...325-array-index-products-near-2-53.ts.json | 6 ++ .../7325-array-index-products-near-2-53.ts | 82 ++++++++++++++++++ ...325-array-index-products-near-2-53.ts.json | 1 + 9 files changed, 271 insertions(+), 17 deletions(-) create mode 100644 packages/compiler/test/array-index-product-emission.test.ts create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7325-array-index-products-near-2-53.ts.json create mode 100644 tests/corpus/7325-array-index-products-near-2-53.ts create mode 100644 tests/harness/corpus-costs/7325-array-index-products-near-2-53.ts.json diff --git a/packages/compiler/src/backend/llvm/dense-array-access.ts b/packages/compiler/src/backend/llvm/dense-array-access.ts index 4488f90cb..ba3b48b02 100644 --- a/packages/compiler/src/backend/llvm/dense-array-access.ts +++ b/packages/compiler/src/backend/llvm/dense-array-access.ts @@ -27,7 +27,7 @@ import { storeArrCount, } from "./common.js"; import type { LlValue, LlvmEmitterContext } from "./expr-context.js"; -import { exactInteger, widenInteger } from "./integer-values.js"; +import { indexInteger } from "./integer-values.js"; const SCR_ARR_VALUE = 1; @@ -74,13 +74,16 @@ function emitDenseGuard( ): DenseGuard { const B = host.B; const size = host.sizeType; - const integer = exactInteger(host, index, indexExpr); + const integer = indexInteger(host, index, indexExpr); let wide: string; let integral: string | null = null; - if (integer) { - // A proven integer needs no double round-trip. Negative signed values - // compare above any length or capacity as unsigned 64-bit integers. - wide = widenInteger(host, integer); + if (integer !== null) { + // A proven integer, or a wide view that equals the double at every + // index below a length, needs no double round-trip. Negative signed + // values compare above any length or capacity as unsigned 64-bit + // integers, and every index that fails the comparisons takes the + // double's path. + wide = integer; } else { // freeze keeps NaN/out-of-range conversions from producing poison; the // exact round-trip comparison then rejects every value that is not an diff --git a/packages/compiler/src/backend/llvm/expr-context.ts b/packages/compiler/src/backend/llvm/expr-context.ts index 9dd6bbcac..df6dfc608 100644 --- a/packages/compiler/src/backend/llvm/expr-context.ts +++ b/packages/compiler/src/backend/llvm/expr-context.ts @@ -15,11 +15,23 @@ export interface LlValue { /** An exact integer snapshot, excluding negative zero, with no discarded * high bits and the proven range of its original value. */ integer?: { name: string; type: "i32" | "i64"; signed: boolean; range: IntegerRange }; + /** An i64 holding the exact integer that the arithmetic producing this + * number computes, when that integer has no proven safe range: its + * absolute value is at most `bound`, and the double equals it whenever + * its absolute value is below `below` (a zero may be -0). Index consumers + * use it in place of a round trip (integer-values.ts `indexInteger`). */ + wideInteger?: WideInteger; slot?: boolean; /** A sequence local held in a capture box; release the box itself. */ boxed?: boolean; } +export interface WideInteger { + name: string; + below: number; + bound: number; +} + export type ExprOf = Extract; export type LibCallExpr = ExprOf<"libCall">; type LibCallPrefixOf = T extends `${infer Prefix}.${string}` ? Prefix : never; diff --git a/packages/compiler/src/backend/llvm/expr-primitives.ts b/packages/compiler/src/backend/llvm/expr-primitives.ts index 354004e9e..4d3904bde 100644 --- a/packages/compiler/src/backend/llvm/expr-primitives.ts +++ b/packages/compiler/src/backend/llvm/expr-primitives.ts @@ -16,6 +16,7 @@ import { int32Number, integerNumber, narrowInteger, + wideArithmetic, widenInteger, } from "./integer-values.js"; import { @@ -337,7 +338,14 @@ export function emitOperatorExpr( if (arith[e.op] !== undefined) { B.line(`${t} = ${arith[e.op]} double ${l.name}, ${r.name}`); const uint32 = emitWrappingSum(host, e, l, r, li, ri); - if (uint32 !== undefined) return { name: t, type: e.type, uint32 }; + // The exact integer beside the double, for index consumers. + const wideInteger = wideArithmetic(host, e.op, l, li, r, ri); + if (uint32 !== undefined || wideInteger) { + const value: LlValue = { name: t, type: e.type }; + if (uint32 !== undefined) value.uint32 = uint32; + if (wideInteger) value.wideInteger = wideInteger; + return value; + } } else B.line(`${t} = fcmp ${cmp[e.op]} double ${l.name}, ${r.name}`); } else if (bit[e.op] !== undefined) { if (e.left.type.kind !== "f64" || e.right.type.kind !== "f64") { diff --git a/packages/compiler/src/backend/llvm/integer-values.ts b/packages/compiler/src/backend/llvm/integer-values.ts index 63e279d04..ddbcdd8da 100644 --- a/packages/compiler/src/backend/llvm/integer-values.ts +++ b/packages/compiler/src/backend/llvm/integer-values.ts @@ -1,6 +1,6 @@ import type { IrExpr, IrType } from "../../ir/ir.js"; import type { IntegerRange } from "../../ir/integer-ranges.js"; -import type { LlValue, LlvmEmitterContext } from "./expr-context.js"; +import type { LlValue, LlvmEmitterContext, WideInteger } from "./expr-context.js"; export type LlInteger = NonNullable; @@ -51,6 +51,89 @@ export function exactInteger( return { name, type: "i64", signed: true, range }; } +/** Doubles hold every integer of smaller magnitude exactly. */ +const EXACT_DOUBLES = 2 ** 53; +/** Wide views stay within this magnitude: a product of two int32s reaches + * 2^62, one exact term of at most 2^53 may be added to it, and no i64 + * result wraps. */ +const WIDE_BOUND = 2 ** 62 + 2 ** 53; +/** Array and string indexes are below 2^32 - 1: a wide view exact below + * this many is exact at every index a length can admit. */ +const INDEX_LIMIT = 2 ** 32; + +function magnitude(range: IntegerRange): number { + return Math.max(Math.abs(range.min), Math.abs(range.max)); +} + +/** The wide view of `l op r` for `+`, `-` and `*` when the exact operands + * have no proven result range (their product or sum may pass 2^53). + * + * - A product of two exact integers equals its double whenever its + * magnitude is below 2^53: the double product of integers is exact there. + * - A sum or difference of two exact integers likewise. + * - A sum or difference of a wide view (exact below T) and an exact integer + * of magnitude at most B equals its double below min(T - B, 2^53): the + * view's magnitude is then below T, so its double is exact, and the + * double sum of two exact integers below 2^53 is exact. + * + * Every i64 result stays within 2^62 + 2^53 in magnitude, so none wraps. A view + * exact below fewer than 2^32 helps no index and is not kept. */ +export function wideArithmetic( + host: LlvmEmitterContext, + op: string, + l: LlValue, + li: LlInteger | null, + r: LlValue, + ri: LlInteger | null, +): WideInteger | null { + if (op !== "+" && op !== "-" && op !== "*") return null; + const view = (value: LlValue, integer: LlInteger | null): WideInteger | null => + integer + ? { name: "", below: Infinity, bound: magnitude(integer.range) } + : (value.wideInteger ?? null); + const lv = view(l, li), + rv = view(r, ri); + if (!lv || !rv) return null; + let below: number, bound: number; + if (op === "*") { + if (lv.below !== Infinity || rv.below !== Infinity) return null; + below = EXACT_DOUBLES; + bound = lv.bound * rv.bound; + } else if (lv.below === Infinity && rv.below === Infinity) { + below = EXACT_DOUBLES; + bound = lv.bound + rv.bound; + } else if (lv.below === Infinity || rv.below === Infinity) { + const [exact, wide] = lv.below === Infinity ? [lv, rv] : [rv, lv]; + below = Math.min(wide.below - exact.bound, EXACT_DOUBLES); + bound = exact.bound + wide.bound; + } else return null; + if (!(bound <= WIDE_BOUND) || below < INDEX_LIMIT) return null; + const operand = (value: LlValue, integer: LlInteger | null): string => + integer ? widenInteger(host, integer) : value.wideInteger!.name; + const left = operand(l, li), + right = operand(r, ri); + const name = host.B.tmp(); + host.B.line( + `${name} = ${op === "+" ? "add" : op === "-" ? "sub" : "mul"} nsw i64 ${left}, ${right}`, + ); + return { name, below, bound }; +} + +/** An i64 index for a consumer that compares it, unsigned, with a length or + * capacity below 2^32 and takes the double's path otherwise: the exact + * integer of `value`, or its wide view, which equals the double at every + * such index. Null when the index needs the double round trip. */ +export function indexInteger( + host: LlvmEmitterContext, + value: LlValue, + expr?: IrExpr, +): string | null { + const integer = exactInteger(host, value, expr); + if (integer) return widenInteger(host, integer); + const wide = value.wideInteger; + return wide && wide.below >= INDEX_LIMIT ? wide.name : null; +} + export function widenInteger(host: LlvmEmitterContext, value: LlInteger): string { if (value.type === "i64") return value.name; const wide = host.B.tmp(); diff --git a/packages/compiler/src/backend/llvm/local-array-reads.ts b/packages/compiler/src/backend/llvm/local-array-reads.ts index cdd00c4e7..b9fc42392 100644 --- a/packages/compiler/src/backend/llvm/local-array-reads.ts +++ b/packages/compiler/src/backend/llvm/local-array-reads.ts @@ -13,7 +13,7 @@ import { analyzeCallLifetimes, type CallLifetimes } from "./call-lifetimes.js"; import type { LlValue, LlvmEmitterContext } from "./expr-context.js"; import { ReferenceEffects } from "./reference-effects.js"; import { SCR_ARR_FIELD, loadArrCount, scrArrStateAt } from "./common.js"; -import { exactInteger, widenInteger } from "./integer-values.js"; +import { indexInteger } from "./integer-values.js"; import { READ_FALLBACK } from "./dense-array-access.js"; export interface LocalArrayRead { @@ -605,11 +605,11 @@ export function emitBorrowedArrayRead( return; } const index = host.emitExpr(read.index); - // A proven integer index (an int32 local, say) needs no double - // round-trip; a negative one compares above the capacity as unsigned. - const exact = - loopIndex === null && host.sizeType === "i64" ? exactInteger(host, index, read.index) : null; - const integerIndex = loopIndex ?? (exact ? widenInteger(host, exact) : null); + // A proven integer index (an int32 local, say), or a wide view exact at + // every index below the capacity, needs no double round-trip; a negative + // one compares above the capacity as unsigned. + const integerIndex = + loopIndex ?? (host.sizeType === "i64" ? indexInteger(host, index, read.index) : null); const slow = B.newLabel("borrow.array.slow"), join = B.newLabel("borrow.array.join"); emitDenseReferenceArrayRead( @@ -644,8 +644,12 @@ export function emitLocalArrayRead( ): { slot: string; type: IrType } | null { const B = host.B; const array = host.emitStableReceiver(read.array, [read.index]); - const integerIndex = inline ? host.emitIntegerLoopIndex(read.index) : null; + const loopIndex = inline ? host.emitIntegerLoopIndex(read.index) : null; const index = host.emitExpr(read.index); + // As in emitBorrowedArrayRead: an exact integer or wide view needs no + // round trip, and only the loop's own counter is an induction variable. + const integerIndex = + loopIndex ?? (inline && host.sizeType === "i64" ? indexInteger(host, index, read.index) : null); const nullable = read.type.kind === "union" ? host.nullableUnions.get(read.type.unionId) : null; if (nullable) { // A nullable union is the element pointer itself: borrowed from an @@ -665,7 +669,7 @@ export function emitLocalArrayRead( array, index, integerIndex, - integerIndex !== null, + loopIndex !== null, store, no, slow, @@ -726,7 +730,7 @@ export function emitLocalArrayRead( array, index, integerIndex, - integerIndex !== null, + loopIndex !== null, storeValue, no, slow, diff --git a/packages/compiler/test/array-index-product-emission.test.ts b/packages/compiler/test/array-index-product-emission.test.ts new file mode 100644 index 000000000..bfd891398 --- /dev/null +++ b/packages/compiler/test/array-index-product-emission.test.ts @@ -0,0 +1,55 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { compile } from "../src/index.js"; + +async function emitLlvm(source: string): Promise { + const dir = await mkdtemp(join(tmpdir(), "scriptc-index-product-")); + try { + const entry = join(dir, "main.ts"); + const outPath = join(dir, "main.ll"); + await writeFile(entry, source); + const result = await compile(entry, { outDir: dir, outPath, outputKind: "llvm" }); + expect(result.ok, JSON.stringify(result.diagnostics)).toBe(true); + return await readFile(outPath, "utf8"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +} + +const source = ` +function at(a: number[], x: number, y: number, k: number): number | undefined { + return a[x * y + k]; +} +function cancel(a: number[], p: number, q: number, r: number, s: number): number | undefined { + return a[p * q - r * s]; +} +const a = [1, 2, 3]; +const v = process.argv.length; +console.log(at(a, v | 0, (v + 1) | 0, -v | 0), cancel(a, v | 0, v | 0, v | 0, v | 0)); +`; + +function body(llvm: string, name: string): string { + return new RegExp(`^define [^@]*@sc_\\w+?_${name}(\\.i32)?\\([^]*?^}`, "m").exec(llvm)?.[0] ?? ""; +} + +test("an int32 product plus an int32 indexes as an exact i64 without a round trip", async () => { + const llvm = await emitLlvm(source); + // The int32 entry: the index is the i64 product and sum, compared with + // the array's counts; the double goes only to the slow path's runtime call. + const at = /^define [^@]*@sc_bf_at\.i32\([^]*?^}/m.exec(llvm)?.[0] ?? ""; + const product = /(%\S+) = mul nsw i64 %\S+, %\S+/.exec(at)?.[1]; + expect(product).toBeDefined(); + expect(at).toContain(`= add nsw i64 ${product}, `); + expect(at).not.toMatch(/fptosi double %\S+ to i64/); +}); + +test("a difference of two int32 products keeps the double's round trip", async () => { + const llvm = await emitLlvm(source); + // Each product may pass 2^53, where its double rounds before the + // difference: the index keeps the exactness test of the double. + const cancel = body(llvm, "cancel"); + expect(cancel).toMatch(/fptosi double %\S+ to i64/); + expect(cancel).toMatch(/fcmp oeq double/); +}); diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7325-array-index-products-near-2-53.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7325-array-index-products-near-2-53.ts.json new file mode 100644 index 000000000..3239c26bf --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7325-array-index-products-near-2-53.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7325-array-index-products-near-2-53.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7325-array-index-products-near-2-53.ts b/tests/corpus/7325-array-index-products-near-2-53.ts new file mode 100644 index 000000000..41e60b09d --- /dev/null +++ b/tests/corpus/7325-array-index-products-near-2-53.ts @@ -0,0 +1,82 @@ +// Element reads and writes indexed by products of int32 values plus or +// minus int32 terms. The products can pass the int32 range and 2^53, where +// a double product rounds: an index that lands in an array's range reads +// and writes that element, and every other one (negative, past the length, +// rounded past 2^53) keeps the property key Node computes from the double. + +function show(label: string, value: unknown): void { + console.log(label, Object.is(value, -0) ? "-0" : String(value)); +} + +function read(a: number[], x: number, y: number, k: number): number | undefined { + return a[x * y + k]; +} + +function readBack(a: number[], x: number, y: number, k: number): number | undefined { + return a[x * y - k]; +} + +function write(a: number[], x: number, y: number, k: number, v: number): void { + a[x * y + k] = v; +} + +function rows(n: number, cols: number, a: number[]): number { + let s = 0; + for (let i = 0; i < n; i++) for (let j = 0; j < cols; j++) s += a[i * cols + j]! * (j + 1); + return s; +} + +const a: number[] = []; +for (let i = 0; i < 64; i++) a.push(i * 10); + +const big = 2147483647; +const half = 65536; + +// In range. +show("3*5+2", read(a, 3 | 0, 5 | 0, 2 | 0)); +show("7*-1+9", read(a, 7 | 0, -1 | 0, 9 | 0)); +show("-0 index", read(a, -1 | 0, 0 | 0, 0 | 0)); +// The product passes the int32 range; the difference lands in range. +show("2^16*2^15-2^31+12", readBack(a, half | 0, 32768 | 0, (-2147483648 * -1 - 12) | 0)); +show("(2^31-1)*1-(2^31-1)+5", read(a, big | 0, 1 | 0, -2147483642 | 0)); +show("2^16*2^16-big+20", readBack(a, half | 0, half | 0, (big - 20) | 0)); +// Out of range or negative: holes and named keys. +show("past end", read(a, 8 | 0, 8 | 0, 1 | 0)); +show("negative", read(a, -3 | 0, 5 | 0, 0 | 0)); +show("2^31*2^31", read(a, -2147483648 | 0, -2147483648 | 0, 7 | 0)); + +// Products around 2^53, where the double product rounds. +const keys: number[] = []; +const x = 134217727; // 2^27 - 1 +const y = 67108865; // 2^26 + 1: x * y = 2^53 + 2^26 - 1 +const near: [number, number, number][] = [ + [x, y, 0], + [x, y, 1], + [x, y, -67108864], + [x, y, -67108863], + [94906265, 94906265, 0], + [94906267, 94906267, -3], + [-2147483648, 2147483647, 2147483647], +]; +for (const [p, q, k] of near) { + const sparse: number[] = []; + write(sparse, p | 0, q | 0, k | 0, 1); + // The key the double computes; a local holds only the double. + const key: number = p * q + k; + keys.push(key); + show(`write ${p}*${q}+${k}`, `${sparse[key]} ${sparse[key + 1]} ${sparse[key - 1]} ${sparse.length}`); + show(`read ${p}*${q}+${k}`, read(sparse, p | 0, q | 0, k | 0)); +} +show("keys", keys.join(",")); + +// A difference of two products above 2^53 keeps the double's rounding. +function cancel(sparse: number[], p: number, q: number, r: number, s: number): number | undefined { + return sparse[p * q - r * s]; +} +const cancelled: number[] = [7, 8, 9]; +show("cancel", cancel(cancelled, x | 0, y | 0, x | 0, y | 0)); +show("cancel+1", cancel(cancelled, x | 0, y | 0, (9007199254740992 / 67108864) | 0, 67108864 | 0)); + +// Row-major sums with a width from JSON. +const dims = JSON.parse("[[8, 8], [4, 16], [16, 4], [3, 7]]") as [number, number][]; +for (const [n, cols] of dims) show(`rows ${n}x${cols}`, rows(n | 0, cols | 0, a)); diff --git a/tests/harness/corpus-costs/7325-array-index-products-near-2-53.ts.json b/tests/harness/corpus-costs/7325-array-index-products-near-2-53.ts.json new file mode 100644 index 000000000..f71f3eb98 --- /dev/null +++ b/tests/harness/corpus-costs/7325-array-index-products-near-2-53.ts.json @@ -0,0 +1 @@ +{ "plain": 1800, "san": 2400 } From d8d75dfa189e3e7d4fce34b285d1dc100dc85dbe Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 13:32:25 -0500 Subject: [PATCH 065/197] Support Duplex.from over streams, pairs, promises and async functions --- .../builtin-apis/streams-and-readline.mdx | 5 +- packages/cli/test/native-link-info.test.ts | 4 +- .../compiler/src/backend/llvm/lib-shared.ts | 10 + packages/compiler/src/backend/runtime-abi.ts | 4 +- .../src/backend/runtime-pack-targets.test.ts | 2 +- .../compiler/src/backend/runtime-pack.test.ts | 2 +- .../src/frontend/lowering/lower-stream.ts | 149 +++++++- packages/compiler/src/ir/builtin-effects.ts | 2 + .../compiler/src/ir/builtin-signatures.ts | 10 + packages/compiler/src/ir/runtime-features.ts | 7 + packages/compiler/src/ir/validate.ts | 30 +- .../test/native-codegen-integration.test.ts | 4 +- .../corpus/7424-duplex-from-streams.mjs.json | 6 + .../corpus/7425-duplex-from-streams.ts.json | 6 + .../corpus/7426-duplex-from-promise.mjs.json | 6 + .../7427-duplex-from-async-function.mjs.json | 6 + .../runtime-darwin-arm64/scripts/build.mjs | 2 +- .../runtime-pack-common/scripts/build.mjs | 2 +- packages/runtime/src/scr_console.c | 2 +- packages/runtime/src/scr_core.h | 2 +- packages/runtime/src/scr_stream.c | 360 +++++++++++++++++- packages/runtime/src/scr_stream.h | 14 + packages/runtime/src/scr_stream_ops.c | 168 +++++++- tests/corpus/7424-duplex-from-streams.mjs | 39 ++ tests/corpus/7425-duplex-from-streams.ts | 72 ++++ tests/corpus/7426-duplex-from-promise.mjs | 27 ++ .../7427-duplex-from-async-function.mjs | 40 ++ .../7424-duplex-from-streams.mjs.json | 1 + .../7425-duplex-from-streams.ts.json | 1 + .../7426-duplex-from-promise.mjs.json | 1 + .../7427-duplex-from-async-function.mjs.json | 1 + .../self-hosting-native-toolchain.test.ts | 2 +- 32 files changed, 958 insertions(+), 29 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7424-duplex-from-streams.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7425-duplex-from-streams.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7426-duplex-from-promise.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7427-duplex-from-async-function.mjs.json create mode 100644 tests/corpus/7424-duplex-from-streams.mjs create mode 100644 tests/corpus/7425-duplex-from-streams.ts create mode 100644 tests/corpus/7426-duplex-from-promise.mjs create mode 100644 tests/corpus/7427-duplex-from-async-function.mjs create mode 100644 tests/harness/corpus-costs/7424-duplex-from-streams.mjs.json create mode 100644 tests/harness/corpus-costs/7425-duplex-from-streams.ts.json create mode 100644 tests/harness/corpus-costs/7426-duplex-from-promise.mjs.json create mode 100644 tests/harness/corpus-costs/7427-duplex-from-async-function.mjs.json diff --git a/docs/content/limitations/builtin-apis/streams-and-readline.mdx b/docs/content/limitations/builtin-apis/streams-and-readline.mdx index 060a0bf95..cfd52e054 100644 --- a/docs/content/limitations/builtin-apis/streams-and-readline.mdx +++ b/docs/content/limitations/builtin-apis/streams-and-readline.mdx @@ -22,8 +22,9 @@ `Readable.from` accepts `objectMode` and `highWaterMark` options. The `Readable` operators (`map`, `filter`, `forEach`, `toArray`, `some`, `find`, `every`, `flatMap`, `drop`, `take`, and `reduce`) accept `concurrency`, `highWaterMark`, and `signal` options; their callbacks take only the chunk, `map` returns strings or Buffers, and `flatMap` returns strings or arrays of them. `Duplex.from` accepts an async generator function, which reads what is written to the Duplex, a `Duplex`, `Transform` or `PassThrough`, which it returns, and the values `Readable.from` accepts, which give a Duplex that is not writable. - A lone `Readable` or `Writable`, a promise, a `{ readable, writable }` pair, a web stream, and a body function that declares the options argument are unsupported in `Duplex.from`. - `stream.compose` accepts streams and async generator functions in any position, and an array, a string, or a generator as the first stage; one stage is `Duplex.from` of it. + It also accepts a lone `Readable` or `Writable` and a `{ readable, writable }` pair, which the Duplex reads and writes through, a promise, whose value it reads, and an async function, which reads what is written and must resolve to `null` or `undefined`. + A web stream and a body function that declares the options argument are unsupported in `Duplex.from`. + `stream.compose` accepts streams and async generator functions in any position, an async function as the last stage, and an array, a string, or a generator as the first stage; one stage is `Duplex.from` of it. Web streams as compose stages are unsupported, and the `ERR_INVALID_ARG_VALUE` message for a stream that is no longer readable or writable names the stream's class instead of printing the stream. `readable.compose(stage, { signal })` is `stream.compose(readable, stage)` with the signal added; the options are an object literal whose `signal` is a variable or a property read. `pipeline` from `node:stream/promises` accepts a trailing `{ signal, end }` object, with or without function stages, and `finished` accepts `readable`, `writable`, `error`, `cleanup`, and `signal` options, with boolean literals for the flags. diff --git a/packages/cli/test/native-link-info.test.ts b/packages/cli/test/native-link-info.test.ts index 484b03e8b..1f2fb241a 100644 --- a/packages/cli/test/native-link-info.test.ts +++ b/packages/cli/test/native-link-info.test.ts @@ -72,7 +72,7 @@ describe.runIf(supported)("macOS arm64 native link info", () => { minimum_os: "14.0", }, program: { object, entry_symbol: "main" }, - runtime_abi: { version: 29, marker: "scr_runtime_abi_v29" }, + runtime_abi: { version: 30, marker: "scr_runtime_abi_v30" }, runtime_pack: { kind: "precompiled", package: "@scriptc/runtime-darwin-arm64", @@ -165,6 +165,6 @@ describe.runIf(supported)("macOS arm64 native link info", () => { () => null, (failure: { stderr?: string }) => failure, ); - expect(error?.stderr).toContain("scr_runtime_abi_v29"); + expect(error?.stderr).toContain("scr_runtime_abi_v30"); }); }); diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index 94318c3aa..a790ad197 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -1129,6 +1129,11 @@ export const LIB_FN_SYMS: Record = { "stream.decodeStrings": "scr_stream_decode_strings", "stream.duplexFromGen": "scr_stream_duplex_from_gen", "stream.duplexFromDyn": "scr_stream_duplex_from_dyn", + "stream.duplexifyReadable": "scr_stream_duplexify_readable", + "stream.duplexifyWritable": "scr_stream_duplexify_writable", + "stream.duplexifyPair": "scr_stream_duplexify_pair", + "stream.duplexFromPromise": "scr_stream_duplex_from_promise", + "stream.duplexFromAsyncFn": "scr_stream_duplex_from_async_fn", "writable.writeEncoding": "scr_stream_write_encoding", "writable.setDefaultEncoding": "scr_stream_set_default_encoding", "stream.chunkDone": "scr_stream_chunk_done", @@ -1737,6 +1742,11 @@ export const USES_TIMERS_LIB_FNS = new Set([ "stream.setConstruct", "stream.duplexFromGen", "stream.duplexFromDyn", + "stream.duplexifyReadable", + "stream.duplexifyWritable", + "stream.duplexifyPair", + "stream.duplexFromPromise", + "stream.duplexFromAsyncFn", "stream.setWritev", "stream.setWritevDefault", "process.activeResources", diff --git a/packages/compiler/src/backend/runtime-abi.ts b/packages/compiler/src/backend/runtime-abi.ts index 948fa6359..fcc8d160e 100644 --- a/packages/compiler/src/backend/runtime-abi.ts +++ b/packages/compiler/src/backend/runtime-abi.ts @@ -2,8 +2,8 @@ * of the link contract: a program object keeps this symbol undefined and the * matching runtime defines it, so incompatible manual links fail before the * program can start. */ -export const RUNTIME_ABI_VERSION = 29 as const; -export const RUNTIME_ABI_MARKER = "scr_runtime_abi_v29" as const; +export const RUNTIME_ABI_VERSION = 30 as const; +export const RUNTIME_ABI_MARKER = "scr_runtime_abi_v30" as const; /** Object consumption is useful today, but the complete scr_* surface may * still change before 1.0. The versioned marker prevents accidental mixing; diff --git a/packages/compiler/src/backend/runtime-pack-targets.test.ts b/packages/compiler/src/backend/runtime-pack-targets.test.ts index a97769328..58e2b4855 100644 --- a/packages/compiler/src/backend/runtime-pack-targets.test.ts +++ b/packages/compiler/src/backend/runtime-pack-targets.test.ts @@ -52,7 +52,7 @@ async function fixture(target: NativeTargetSpec, workers = false) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 29, marker: "scr_runtime_abi_v29" }, + runtime_abi: { version: 30, marker: "scr_runtime_abi_v30" }, compiler: { command: "fixture", identity: "fixture", target: target.llvmTriple }, macros: { executable: [], excluded: ["SCR_LIB"], sanitizer: "external-toolchain-required" }, flavors: { diff --git a/packages/compiler/src/backend/runtime-pack.test.ts b/packages/compiler/src/backend/runtime-pack.test.ts index 85038d8c5..b898a3e62 100644 --- a/packages/compiler/src/backend/runtime-pack.test.ts +++ b/packages/compiler/src/backend/runtime-pack.test.ts @@ -127,7 +127,7 @@ async function fixture(target: NativeTargetSpec = MACOS_ARM64_TARGET) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 29, marker: "scr_runtime_abi_v29" }, + runtime_abi: { version: 30, marker: "scr_runtime_abi_v30" }, compiler: { command: "clang", identity: "fixture clang", diff --git a/packages/compiler/src/frontend/lowering/lower-stream.ts b/packages/compiler/src/frontend/lowering/lower-stream.ts index 6aa8aef96..7c0c2b46c 100644 --- a/packages/compiler/src/frontend/lowering/lower-stream.ts +++ b/packages/compiler/src/frontend/lowering/lower-stream.ts @@ -3772,11 +3772,11 @@ function duplexFromBody(lowerer: Lowerer, node: ts.Expression, loc: SrcLoc): IrE if (fn !== null) { const isAsync = (fn.modifiers?.nodes ?? []).some((m) => m.kind === ts.SyntaxKind.AsyncKeyword); const isGen = !ts.isArrowFunction(fn) && fn.asteriskToken !== undefined; - if (!isAsync || !isGen) { + if (!isAsync) { lowerer.noLowering( - "Duplex.from over a function that is not an async generator function", + "Duplex.from over a function that is not async", node, - "write the body as async function* (source) { for await (const chunk of source) ... }", + "write the body as async function* (source) { ... } or async function (source) { ... }", ); } if (fn.parameters.nodes.length > 1) { @@ -3794,6 +3794,34 @@ function duplexFromBody(lowerer: Lowerer, node: ts.Expression, loc: SrcLoc): IrE nextT: UNDEFINED_T, }; const lowered = lowerStageFunction(lowerer, fn, sourceT); + if (!isGen) { + // an async function: a writable Duplexify whose final waits for + // the function's promise + const asyncSource = duplexSourceHelper(lowerer, sourceT, loc); + const out = lowered.out; + if (out.kind !== "promise" || !asyncSource) + return lowerer.noLowering("Duplex.from over a function with this return type", node); + if ( + out.inner.kind !== "void" && + !canConvertToDyn( + out.inner, + (id) => lowerer.shapes.get(id), + (id) => lowerer.unions.get(id), + ) + ) { + lowerer.noLowering( + `Duplex.from over a function resolving to '${lowerer.fmt(out.inner)}'`, + node, + ); + } + return { + kind: "libCall", + fn: "stream.duplexFromAsyncFn", + args: [lowered.closure, asyncSource, pipelineBoxHelper(lowerer, out, loc)], + type: duplex, + loc, + }; + } const helpers = lowered.out.kind === "generator" ? fromGenHelpers(lowerer, lowered.out, loc) : null; const source = duplexSourceHelper(lowerer, sourceT, loc); @@ -3818,6 +3846,45 @@ function duplexFromBody(lowerer: Lowerer, node: ts.Expression, loc: SrcLoc): IrE } const bodySides = bodyT?.kind === "object" ? streamSidesOf(lowerer, lowerer.classes.get(bodyT.className)) : null; + let literal: ts.Expression = node; + while (ts.isParenthesizedExpression(literal)) literal = literal.expression; + if (bodySides === null && ts.isObjectLiteralExpression(literal)) + return duplexFromPair(lowerer, literal, loc); + if (bodyT?.kind === "promise") { + // a Duplexify that reads the promise's value + const valueT = bodyT.inner; + if ( + valueT.kind !== "void" && + !canConvertToDyn( + valueT, + (id) => lowerer.shapes.get(id), + (id) => lowerer.unions.get(id), + ) + ) { + lowerer.noLowering(`Duplex.from over a promise of '${lowerer.fmt(valueT)}'`, node); + } + const promise = lowerer.lowerExpr(node); + if (promise.type.kind !== "promise") { + // a checked-dynamic promise (an untyped function's result) + return { + kind: "libCall", + fn: "stream.duplexFromDyn", + args: [lowerer.coerceToExpected(promise, DYN)], + type: { kind: "object", className: "%Duplex" }, + loc, + }; + } + return { + kind: "libCall", + fn: "stream.duplexFromPromise", + args: [ + promise, + pipelineBoxHelper(lowerer, promise.type as IrType & { kind: "promise" }, loc), + ], + type: { kind: "object", className: "%Duplex" }, + loc, + }; + } if (bodySides === null) return lowerFromSource(lowerer, node, node, undefined, true); let stream: IrExpr; try { @@ -3830,16 +3897,88 @@ function duplexFromBody(lowerer: Lowerer, node: ts.Expression, loc: SrcLoc): IrE stream.type.kind === "object" ? streamSidesOf(lowerer, lowerer.classes.get(stream.type.className)) : null; + // _duplexify({ readable }) / _duplexify({ writable }) + if (sides === "r" || sides === "w") { + return { + kind: "libCall", + fn: sides === "r" ? "stream.duplexifyReadable" : "stream.duplexifyWritable", + args: [stream], + type: duplex, + loc, + }; + } if (sides !== "rw") { lowerer.noLowering( - `Duplex.from over a ${sides === "r" ? "Readable" : sides === "w" ? "Writable" : "value of this type"}`, + "Duplex.from over a value of this type", node, - "an async generator function, an array, a string, a generator, an async generator, or a Duplex, Transform or PassThrough is the lowered body", + "an async generator function, an iterable, a stream, or a { readable, writable } pair is the lowered body", ); } return stream; } +/** Duplex.from({ readable, writable }): each member that is a stream with + * that side is used as it is, any other member is Duplex.from of it, and + * the pair reads and writes through _duplexify (duplexify.js). The + * members evaluate in source order. */ +function duplexFromPair( + lowerer: Lowerer, + literal: ts.ObjectLiteralExpression, + loc: SrcLoc, +): IrExpr { + const duplex: IrType = { kind: "object", className: "%Duplex" }; + const sides: { readable?: IrExpr; writable?: IrExpr } = {}; + for (const p of literal.properties.nodes) { + const key = + (ts.isPropertyAssignment(p) || ts.isShorthandPropertyAssignment(p)) && ts.isIdentifier(p.name) + ? p.name.text + : null; + if (key !== "readable" && key !== "writable") { + lowerer.noLowering( + "Duplex.from over an object with members other than readable and writable", + p, + "pass { readable, writable } with streams or Duplex.from bodies", + ); + } + const value = ts.isShorthandPropertyAssignment(p) + ? (p.name as ts.Identifier) + : (p as ts.PropertyAssignment).initializer; + let valueT: IrType | null; + try { + valueT = lowerer.mapTypeOf(lowerer.typeOf(value)); + } catch { + valueT = null; + } + const valueSides = + valueT?.kind === "object" + ? streamSidesOf(lowerer, lowerer.classes.get(valueT.className)) + : null; + const wanted = key === "readable" ? "r" : "w"; + const direct = valueSides === "rw" || valueSides === wanted; + sides[key!] = direct + ? lowerStreamArg(lowerer, value, "compose") + : duplexFromBody(lowerer, value, loc); + } + const { readable, writable } = sides; + if (readable && writable) + return { + kind: "libCall", + fn: "stream.duplexifyPair", + args: [readable, writable], + type: duplex, + loc, + }; + if (readable) + return { kind: "libCall", fn: "stream.duplexifyReadable", args: [readable], type: duplex, loc }; + if (writable) + return { kind: "libCall", fn: "stream.duplexifyWritable", args: [writable], type: duplex, loc }; + return lowerer.noLowering( + "Duplex.from over an object with neither readable nor writable", + literal, + "pass { readable, writable } with streams or Duplex.from bodies", + ); +} + /** The checker type of a stage's string or Buffer chunks, or null for * other chunk types. */ function stageChunkCheckerType(lowerer: Lowerer, chunk: IrType, at: ts.Node): ts.Type | null { diff --git a/packages/compiler/src/ir/builtin-effects.ts b/packages/compiler/src/ir/builtin-effects.ts index bab07c458..734f9ee80 100644 --- a/packages/compiler/src/ir/builtin-effects.ts +++ b/packages/compiler/src/ir/builtin-effects.ts @@ -384,6 +384,8 @@ export const MAY_THROW_LIB_FNS: ReadonlySet = new Set([ "writable.setDefaultEncoding", "stream.duplexFromGen", "stream.duplexFromDyn", + "stream.duplexFromPromise", + "stream.duplexFromAsyncFn", "stream.destroy", "stream.destroyErr", "stream.iteratorClose", diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index 1708ad01a..ad8b907da 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -4069,6 +4069,16 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ "stream.decodeStrings": { argTypes: [null, BOOL], result: VOID }, "stream.duplexFromGen": { argTypes: [null, null, null, null, null, null], result: VOID }, "stream.duplexFromDyn": { argTypes: [DYN], result: VOID }, + /** Duplex.from over a lone Readable or Writable, or a { readable, + * writable } pair of streams (duplexify.js's _duplexify). */ + "stream.duplexifyReadable": { argTypes: [null], result: VOID }, + "stream.duplexifyWritable": { argTypes: [null], result: VOID }, + "stream.duplexifyPair": { argTypes: [null, null], result: VOID }, + /** Duplex.from(promise): the promise and its box helper. */ + "stream.duplexFromPromise": { argTypes: [null, null], result: VOID }, + /** Duplex.from(asyncFunction): the stage, its source helper and the + * promise's box helper. */ + "stream.duplexFromAsyncFn": { argTypes: [null, null, null], result: VOID }, "writable.writeEncoding": { argTypes: [null, STRING], result: VOID }, "writable.setDefaultEncoding": { argTypes: [null, DYN], result: VOID }, /** Whether a readable.nextChunkDyn step answered the end (the done diff --git a/packages/compiler/src/ir/runtime-features.ts b/packages/compiler/src/ir/runtime-features.ts index da9cab1c3..84b767fab 100644 --- a/packages/compiler/src/ir/runtime-features.ts +++ b/packages/compiler/src/ir/runtime-features.ts @@ -221,6 +221,11 @@ const STREAM_LIB_FNS: ReadonlySet = new Set([ "stream.decodeStrings", "stream.duplexFromGen", "stream.duplexFromDyn", + "stream.duplexFromPromise", + "stream.duplexFromAsyncFn", + "stream.duplexifyReadable", + "stream.duplexifyWritable", + "stream.duplexifyPair", "console.bind", "crypto.hashStream", "crypto.hashStreamPiped", @@ -277,6 +282,8 @@ const READABLE_DYN_ASYNC_LIB_FNS: ReadonlySet = new Set([ "readable.fromIter", "stream.duplexFromGen", "stream.duplexFromDyn", + "stream.duplexFromPromise", + "stream.duplexFromAsyncFn", ]); /** libCall name prefixes of the Readable entry points that await through the dyn-async unit. */ diff --git a/packages/compiler/src/ir/validate.ts b/packages/compiler/src/ir/validate.ts index a76784083..d08d0084f 100644 --- a/packages/compiler/src/ir/validate.ts +++ b/packages/compiler/src/ir/validate.ts @@ -5499,6 +5499,11 @@ function validateFunction( e.fn === "stream.decodeStrings" || e.fn === "stream.duplexFromGen" || e.fn === "stream.duplexFromDyn" || + e.fn === "stream.duplexifyReadable" || + e.fn === "stream.duplexifyWritable" || + e.fn === "stream.duplexifyPair" || + e.fn === "stream.duplexFromPromise" || + e.fn === "stream.duplexFromAsyncFn" || e.fn === "stream.setConstruct" || e.fn === "stream.setWritev" || e.fn === "stream.setWritevDefault" || @@ -5554,7 +5559,30 @@ function validateFunction( } return true; } - if (e.fn === "stream.duplexFromGen" || e.fn === "stream.duplexFromDyn") { + if (e.fn === "stream.duplexFromPromise") { + // Duplex.from(promise): (promise, box) → a Duplex. + if (e.args[0]?.type.kind !== "promise" || e.args[1]?.type.kind !== "func") + err(`libCall ${e.fn} takes a promise and its box helper`, e.loc); + if (!isStreamObject(e.type)) err(`libCall ${e.fn} must return a stream class`, e.loc); + return true; + } + if ( + e.fn === "stream.duplexifyReadable" || + e.fn === "stream.duplexifyWritable" || + e.fn === "stream.duplexifyPair" + ) { + // Duplex.from over streams: (r), (w) or (r, w) → a Duplex. + const want = e.fn === "stream.duplexifyPair" ? 2 : 1; + if (e.args.length !== want || !e.args.every((a) => isStreamObject(a.type))) + err(`libCall ${e.fn} takes ${want} stream argument(s)`, e.loc); + if (!isStreamObject(e.type)) err(`libCall ${e.fn} must return a stream class`, e.loc); + return true; + } + if ( + e.fn === "stream.duplexFromGen" || + e.fn === "stream.duplexFromDyn" || + e.fn === "stream.duplexFromAsyncFn" + ) { // Duplex.from: (stage, source, helpers...) or (body) → a Duplex. if (!isStreamObject(e.type)) { err(`libCall ${e.fn} must return a stream class, got ${e.type.kind}`, e.loc); diff --git a/packages/compiler/test/native-codegen-integration.test.ts b/packages/compiler/test/native-codegen-integration.test.ts index bb6ccf874..169e89c2e 100644 --- a/packages/compiler/test/native-codegen-integration.test.ts +++ b/packages/compiler/test/native-codegen-integration.test.ts @@ -378,7 +378,7 @@ done: expect(await symbols(helperObject, ["-u"])).toEqual(await symbols(clangObject, ["-u"])); expect(await symbols(helperObject, ["-gU"])).toEqual(await symbols(clangObject, ["-gU"])); expect(await symbols(helperObject, ["-gU"])).toEqual(["0000000000000000 T _main"]); - expect(await symbols(helperObject, ["-u"])).toContain("_scr_runtime_abi_v29"); + expect(await symbols(helperObject, ["-u"])).toContain("_scr_runtime_abi_v30"); const clangExe = join(dir, "clang-program"); const linkDriver = join(dir, "clang-driver.c"); @@ -712,7 +712,7 @@ done: .split("\n"); expect(undefinedSymbols).toContain("_sf_scale"); expect(undefinedSymbols).toContain("_sf_callback_mix"); - expect(undefinedSymbols).toContain("_scr_runtime_abi_v29"); + expect(undefinedSymbols).toContain("_scr_runtime_abi_v30"); }); test("helper and clang object paths retain the same outbound FFI C ABI", async () => { diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7424-duplex-from-streams.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7424-duplex-from-streams.mjs.json new file mode 100644 index 000000000..a3801c775 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7424-duplex-from-streams.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7424-duplex-from-streams.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7425-duplex-from-streams.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7425-duplex-from-streams.ts.json new file mode 100644 index 000000000..069ae4ef4 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7425-duplex-from-streams.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7425-duplex-from-streams.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7426-duplex-from-promise.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7426-duplex-from-promise.mjs.json new file mode 100644 index 000000000..e44b3f001 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7426-duplex-from-promise.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7426-duplex-from-promise.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7427-duplex-from-async-function.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7427-duplex-from-async-function.mjs.json new file mode 100644 index 000000000..22d092c60 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7427-duplex-from-async-function.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7427-duplex-from-async-function.mjs" + ], + "diags": [] +} diff --git a/packages/runtime-darwin-arm64/scripts/build.mjs b/packages/runtime-darwin-arm64/scripts/build.mjs index 71126f806..aa251dc34 100644 --- a/packages/runtime-darwin-arm64/scripts/build.mjs +++ b/packages/runtime-darwin-arm64/scripts/build.mjs @@ -340,7 +340,7 @@ async function build() { package: packageManifest.name, version: packageManifest.version, target: RUNTIME_PACK_MATRIX.target, - runtime_abi: { version: 29, marker: "scr_runtime_abi_v29" }, + runtime_abi: { version: 30, marker: "scr_runtime_abi_v30" }, compiler: { command: compiler, identity: compilerVersion, diff --git a/packages/runtime-pack-common/scripts/build.mjs b/packages/runtime-pack-common/scripts/build.mjs index b0ec0005b..acf463824 100644 --- a/packages/runtime-pack-common/scripts/build.mjs +++ b/packages/runtime-pack-common/scripts/build.mjs @@ -437,7 +437,7 @@ async function build() { package: packageManifest.name, version: packageManifest.version, target: matrix.target, - runtime_abi: { version: 29, marker: "scr_runtime_abi_v29" }, + runtime_abi: { version: 30, marker: "scr_runtime_abi_v30" }, compiler: { command: compiler, identity: compilerVersion, diff --git a/packages/runtime/src/scr_console.c b/packages/runtime/src/scr_console.c index 317ee124b..5b7f8da0f 100644 --- a/packages/runtime/src/scr_console.c +++ b/packages/runtime/src/scr_console.c @@ -148,7 +148,7 @@ static void scr_enable_utf8_console_output(void) { } #endif -void scr_runtime_abi_v29(void) {} +void scr_runtime_abi_v30(void) {} void scr_init(void) { #ifdef SCR_WORKERS diff --git a/packages/runtime/src/scr_core.h b/packages/runtime/src/scr_core.h index 6f3aae93a..57928cfb5 100644 --- a/packages/runtime/src/scr_core.h +++ b/packages/runtime/src/scr_core.h @@ -81,7 +81,7 @@ void scr_init(void); /* Program objects emitted by the bundled LLVM helper reference this symbol. * Its versioned spelling makes a mismatched manual runtime link fail before * the program can start. */ -void scr_runtime_abi_v29(void); +void scr_runtime_abi_v30(void); void scr_runtime_workers_v8(void); /* ── the trap funnel (scr_console.c; scr_library.c under -DSCR_LIB) ────── diff --git a/packages/runtime/src/scr_stream.c b/packages/runtime/src/scr_stream.c index 4a53da4c0..992203eb4 100644 --- a/packages/runtime/src/scr_stream.c +++ b/packages/runtime/src/scr_stream.c @@ -1769,18 +1769,26 @@ ScrStream *scr_stream_from_dyn(ScrDyn *src, ScrDyn *opts) { } /* Duplex.from(body) over a checked-dynamic body (duplexify.js): a native - * Duplex answers itself, an iterable reads as Duplex.from does, and the - * other bodies Node accepts (a lone Readable or Writable, a function, a - * promise, a { readable, writable } pair) are not supported yet. */ + * Duplex answers itself, a lone native Readable or Writable is wrapped, a + * promise reads as its value, an iterable reads as Duplex.from does, and + * the other bodies Node accepts (a function, a thenable, a { readable, + * writable } pair) are not supported yet. */ static ScrStream *scr_stream_boxed(const ScrDyn *value); +static ScrStream *scr_dupx_from_boxed(ScrDyn *boxed); ScrStream *scr_stream_duplex_from_dyn(ScrDyn *body) { ScrStream *native = scr_stream_of_handle(body); /* +1 */ if (!native) native = scr_stream_boxed(body); /* +1 */ - if (native && native->st->has_r && native->st->has_w) return native; - bool unsupported = native != NULL; - if (native) scr_stream_release(native); - if (!unsupported && (body->kind == SCR_DYN_FUNC || body->kind == SCR_DYN_PROMISE)) unsupported = true; + if (native) { + if (native->st->has_r && native->st->has_w) return native; + /* _duplexify({ readable }) / _duplexify({ writable }) */ + ScrStream *d = native->st->has_r ? scr_stream_duplexify(native, NULL) + : scr_stream_duplexify(NULL, native); + scr_stream_release(native); + return d; + } + if (body->kind == SCR_DYN_PROMISE) return scr_dupx_from_boxed(scr_dyn_retain(body)); + bool unsupported = body->kind == SCR_DYN_FUNC; if (!unsupported && body->kind == SCR_DYN_OBJ) { static const char *const keys[] = {"then", "readable", "writable"}; for (size_t i = 0; i < 3 && !unsupported; i++) { @@ -4709,6 +4717,344 @@ ScrStream *scr_stream_compose_stages(double n, ScrStream **streams, double count return scr_stream_compose_impl((size_t)n, streams, &fs, abort); } +/* ── Duplex.from over a stream or a { readable, writable } pair ───────── + * duplexify.js's _duplexify({ readable, writable }): a Duplexify that + * writes to w (write() answering false waits for 'drain'; final ends w and + * waits for its 'finish') and reads r through 'readable' and read(), with + * each side's end of stream (eos) finishing the Duplex. The readable and + * writable object modes follow r's and w's. */ + +enum { SCR_DUPX_UNSET, SCR_DUPX_NULL, SCR_DUPX_WAIT }; + +typedef struct ScrDuplexify { + size_t rc; + ScrStream *r, *w; /* +1 or NULL */ + ScrStream *d; /* +1 until teardown */ + ScrClosure *drain_l, *finish_l, *readable_l, *end_l; /* +1, the listeners, or NULL */ + bool readable, writable; + bool ondrain, onfinish, onreadable; + int onclose; /* _duplexify's onclose: undefined, null, or d's destroy callback */ + struct ScrDuplexify *live_prev, *live_next; +} ScrDuplexify; + +static SCR_TL ScrDuplexify *scr_dupx_live = NULL; + +static void *scr_dupx_retain_v(void *p) { + ((ScrDuplexify *)p)->rc++; + return p; +} + +static void scr_dupx_release_v(void *p) { + ScrDuplexify *x = p; + if (--x->rc > 0) return; + if (x->r) scr_stream_release(x->r); + if (x->w) scr_stream_release(x->w); + free(x); +} + +static ScrClosure *scr_dupx_closure(void *fn, ScrDuplexify *x) { + ScrClosure *cl = scr_closure_new(fn, 1); + cl->caps[0] = scr_box_new_obj(&scr_dupx_retain_v, &scr_dupx_release_v, NULL); + scr_box_set_ref(cl->caps[0], scr_dupx_retain_v(x)); + return cl; +} + +static void scr_dupx_on(ScrStream *s, const char *name, ScrClosure **slot, void *fn, ScrDuplexify *x) { + ScrStr *n = scr_str_new(name, strlen(name)); + *slot = scr_dupx_closure(fn, x); + scr_emitter_release( + scr_emitter_on((ScrEmitter *)s, n, scr_closure_retain(*slot), scr_ee_inv_fixed0, false, false)); + scr_str_release(n); +} + +/* Drop the Duplex and the listeners on r and w. */ +static void scr_dupx_teardown(ScrDuplexify *x) { + if (!x->d) return; + if (x->w) { + scr_compose_off(x->w, "drain", &x->drain_l); + scr_compose_off(x->w, "finish", &x->finish_l); + } + if (x->r) { + scr_compose_off(x->r, "readable", &x->readable_l); + scr_compose_off(x->r, "end", &x->end_l); + } + if (x->live_prev) x->live_prev->live_next = x->live_next; + else scr_dupx_live = x->live_next; + if (x->live_next) x->live_next->live_prev = x->live_prev; + x->live_prev = x->live_next = NULL; + ScrStream *d = x->d; + x->d = NULL; + scr_stream_release(d); + scr_dupx_release_v(x); /* the live list's reference */ +} + +static void scr_dupx_atexit(void) { + while (scr_dupx_live) scr_dupx_teardown(scr_dupx_live); +} + +/* d's destroy callback: callback(err), then the context lets go. */ +static void scr_dupx_destroyed(ScrDuplexify *x, ScrError *err) { + ScrStream *d = scr_stream_retain(x->d); + scr_dupx_teardown(x); + scr_stream_destroy_done(d, err ? scr_error_retain(err) : NULL); + scr_stream_release(d); +} + +/* onfinished(err): the pending destroy callback, else d.destroy(err) */ +static void scr_dupx_onfinished(ScrDuplexify *x, ScrError *err) { + int onclose = x->onclose; + x->onclose = SCR_DUPX_NULL; + if (!x->d) return; + if (onclose == SCR_DUPX_WAIT) { + scr_dupx_destroyed(x, err); + } else if (err) { + ScrStream *d = scr_stream_retain(x->d); + scr_stream_release(scr_stream_destroy(d, err)); + scr_stream_release(d); + } +} + +/* eos(w): writable = false; an error destroys r */ +static void scr_dupx_w_eos(ScrClosure *cb, ScrStream *s, ScrError *err) { + (void)s; + ScrDuplexify *x = scr_box_get_ref(cb->caps[0]); + x->writable = false; + if (err && x->r) scr_stream_release(scr_stream_destroy(x->r, err)); + scr_dupx_onfinished(x, err); + scr_dupx_release_v(x); +} + +/* eos(r): readable = false; an error destroys w */ +static void scr_dupx_r_eos(ScrClosure *cb, ScrStream *s, ScrError *err) { + (void)s; + ScrDuplexify *x = scr_box_get_ref(cb->caps[0]); + x->readable = false; + if (err && x->w) scr_stream_release(scr_stream_destroy(x->w, err)); + scr_dupx_onfinished(x, err); + scr_dupx_release_v(x); +} + +/* d._write: w.write(chunk) ? callback() : (ondrain = callback) */ +static void scr_dupx_write_inv(ScrClosure *cb, ScrStream *s, ScrBytes *chunk) { + ScrDuplexify *x = scr_box_get_ref(cb->caps[0]); + bool ok = scr_stream_writes_values(s) ? scr_stream_write_dyn(x->w, (ScrDyn *)chunk, NULL) + : scr_stream_write(x->w, chunk, NULL); + if (ok) { + if (!scr_exc_pending()) scr_stream_write_done(s, NULL); + } else { + x->ondrain = true; + } + scr_dupx_release_v(x); +} + +/* d._final: w.end(); onfinish = callback */ +static void scr_dupx_final_inv(ScrClosure *cb, ScrStream *s) { + (void)s; + ScrDuplexify *x = scr_box_get_ref(cb->caps[0]); + ScrStream *r = scr_stream_end(x->w, NULL, NULL, NULL); + if (r) scr_stream_release(r); + x->onfinish = true; + scr_dupx_release_v(x); +} + +static void scr_dupx_drain_l(ScrClosure *cb) { + ScrDuplexify *x = scr_box_get_ref(cb->caps[0]); + if (x->ondrain && x->d) { + x->ondrain = false; + scr_stream_write_done(x->d, NULL); + } + scr_dupx_release_v(x); +} + +static void scr_dupx_finish_l(ScrClosure *cb) { + ScrDuplexify *x = scr_box_get_ref(cb->caps[0]); + if (x->onfinish && x->d) { + x->onfinish = false; + scr_stream_final_done(x->d, NULL); + } + scr_dupx_release_v(x); +} + +/* d._read: push r.read() until it answers null (then wait for 'readable') + * or d is full */ +static void scr_dupx_pull(ScrDuplexify *x) { + ScrStream *d = x->d; + while (d && x->r) { + ScrEntryKind kind = SCR_ENTRY_BYTES; + void *chunk = scr_stream_read_n_kind(x->r, -1, &kind); + if (scr_exc_pending()) return; + if (!chunk) { + x->onreadable = true; + return; + } + bool more; + if (d->st->r.object_entries) { + more = scr_stream_add_entry(d, chunk, kind) > 0; + } else if (kind == SCR_ENTRY_STR) { + more = scr_stream_push_str(d, chunk); + scr_str_release(chunk); + } else if (kind == SCR_ENTRY_DYN) { + more = scr_stream_push_dyn(d, chunk); + scr_dyn_release(chunk); + } else { + more = scr_stream_push(d, chunk); + scr_bytes_release(chunk); + } + if (!more || scr_exc_pending()) return; + } +} + +static void scr_dupx_read_inv(ScrClosure *cb, ScrStream *s, double size) { + (void)s; + (void)size; + ScrDuplexify *x = scr_box_get_ref(cb->caps[0]); + x->onreadable = false; + scr_dupx_pull(x); + scr_dupx_release_v(x); +} + +static void scr_dupx_readable_l(ScrClosure *cb) { + ScrDuplexify *x = scr_box_get_ref(cb->caps[0]); + if (x->onreadable && x->d) { + x->onreadable = false; + scr_dupx_pull(x); + } + scr_dupx_release_v(x); +} + +static void scr_dupx_end_l(ScrClosure *cb) { + ScrDuplexify *x = scr_box_get_ref(cb->caps[0]); + if (x->d) scr_stream_push_null(x->d); + scr_dupx_release_v(x); +} + +/* d._destroy */ +static void scr_dupx_destroy_inv(ScrClosure *cb, ScrStream *s, ScrError *err) { + (void)s; + ScrDuplexify *x = scr_box_get_ref(cb->caps[0]); + ScrError *e = err ? scr_error_retain(err) : NULL; + if (!e && x->onclose != SCR_DUPX_NULL) e = scr_compose_abort_error(); + x->onreadable = x->ondrain = x->onfinish = false; + if (x->onclose == SCR_DUPX_NULL) { + if (x->d) scr_dupx_destroyed(x, e); + } else { + x->onclose = SCR_DUPX_WAIT; + if (x->w && !x->w->st->destroyed) scr_stream_release(scr_stream_destroy(x->w, e)); + if (x->r && !x->r->st->destroyed) scr_stream_release(scr_stream_destroy(x->r, e)); + } + if (e) scr_error_release(e); + scr_dupx_release_v(x); +} + +/* _duplexify({ readable: r, writable: w }): r and w borrowed, either NULL. + * +1. */ +ScrStream *scr_stream_duplexify(ScrStream *r, ScrStream *w) { + ScrDuplexify *x = calloc(1, sizeof *x); + if (!x) scr_stream_oom(); + x->rc = 1; + x->r = r ? scr_stream_retain(r) : NULL; + x->w = w ? scr_stream_retain(w) : NULL; + x->onclose = SCR_DUPX_UNSET; + x->readable = r && scr_stream_prop(r, "readable") != 0; + x->writable = w && scr_stream_prop(w, "writable") != 0; + /* new Duplexify({ readableObjectMode, writableObjectMode, readable, writable }) */ + bool r_object = r && r->st->has_r && r->st->r.object_entries; + bool w_object = w && w->st->has_w && w->st->w.object_mode; + ScrStream *d = scr_stream_new_duplex( + r_object ? 16 : -1, w_object ? 16 : -1, true, true, true, x->readable, x->writable, + x->readable ? scr_dupx_closure((void *)&scr_dupx_read_inv, x) : NULL, + x->readable ? &scr_dupx_read_inv : NULL, + x->writable ? scr_dupx_closure((void *)&scr_dupx_write_inv, x) : NULL, + x->writable ? &scr_dupx_write_inv : NULL, + x->writable ? scr_dupx_closure((void *)&scr_dupx_final_inv, x) : NULL, + x->writable ? &scr_dupx_final_inv : NULL, + scr_dupx_closure((void *)&scr_dupx_destroy_inv, x), &scr_dupx_destroy_inv); + if (r_object) d->st->r.object_entries = true; + if (w_object) scr_stream_set_object_mode(d, false, true); + d->cls = "Duplexify"; + x->d = scr_stream_retain(d); + if (x->writable) { + scr_closure_release( + scr_stream_finished(w, scr_dupx_closure((void *)&scr_dupx_w_eos, x), &scr_dupx_w_eos)); + scr_dupx_on(w, "drain", &x->drain_l, (void *)&scr_dupx_drain_l, x); + scr_dupx_on(w, "finish", &x->finish_l, (void *)&scr_dupx_finish_l, x); + } + if (x->readable) { + scr_closure_release( + scr_stream_finished(r, scr_dupx_closure((void *)&scr_dupx_r_eos, x), &scr_dupx_r_eos)); + scr_dupx_on(r, "readable", &x->readable_l, (void *)&scr_dupx_readable_l, x); + scr_dupx_on(r, "end", &x->end_l, (void *)&scr_dupx_end_l, x); + } + static SCR_TL bool registered = false; + if (!registered) { + registered = true; + scr_atexit(scr_dupx_atexit); + } + x->live_next = scr_dupx_live; + if (scr_dupx_live) scr_dupx_live->live_prev = x; + scr_dupx_live = x; /* the reference x was created with */ + return d; +} + +/* Duplex.from(promise) (duplexify.js): a Duplexify({ objectMode: true, + * writable: false, read() {} }) that pushes the promise's value unless it + * is nullish, then ends; a rejection destroys it with the reason. */ +typedef struct { + ScrStream *d; /* +1 */ + ScrDyn *boxed; /* +1, the promise boxed */ +} ScrDupxPromise; + +static void scr_dupx_noop_read(ScrClosure *cb, ScrStream *s, double size) { + (void)cb; + (void)s; + (void)size; +} + +static void scr_dupx_promise_entry(ScrFiber *fiber, void *arg) { + (void)fiber; + ScrDupxPromise p = *(ScrDupxPromise *)arg; + free(arg); + /* the boxed promise's payload is a dyn value (NULL: undefined) */ + ScrDyn *v = scr_await_ref(p.boxed->v.promise); + if (!scr_exc_pending()) { + if (v && v->kind != SCR_DYN_NULL && v->kind != SCR_DYN_UNDEF) scr_stream_push_object(p.d, v); + if (!scr_exc_pending()) scr_stream_push_null(p.d); + if (scr_exc_pending()) scr_exc_clear(); + if (v) scr_dyn_release(v); + } else { + ScrError *err = scr_pump_caught(); + scr_stream_release(scr_stream_destroy(p.d, err)); + if (scr_exc_pending()) scr_exc_clear(); + scr_error_release(err); + } + scr_stream_release(p.d); + scr_dyn_release(p.boxed); +} + +/* The readable Duplexify over a boxed promise (moves boxed). +1. */ +static ScrStream *scr_dupx_from_boxed(ScrDyn *boxed) { + ScrClosure *read = scr_closure_new((void *)&scr_dupx_noop_read, 0); + ScrStream *d = scr_stream_new_duplex(16, 16, true, true, true, true, false, read, + &scr_dupx_noop_read, NULL, NULL, NULL, NULL, NULL, NULL); + scr_stream_set_object_mode(d, true, true); + d->cls = "Duplexify"; + ScrDupxPromise *ctx = malloc(sizeof *ctx); + if (!ctx) scr_stream_oom(); + ctx->d = scr_stream_retain(d); + ctx->boxed = boxed; + scr_promise_release(scr_async_spawn_after(ctx->boxed->v.promise, &scr_dupx_promise_entry, ctx)); + return d; +} + +/* p and box borrowed (box: the compiler's promise → dyn promise helper). +1. */ +ScrStream *scr_stream_duplex_from_promise(ScrPromise *p, ScrClosure *box) { + return scr_dupx_from_boxed(((ScrPipeBoxFn)box->fn)(box, scr_promise_retain(p))); +} + +ScrStream *scr_stream_duplexify_readable(ScrStream *r) { return scr_stream_duplexify(r, NULL); } +ScrStream *scr_stream_duplexify_writable(ScrStream *w) { return scr_stream_duplexify(NULL, w); } +ScrStream *scr_stream_duplexify_pair(ScrStream *r, ScrStream *w) { return scr_stream_duplexify(r, w); } + /* ── node:stream/promises ───────────────────────────────────────────── * The promise forms ride the callback machinery above with a settling * watcher: caps[0] boxes the pending promise; the terminal status diff --git a/packages/runtime/src/scr_stream.h b/packages/runtime/src/scr_stream.h index 623f60465..d276700c8 100644 --- a/packages/runtime/src/scr_stream.h +++ b/packages/runtime/src/scr_stream.h @@ -545,6 +545,20 @@ void scr_stream_set_writev_default(ScrStream *s, ScrClosure *cb, ScrStreamWritev ScrStream *scr_stream_duplex_from_gen(ScrClosure *stage, ScrClosure *source, ScrClosure *next, ScrClosure *read, ScrClosure *ret, ScrClosure *thr); ScrStream *scr_stream_duplex_from_dyn(struct ScrDyn *body); +/* Duplex.from over an async function (the compiled stage takes the source + * generator and answers its promise; source and the promise's box helper + * borrowed): a writable Duplexify whose final waits for the promise. +1. */ +ScrStream *scr_stream_duplex_from_async_fn(ScrClosure *stage, ScrClosure *source, ScrClosure *box); +/* Duplex.from over a Readable, a Writable, or a { readable, writable } + * pair (duplexify.js's _duplexify): a Duplexify that reads r and writes w + * (either NULL in scr_stream_duplexify; streams borrowed). +1. */ +ScrStream *scr_stream_duplexify(ScrStream *r, ScrStream *w); +ScrStream *scr_stream_duplexify_readable(ScrStream *r); +ScrStream *scr_stream_duplexify_writable(ScrStream *w); +ScrStream *scr_stream_duplexify_pair(ScrStream *r, ScrStream *w); +/* Duplex.from(promise): a readable Duplexify of the promise's value. p and + * box (the compiler's promise → dyn promise helper) borrowed. +1. */ +ScrStream *scr_stream_duplex_from_promise(struct ScrPromise *p, ScrClosure *box); /* decodeStrings (false keeps string writes as strings), the writable * defaultEncoding option (a known encoding, borrowed), and * setDefaultEncoding (validated; answers the stream +1). The chaining diff --git a/packages/runtime/src/scr_stream_ops.c b/packages/runtime/src/scr_stream_ops.c index 1db90bc5a..75ce416ac 100644 --- a/packages/runtime/src/scr_stream_ops.c +++ b/packages/runtime/src/scr_stream_ops.c @@ -2042,7 +2042,7 @@ ScrStream *scr_stream_from_async_iter(ScrDyn *iter, double flags, double hwm) { * (the chunk, or the done marker), and ack() is fromAsyncGen's * `process.nextTick(cb)` followed by a fresh deferred. */ -enum { CHAN_NONE, CHAN_WRITE, CHAN_FINAL, CHAN_ABORT }; +enum { CHAN_NONE, CHAN_WRITE, CHAN_FINAL, CHAN_ABORT, CHAN_FINAL_ASYNC }; typedef struct OpChan OpChan; struct OpChan { @@ -2052,6 +2052,12 @@ struct OpChan { ScrPromise *resolve; /* +1: the deferred a write or the final settles, NULL once settled */ int settled; /* CHAN_*: what settled the deferred, for ack() */ bool aborted; /* destroy ran (fromAsyncGen's ac.abort()) */ + /* An async function body: its promise's then() (duplexify.js) settles + * `promise` there; the final waits for it. */ + bool finalized; /* final() has started */ + bool body_settled; /* the then() callbacks ran */ + bool final_waiting; /* the final's callback awaits the settlement */ + ScrError *body_err; /* +1: what awaiting the then() result throws, or NULL */ OpChan *live_prev, *live_next; }; @@ -2059,6 +2065,7 @@ static SCR_TL OpChan *op_chan_live = NULL; static void op_chan_drop(OpObj *o) { OpChan *c = (OpChan *)o; + if (c->body_err) scr_error_release(c->body_err); if (c->d) scr_stream_release(c->d); if (c->promise) scr_promise_release(c->promise); if (c->resolve) scr_promise_release(c->resolve); @@ -2136,10 +2143,49 @@ static ScrError *op_chan_abort_error(void) { /* After the source's await: process.nextTick(cb), then (unless done) the * aborted check and a fresh deferred. */ +static void op_chan_final_async_wait(OpChan *c); + +/* The final's completion after `await promise`: process.nextTick(cb, err) */ +static void op_chan_final_async_done(ScrClosure *job) { + OpChan *c = op_chan_of(job); + if (c->d) { + ScrError *err = c->body_err ? scr_error_retain(c->body_err) : NULL; + ScrStream *d = scr_stream_retain(c->d); + scr_stream_final_done(d, err); + scr_stream_release(d); + } + op_unref(c); +} + +/* `await promise` resumed: the completion follows on the next tick */ +static void op_chan_final_async_resume(ScrClosure *job) { + OpChan *c = op_chan_of(job); + scr_next_tick(op_chan_closure((void *)&op_chan_final_async_done, c)); + op_unref(c); +} + +/* The final's async callback (run by the source's nextTick): it awaits + * the then() result, which may still be pending. */ +static void op_chan_final_async_tick(ScrClosure *job) { + OpChan *c = op_chan_of(job); + op_chan_final_async_wait(c); + op_unref(c); +} + +static void op_chan_final_async_wait(OpChan *c) { + if (c->body_settled) scr_queue_microtask(op_chan_closure((void *)&op_chan_final_async_resume, c)); + else c->final_waiting = true; +} + static void op_chan_ack_inv(ScrClosure *cb) { OpChan *c = op_chan_of(cb); int settled = c->settled; c->settled = CHAN_NONE; + if (settled == CHAN_FINAL_ASYNC) { + scr_next_tick(op_chan_closure((void *)&op_chan_final_async_tick, c)); + op_unref(c); + return; + } if (c->d && (settled == CHAN_WRITE || settled == CHAN_FINAL)) { ScrClosure *tick = scr_closure_new((void *)&op_chan_done_tick, 2); tick->caps[0] = scr_box_new_obj(&scr_stream_retain_v, &scr_stream_release_v, &scr_stream_trace); @@ -2261,6 +2307,126 @@ ScrStream *scr_stream_duplex_from_gen(ScrClosure *stage, ScrClosure *source, Scr return d; } +/* ── Duplex.from(asyncFunction) ── duplexify.js: the function's promise + * must settle to a nullish value (else ERR_INVALID_RETURN_VALUE when the + * final awaits it), a return before the input was consumed destroys the + * Duplex, a rejection destroys it with the reason, and the Duplexify is + * writable only, its final waiting for the promise. */ + +typedef ScrPromise *(*OpAsyncStageFn)(ScrClosure *, ScrGen *); +typedef ScrDyn *(*OpBoxFn)(ScrClosure *, ScrPromise *); + +typedef struct { + OpChan *c; /* +1 */ + ScrDyn *boxed; /* +1, the function's promise boxed */ +} OpAsyncBody; + +static void op_async_body_entry(ScrFiber *fiber, void *arg) { + (void)fiber; + OpAsyncBody b = *(OpAsyncBody *)arg; + free(arg); + OpChan *c = b.c; + ScrDyn *v = scr_await_ref(b.boxed->v.promise); /* NULL: undefined */ + ScrStream *d = c->d ? scr_stream_retain(c->d) : NULL; + if (!scr_exc_pending()) { + if (v && v->kind != SCR_DYN_NULL && v->kind != SCR_DYN_UNDEF) { + /* the then() callback throws: awaiting its result rejects */ + char detail[160]; + const char *got = scr_dyn_specific_type(v, detail, sizeof detail); + char msg[320]; + int n = snprintf(msg, sizeof msg, + "Expected nully to be returned from the \"body\" function but got %s.", got); + c->body_err = op_coded_error(SCR_ERR_TYPE, "ERR_INVALID_RETURN_VALUE", msg); + (void)n; + } else if (!c->finalized && d) { + /* returned without consuming the input: destroyer(d) */ + ScrError *e = op_chan_abort_error(); + scr_stream_release(scr_stream_destroy(d, e)); + scr_error_release(e); + } + if (v) scr_dyn_release(v); + } else { + ScrCaught *caught = scr_exc_take(); + ScrError *err = op_caught_error(caught); + scr_caught_release(caught); + if (d) scr_stream_release(scr_stream_destroy(d, err)); + scr_error_release(err); + } + if (scr_exc_pending()) scr_exc_clear(); + if (d) scr_stream_release(d); + c->body_settled = true; + if (c->final_waiting) { + c->final_waiting = false; + op_chan_final_async_wait(c); + } + op_unref(c); + scr_dyn_release(b.boxed); +} + +/* Duplexify's final(cb) for an async function body: finalized, then + * fromAsyncGen's final with the awaiting callback */ +static void op_chan_final_async_inv(ScrClosure *cb, ScrStream *s) { + (void)s; + OpChan *c = op_chan_of(cb); + c->finalized = true; + op_chan_settle(c, scr_stream_from_done(), CHAN_FINAL_ASYNC); + op_unref(c); +} + +/* fromAsyncGen's destroy alone (no from() iterator to close) */ +static void op_chan_destroy_inv(ScrClosure *cb, ScrStream *s, ScrError *err) { + OpChan *c = op_chan_of(cb); + c->aborted = true; + if (c->resolve) op_chan_settle(c, scr_stream_from_done(), CHAN_ABORT); + op_chan_retire(c); + op_unref(c); + scr_stream_destroy_done(s, err ? scr_error_retain(err) : NULL); +} + +/* Duplex.from(fn) for an async function: the compiled stage (borrowed) + * takes the source generator and answers its promise; source and box + * (promise → dyn promise) borrowed. +1, or NULL with the function's throw + * pending. */ +ScrStream *scr_stream_duplex_from_async_fn(ScrClosure *stage, ScrClosure *source, ScrClosure *box) { + op_sentinels_init(); + static SCR_TL bool registered = false; + if (!registered) { + registered = true; + scr_atexit(op_chan_atexit); + } + OpChan *c = op_alloc(sizeof *c); + c->base.rc = 1; + c->base.drop = &op_chan_drop; + op_chan_defer(c); + ScrGen *src = ((OpSourceFn)source->fn)(source, op_chan_closure((void *)&op_chan_next_inv, c), + op_chan_closure((void *)&op_chan_ack_inv, c)); + ScrPromise *p = scr_exc_pending() ? NULL : ((OpAsyncStageFn)stage->fn)(stage, src); + if (scr_exc_pending()) { + if (p) scr_promise_release(p); + op_unref(c); + return NULL; + } + /* Duplexify({ objectMode: true, readable: false, write, final, destroy }) */ + ScrStream *d = scr_stream_new_duplex(16, 16, true, true, true, false, true, NULL, NULL, + op_chan_closure((void *)&op_chan_write_inv, c), &op_chan_write_inv, + op_chan_closure((void *)&op_chan_final_async_inv, c), + &op_chan_final_async_inv, + op_chan_closure((void *)&op_chan_destroy_inv, c), &op_chan_destroy_inv); + scr_stream_set_object_mode(d, true, true); + d->cls = "Duplexify"; + c->d = scr_stream_retain(d); + c->live_next = op_chan_live; + if (op_chan_live) op_chan_live->live_prev = c; + op_chan_live = c; /* the reference c was created with */ + /* value.then(onFulfilled, onRejected) */ + OpAsyncBody *b = malloc(sizeof *b); + if (!b) op_oom(); + b->c = op_ref(c); + b->boxed = ((OpBoxFn)box->fn)(box, p); /* p moves */ + scr_promise_release(scr_async_spawn_after(b->boxed->v.promise, &op_async_body_entry, b)); + return d; +} + /* ── the promise operators ─────────────────────────────────────────── */ enum { PK_DYN, PK_F64, PK_BOOL, PK_STR, PK_VOID }; diff --git a/tests/corpus/7424-duplex-from-streams.mjs b/tests/corpus/7424-duplex-from-streams.mjs new file mode 100644 index 000000000..5a9bdc01a --- /dev/null +++ b/tests/corpus/7424-duplex-from-streams.mjs @@ -0,0 +1,39 @@ +// Duplex.from over a lone Readable, a lone Writable and a { readable, +// writable } pair, as duplexify.js's _duplexify wraps them: the Duplex +// reads through the Readable and writes through the Writable, takes each +// side's object mode from the stream behind it, and closes once both ends +// finish; destroying it without an error destroys the streams behind it. +import { Duplex, PassThrough, Readable, Writable } from "node:stream"; +const log = []; +const r = Readable.from(["a", "b"]); +const dr = Duplex.from(r); +log.push(`dr ${dr.readable} ${dr.writable} ${dr.readableObjectMode} ${dr.writableObjectMode}`); +dr.on("end", () => log.push("dr end")); +dr.on("close", () => log.push("dr close")); +for await (const c of dr) log.push(`dr ${c}`); +const got = []; +const w = new Writable({ write(c, e, cb) { got.push(String(c)); log.push(`w write ${c}`); cb(); } }); +const dw = Duplex.from(w); +log.push(`dw ${dw.readable} ${dw.writable} ${dw.writableObjectMode}`); +dw.on("finish", () => log.push("dw finish")); +dw.on("close", () => log.push("dw close")); +dw.write("x", () => log.push("dw cb x")); +dw.end("y", () => log.push("dw end cb")); +await new Promise((res) => dw.on("close", res)); +const inner = new PassThrough({ objectMode: true }); +const out = new PassThrough({ objectMode: true }); +const pair = Duplex.from({ readable: out, writable: inner }); +log.push(`pair ${pair.readable} ${pair.writable} ${pair.readableObjectMode} ${pair.writableObjectMode}`); +inner.on("data", (v) => { log.push(`inner ${JSON.stringify(v)}`); out.write({ echo: v }); }); +inner.on("finish", () => { log.push("inner finish"); out.end(); }); +pair.on("data", (v) => log.push(`pair data ${JSON.stringify(v)}`)); +pair.write({ n: 1 }); +pair.end({ n: 2 }); +await new Promise((res) => pair.on("close", res)); +log.push("pair closed"); +const killed = Duplex.from(new PassThrough()); +killed.on("error", (e) => log.push(`killed error ${e.name}`)); +killed.on("close", () => log.push(`killed close ${killed.destroyed}`)); +killed.destroy(); +await new Promise((res) => setTimeout(res, 5)); +console.log(log.join("\n")); diff --git a/tests/corpus/7425-duplex-from-streams.ts b/tests/corpus/7425-duplex-from-streams.ts new file mode 100644 index 000000000..a9d85b797 --- /dev/null +++ b/tests/corpus/7425-duplex-from-streams.ts @@ -0,0 +1,72 @@ +// Duplex.from over streams in typed code: a Readable that fails destroys +// the Duplex with its error, a slow Writable's callbacks and the end +// callback follow its pace, a pair whose writable member is an async +// generator function's Duplex, compose() with an async function as its +// last stage, and a Duplex over a promise. +import { Duplex, PassThrough, Readable, Writable, compose } from "node:stream"; +const log: string[] = []; +async function main(): Promise { + const failing = new Readable({ read(): void { this.destroy(new Error("source failed")); } }); + const df = Duplex.from(failing); + df.on("error", (e: Error) => log.push(`df error ${e.message}`)); + df.on("close", () => log.push(`df close ${df.destroyed}`)); + df.resume(); + await new Promise((res) => df.on("close", res)); + const slow = new Writable({ + highWaterMark: 2, + write(c: Buffer, _e: string, cb: (err?: Error | null) => void): void { + log.push(`slow write ${c.toString()}`); + setTimeout(cb, 2); + }, + }); + const ds = Duplex.from(slow); + log.push(`write1 ${ds.write("aa", () => log.push("cb aa"))}`); + log.push(`write2 ${ds.write("bb", () => log.push("cb bb"))}`); + ds.on("drain", () => log.push("ds drain")); + ds.end(() => log.push("ds end cb")); + await new Promise((res) => ds.on("finish", res)); + log.push("ds finished"); + const upper = new PassThrough(); + const dp = Duplex.from({ + readable: Readable.from(["r1", "r2"]), + writable: async function* (source: AsyncIterable): AsyncGenerator { + for await (const c of source) log.push(`gen got ${String(c)}`); + yield "unused"; + }, + }); + dp.on("data", (c: unknown) => log.push(`dp data ${String(c)}`)); + dp.write("w1"); + dp.end(); + await new Promise((res) => dp.on("end", res)); + upper.end(); + await more(); + for (const line of log) console.log(line); +} +async function more(): Promise { + const sink = compose(new PassThrough({ objectMode: true }), async function (source: AsyncIterable): Promise { + for await (const c of source) log.push(`sink ${String(c)}`); + }); + log.push(`sink r=${sink.readable} w=${sink.writable}`); + sink.write("p"); + sink.end("q"); + await new Promise((res) => sink.on("finish", res)); + log.push("sink finished"); + const fromReadable: Duplex = Duplex.from(Readable.from([1, 2])); + const values: unknown[] = []; + for await (const v of fromReadable) values.push(v); + log.push(`readable ${JSON.stringify(values)}`); + const written: string[] = []; + const target = new Writable({ + write(c: Buffer, _e: string, cb: (err?: Error | null) => void): void { + written.push(c.toString()); + cb(); + }, + }); + const fromWritable: Duplex = Duplex.from(target); + fromWritable.end("only"); + await new Promise((res) => fromWritable.on("finish", res)); + log.push(`writable ${written.join(",")}`); + const value: Duplex = Duplex.from(Promise.resolve("resolved")); + for await (const v of value) log.push(`promise ${String(v)}`); +} +void main(); diff --git a/tests/corpus/7426-duplex-from-promise.mjs b/tests/corpus/7426-duplex-from-promise.mjs new file mode 100644 index 000000000..46d19586b --- /dev/null +++ b/tests/corpus/7426-duplex-from-promise.mjs @@ -0,0 +1,27 @@ +// Duplex.from over a promise: a Duplex that is not writable and reads the +// promise's value once it settles (nothing for null), then ends; a +// rejection destroys it with the reason. The events interleave with +// microtasks and nextTick callbacks as in Node. +import { Duplex } from "node:stream"; +const log = []; +const tick = (n) => { if (n < 5) { log.push(`tick ${n}`); process.nextTick(() => tick(n + 1)); } }; +const later = (v, ms) => new Promise((res) => setTimeout(() => res(v), ms)); +const a = Duplex.from(Promise.resolve({ answer: 42 })); +log.push(`a ${a.readable} ${a.writable} ${a.readableObjectMode}`); +a.on("data", (v) => log.push(`a data ${JSON.stringify(v)}`)); +a.on("end", () => log.push("a end")); +a.on("close", () => log.push("a close")); +Promise.resolve().then(() => log.push("microtask 1")).then(() => log.push("microtask 2")); +tick(0); +await new Promise((res) => a.on("close", res)); +const b = Duplex.from(later(null, 1)); +b.on("data", () => log.push("b data")); +b.on("end", () => log.push("b end")); +await new Promise((res) => b.on("close", res)); +const c = Duplex.from(Promise.reject(new Error("nope"))); +c.on("error", (e) => log.push(`c error ${e.message}`)); +c.on("close", () => log.push(`c close ${c.destroyed}`)); +await new Promise((res) => c.on("close", res)); +const d = Duplex.from(later("text", 1)); +for await (const v of d) log.push(`d ${v}`); +console.log(log.join("\n")); diff --git a/tests/corpus/7427-duplex-from-async-function.mjs b/tests/corpus/7427-duplex-from-async-function.mjs new file mode 100644 index 000000000..66ce35f9a --- /dev/null +++ b/tests/corpus/7427-duplex-from-async-function.mjs @@ -0,0 +1,40 @@ +// Duplex.from over an async function: a Duplex that is only writable, whose +// writes reach the function's source and whose end waits for the +// function's promise. A value other than null or undefined is +// ERR_INVALID_RETURN_VALUE, a return before the input is consumed aborts +// the Duplex, and a throw destroys it. +import { Duplex } from "node:stream"; +const log = []; +const sink = Duplex.from(async function (source) { + for await (const c of source) log.push(`sink got ${c}`); + log.push("sink done"); +}); +log.push(`sink ${sink.readable} ${sink.writable} ${sink.writableObjectMode}`); +sink.on("finish", () => log.push("sink finish")); +sink.on("close", () => log.push("sink close")); +sink.write("a", () => log.push("cb a")); +sink.end("b", () => log.push("end cb")); +await new Promise((res) => sink.on("close", res)); +const bad = Duplex.from(async function (source) { + for await (const c of source) log.push(`bad got ${c}`); + return 5; +}); +bad.on("error", (e) => log.push(`bad error ${e.code} ${e.message}`)); +bad.on("close", () => log.push("bad close")); +bad.end("x"); +await new Promise((res) => bad.on("close", res)); +const early = Duplex.from(async function (source) { + for await (const c of source) { log.push(`early got ${c}`); break; } +}); +early.on("error", (e) => log.push(`early error ${e.name}`)); +early.on("close", () => log.push(`early close ${early.destroyed}`)); +early.write("1"); +early.write("2"); +await new Promise((res) => early.on("close", res)); +const failing = Duplex.from(async function (source) { + for await (const c of source) throw new Error("fail " + c); +}); +failing.on("error", (e) => log.push(`failing error ${e.message}`)); +failing.write("z"); +await new Promise((res) => failing.on("close", res)); +console.log(log.join("\n")); diff --git a/tests/harness/corpus-costs/7424-duplex-from-streams.mjs.json b/tests/harness/corpus-costs/7424-duplex-from-streams.mjs.json new file mode 100644 index 000000000..14a2052b9 --- /dev/null +++ b/tests/harness/corpus-costs/7424-duplex-from-streams.mjs.json @@ -0,0 +1 @@ +{ "plain": 2607, "san": 3440 } diff --git a/tests/harness/corpus-costs/7425-duplex-from-streams.ts.json b/tests/harness/corpus-costs/7425-duplex-from-streams.ts.json new file mode 100644 index 000000000..9dde8d70e --- /dev/null +++ b/tests/harness/corpus-costs/7425-duplex-from-streams.ts.json @@ -0,0 +1 @@ +{ "plain": 2009, "san": 2743 } diff --git a/tests/harness/corpus-costs/7426-duplex-from-promise.mjs.json b/tests/harness/corpus-costs/7426-duplex-from-promise.mjs.json new file mode 100644 index 000000000..9195268ab --- /dev/null +++ b/tests/harness/corpus-costs/7426-duplex-from-promise.mjs.json @@ -0,0 +1 @@ +{ "plain": 1877, "san": 2357 } diff --git a/tests/harness/corpus-costs/7427-duplex-from-async-function.mjs.json b/tests/harness/corpus-costs/7427-duplex-from-async-function.mjs.json new file mode 100644 index 000000000..c572d9930 --- /dev/null +++ b/tests/harness/corpus-costs/7427-duplex-from-async-function.mjs.json @@ -0,0 +1 @@ +{ "plain": 1811, "san": 2417 } diff --git a/tests/harness/self-hosting-native-toolchain.test.ts b/tests/harness/self-hosting-native-toolchain.test.ts index f143419dc..81ff5b293 100644 --- a/tests/harness/self-hosting-native-toolchain.test.ts +++ b/tests/harness/self-hosting-native-toolchain.test.ts @@ -108,7 +108,7 @@ for (const backend of ["llvm"] as const) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 29, marker: "scr_runtime_abi_v29" }, + runtime_abi: { version: 30, marker: "scr_runtime_abi_v30" }, compiler: { command: "fixture", identity: "fixture", target: target.llvmTriple }, macros: { executable: [], excluded: [], sanitizer: "external-toolchain-required" }, flavors: { From b074a5e20fea2c7dced690f1ae39cb33bff72640 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 13:49:49 -0500 Subject: [PATCH 066/197] Test for-in key presence in the overflow map instead of a fresh key list for-in over a pure index-signature record (`Record`) skips keys deleted during the walk by testing each snapshotted key for presence before visiting it. The test built the overflow map's whole key list and searched it with includes() on every visit, so a walk over n keys took O(n^2) time: 200 walks over 2000 keys took 8.7 s, against 31 ms under Node. The test is now recordOvfHas, the overflow map's hashed presence check, which answers the same question in constant time. The new benchmark workload dictionary-walks walks, prunes and merges Record dictionaries with for-in and Object.keys/values/ entries. It takes 7.1 s before this change and 23 ms after (Node 116 ms). Corpus program 7011 pins for-in's visit order and its handling of keys deleted, re-added and added during the walk. --- benchmarks/runtime/README.md | 1 + .../runtime/workloads/dictionary-walks.json | 6 +++ .../runtime/workloads/dictionary-walks.ts | 53 +++++++++++++++++++ .../src/frontend/lowering/lower-stmts.ts | 10 ++-- .../7011-for-in-dictionary-deletes.ts.json | 6 +++ .../corpus/7011-for-in-dictionary-deletes.ts | 42 +++++++++++++++ .../7011-for-in-dictionary-deletes.ts.json | 1 + 7 files changed, 115 insertions(+), 4 deletions(-) create mode 100644 benchmarks/runtime/workloads/dictionary-walks.json create mode 100644 benchmarks/runtime/workloads/dictionary-walks.ts create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7011-for-in-dictionary-deletes.ts.json create mode 100644 tests/corpus/7011-for-in-dictionary-deletes.ts create mode 100644 tests/harness/corpus-costs/7011-for-in-dictionary-deletes.ts.json diff --git a/benchmarks/runtime/README.md b/benchmarks/runtime/README.md index 2653ec51a..e177e36c0 100644 --- a/benchmarks/runtime/README.md +++ b/benchmarks/runtime/README.md @@ -11,6 +11,7 @@ Application-shaped workloads for measuring the speed of compiled executables. Ea | `channel-messages` | cpu | 200,000 MessageChannel messages in one thread, half numbers and half small objects, all posted before the port delivers any | | `csv-numbers` | cpu | `split`, `Number`, `parseInt`, `toFixed` | | `cyclic-graphs` | cpu | object graphs with back references built and dropped: trees with parent and sibling links, rings of records sharing a long-lived registry, closures capturing their owner | +| `dictionary-walks` | cpu | `Record` dictionaries walked with for-in and `Object.keys`/`values`/`entries` while entries are added, updated and deleted | | `functional-pipeline` | cpu | `filter`/`map`/`reduce`/`flatMap` chains, closures, small records | | `intern-table`, `intern-table-map`, `intern-table-1`, `intern-table-map-1` | cpu | worker threads interning the same type-like objects under string keys and records under number keys, then hits and misses: one shared `InternTable` per kind of key, or a module-level `Map` per thread (the `-map` entries), with 8 threads or 1 (the `-1` entries); under Node.js every thread has its own tables | | `json-records` | cpu | `JSON.stringify`, typed `JSON.parse`, Map aggregation | diff --git a/benchmarks/runtime/workloads/dictionary-walks.json b/benchmarks/runtime/workloads/dictionary-walks.json new file mode 100644 index 000000000..ebfb6ea3f --- /dev/null +++ b/benchmarks/runtime/workloads/dictionary-walks.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/dictionary-walks.ts", + "args": [ + "1" + ] +} diff --git a/benchmarks/runtime/workloads/dictionary-walks.ts b/benchmarks/runtime/workloads/dictionary-walks.ts new file mode 100644 index 000000000..30b18be63 --- /dev/null +++ b/benchmarks/runtime/workloads/dictionary-walks.ts @@ -0,0 +1,53 @@ +// String-keyed dictionaries (`Record`) walked with for-in, +// Object.keys, Object.values and Object.entries while entries are added, +// updated and deleted: per-category totals, a pruning pass and a merge. +let seed = 777; +function random(): number { + seed = (Math.imul(seed, 1103515245) + 12345) >>> 0; + return seed / 4294967296; +} + +const scale = Number(process.argv[2] ?? "1"); +const categories = 4000; +const counts: Record = {}; +for (let i = 0; i < Math.floor(200000 * scale); i++) { + const key = "c" + Math.floor(random() * categories); + counts[key] = (counts[key] ?? 0) + 1; +} + +let walked = 0; +let sum = 0; +for (let pass = 0; pass < 40; pass++) { + for (const key in counts) { + walked++; + sum += counts[key]!; + } +} + +// Prune: delete the small categories during the walk, including keys the +// walk has not reached yet. +let pruned = 0; +for (const key in counts) { + const n = counts[key]!; + if (n < 45) { + delete counts[key]; + pruned++; + } + const ahead = "c" + ((Number(key.slice(1)) + 1) % categories); + if (n > 60 && ahead in counts && counts[ahead]! < 45) { + delete counts[ahead]; + pruned++; + } +} + +// Merge into a second dictionary and read it back every way. +const merged: Record = {}; +for (const key of Object.keys(counts)) merged[key.toUpperCase()] = counts[key]! * 2; +let values = 0; +for (const v of Object.values(merged)) values += v; +let entries = 0; +for (const [key, v] of Object.entries(merged)) entries += key.length + v; + +console.log( + `walked ${walked}, sum ${sum}, pruned ${pruned}, kept ${Object.keys(counts).length}, values ${values}, entries ${entries}`, +); diff --git a/packages/compiler/src/frontend/lowering/lower-stmts.ts b/packages/compiler/src/frontend/lowering/lower-stmts.ts index 213abfec6..d49c7643e 100644 --- a/packages/compiler/src/frontend/lowering/lower-stmts.ts +++ b/packages/compiler/src/frontend/lowering/lower-stmts.ts @@ -11484,11 +11484,13 @@ function lowerForIn(lowerer: Lowerer, stmt: ts.ForInStatement): IrStmt { keysT, loc, ); + // The overflow map's hashed key set answers presence directly; a + // fresh key list per visit would make the walk quadratic. const guard = (kRef: IrExpr): IrExpr => ({ - kind: "arrIntrinsic", - method: "includes", - receiver: { kind: "recordOvfKeys", obj: recvRef(), shapeId, type: keysT, loc }, - args: [kRef], + kind: "recordOvfHas", + obj: recvRef(), + shapeId, + key: kRef, type: BOOL, loc, }); diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7011-for-in-dictionary-deletes.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7011-for-in-dictionary-deletes.ts.json new file mode 100644 index 000000000..3b2b79c3b --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7011-for-in-dictionary-deletes.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7011-for-in-dictionary-deletes.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7011-for-in-dictionary-deletes.ts b/tests/corpus/7011-for-in-dictionary-deletes.ts new file mode 100644 index 000000000..91dac573b --- /dev/null +++ b/tests/corpus/7011-for-in-dictionary-deletes.ts @@ -0,0 +1,42 @@ +// for-in over a string-keyed dictionary visits the keys present when the +// walk starts, in insertion order, and skips a key deleted before its +// turn; keys added during the walk are not visited. A key deleted and +// added again before its turn is visited. +const d: Record = {}; +for (let i = 0; i < 8; i++) d["k" + i] = i; +const seen: string[] = []; +for (const key in d) { + seen.push(key); + if (key === "k1") { + delete d["k3"]; + delete d["k0"]; + d["k8"] = 8; + } + if (key === "k2") { + delete d["k5"]; + d["k5"] = 50; + } + if (key === "k4") delete d["k4"]; +} +console.log(seen.join(",")); +console.log(Object.keys(d).join(","), JSON.stringify(d)); + +// A numeric-looking key is walked in integer order before string keys. +const n: Record = {}; +n["b"] = "x"; +n["2"] = "y"; +n["10"] = "z"; +n["a"] = "w"; +const order: string[] = []; +for (const key in n) { + order.push(key); + if (key === "2") delete n["a"]; +} +console.log(order.join(",")); + +// A larger dictionary walked many times. +const big: Record = {}; +for (let i = 0; i < 3000; i++) big["e" + i] = i; +let total = 0; +for (let pass = 0; pass < 20; pass++) for (const key in big) total += big[key]!; +console.log(total); diff --git a/tests/harness/corpus-costs/7011-for-in-dictionary-deletes.ts.json b/tests/harness/corpus-costs/7011-for-in-dictionary-deletes.ts.json new file mode 100644 index 000000000..e172eed8a --- /dev/null +++ b/tests/harness/corpus-costs/7011-for-in-dictionary-deletes.ts.json @@ -0,0 +1 @@ +{ "plain": 1527, "san": 2135 } From fbe4067b3b3de092dd411845f6965c85524735cc Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 14:15:29 -0500 Subject: [PATCH 067/197] Skip emitter analysis walks whose answers are already known The LLVM emitter's constructor walks the whole module about 33 times. Several walks repeat work another already did, or run where the result is provably empty: - computePublishedTypes and computeCycleMutability run once and are shared (may-throw, immutable fields and traced shapes took their own). - The WebCrypto-asymmetric and Ed25519 runtime-feature conditions join the feature scan's one walk instead of walking the module again. - int32 slots: the fixpoint skips the write walk of a function without a candidate write, dynamicFieldStores stops once every field name is dynamic, and dynamicallyStoredFamilies returns early when no f64 field can be named by a dynamic store. - stableGlobals stops once no candidate is left, the second pass of findAppendedArrayLiterals and the target resolution of findConstantCallbacks run only when there is something to find, boxedUnionIds needs unions, and observesStacks walks for call sites and the "stack" literal only when a dynamic key asks. - everyModuleNode visits type trees without a closure per node. The emitted LLVM IR is unchanged. --- .../compiler/src/backend/cycle-analysis.ts | 9 ++- .../compiler/src/backend/cycle-mutability.ts | 16 ++++++ packages/compiler/src/backend/immutability.ts | 9 ++- .../backend/llvm/appended-array-literals.ts | 1 + .../src/backend/llvm/constant-callbacks.ts | 4 +- packages/compiler/src/backend/llvm/emitter.ts | 10 +++- .../src/backend/llvm/nullable-unions.ts | 2 + packages/compiler/src/backend/may-throw.ts | 10 +++- .../src/backend/stack-observations.ts | 42 +++++++++----- packages/compiler/src/ir/int32-slots.ts | 47 ++++++++++------ packages/compiler/src/ir/runtime-features.ts | 56 ++++++++++++++----- packages/compiler/src/ir/traverse.ts | 11 +++- 12 files changed, 159 insertions(+), 58 deletions(-) diff --git a/packages/compiler/src/backend/cycle-analysis.ts b/packages/compiler/src/backend/cycle-analysis.ts index aa624579f..fbd96e854 100644 --- a/packages/compiler/src/backend/cycle-analysis.ts +++ b/packages/compiler/src/backend/cycle-analysis.ts @@ -11,7 +11,7 @@ import { STRING, VOID, } from "../ir/ir.js"; -import { computeCycleMutability, fieldKey } from "./cycle-mutability.js"; +import { type CycleMutability, computeCycleMutability, fieldKey } from "./cycle-mutability.js"; /** Cycle capability for native code generation. * Nodes are shape units and unions; edges are the references their fields @@ -38,8 +38,10 @@ import { computeCycleMutability, fieldKey } from "./cycle-mutability.js"; export function computeTraced( mod: IrModule, stores?: DynamicFieldStores, + /** computeCycleMutability's answer for this module, when the caller has it. */ + precomputed?: CycleMutability, ): { shapes: Set; unions: Set } { - const { shapes, unions } = analyzeCycles(mod, false, stores); + const { shapes, unions } = analyzeCycles(mod, false, stores, precomputed); return { shapes, unions }; } @@ -93,6 +95,7 @@ function analyzeCycles( mod: IrModule, labels: boolean, stores?: DynamicFieldStores, + precomputed?: CycleMutability, ): { shapes: Set; unions: Set; @@ -125,7 +128,7 @@ function analyzeCycles( const lifetimeRoots = new Set(mod.contracts?.programLifetime ?? []); const programLifetime = (className: string): boolean => lifetimeRoots.size > 0 && lifetimeRoots.has(rootOf(className)); - const mutability = computeCycleMutability(mod, unitOfClass, stores); + const mutability = precomputed ?? computeCycleMutability(mod, unitOfClass, stores); const shapeDefs = [ ...classes.map((c) => ({ key: `object:${c.name}`, diff --git a/packages/compiler/src/backend/cycle-mutability.ts b/packages/compiler/src/backend/cycle-mutability.ts index 78daa6204..95630ef23 100644 --- a/packages/compiler/src/backend/cycle-mutability.ts +++ b/packages/compiler/src/backend/cycle-mutability.ts @@ -69,6 +69,22 @@ export function fieldKey(unit: string, field: string): string { return `${unit}\0${field}`; } +/** The cycle unit of a class: `object:`, the root of its hierarchy, + * as computeImmutableFields and computeTraced group classes. */ +export function hierarchyUnitOfClass(mod: IrModule): (className: string) => string { + const baseOf = new Map((mod.classes ?? []).map((c) => [c.name, c.base ?? null] as const)); + const roots = new Map(); + return (name) => { + const known = roots.get(name); + if (known !== undefined) return `object:${known}`; + let cur = name; + for (let base = baseOf.get(cur); base !== null && base !== undefined; base = baseOf.get(cur)) + cur = base; + roots.set(name, cur); + return `object:${cur}`; + }; +} + export function computeCycleMutability( mod: IrModule, unitOfClass: (className: string) => string, diff --git a/packages/compiler/src/backend/immutability.ts b/packages/compiler/src/backend/immutability.ts index 6c37731e3..21380de8f 100644 --- a/packages/compiler/src/backend/immutability.ts +++ b/packages/compiler/src/backend/immutability.ts @@ -48,6 +48,8 @@ export class ImmutableFields { export function computeImmutableFields( mod: IrModule, stores?: DynamicFieldStores, + /** computeCycleMutability's answer for this module, when the caller has it. */ + precomputed?: CycleMutability, ): ImmutableFields | null { if (mod.lib !== undefined) return null; const baseOf = new Map((mod.classes ?? []).map((c) => [c.name, c.base ?? null] as const)); @@ -62,7 +64,7 @@ export function computeImmutableFields( return cur; }; const unitOfClass = (name: string): string => `object:${rootOf(name)}`; - const mutability = computeCycleMutability(mod, unitOfClass, stores); + const mutability = precomputed ?? computeCycleMutability(mod, unitOfClass, stores); return mutability.all ? null : new ImmutableFields(mutability, unitOfClass); } @@ -213,6 +215,9 @@ export function stableGlobals(mod: IrModule): Set { } return everyStmtChild(s, expr, stmt); }; - for (const fn of mod.functions) fn.body.every(stmt); + for (const fn of mod.functions) { + if (candidates.size === 0) break; + fn.body.every(stmt); + } return candidates; } diff --git a/packages/compiler/src/backend/llvm/appended-array-literals.ts b/packages/compiler/src/backend/llvm/appended-array-literals.ts index 1d4577ae6..09a2976de 100644 --- a/packages/compiler/src/backend/llvm/appended-array-literals.ts +++ b/packages/compiler/src/backend/llvm/appended-array-literals.ts @@ -85,6 +85,7 @@ export function findAppendedArrayLiterals(module: IrModule): ReadonlySet for (const field of fieldAliases.get(binding) ?? []) appendedFields.add(field); const result = new Set(); + if (appendedBindings.size === 0 && appendedFields.size === 0) return result; const consider = (value: IrExpr | null, appended: boolean): void => { if (value && appended && isEmptyLiteral(value)) result.add(value); }; diff --git a/packages/compiler/src/backend/llvm/constant-callbacks.ts b/packages/compiler/src/backend/llvm/constant-callbacks.ts index 8a898d6a0..cf5c1b062 100644 --- a/packages/compiler/src/backend/llvm/constant-callbacks.ts +++ b/packages/compiler/src/backend/llvm/constant-callbacks.ts @@ -252,11 +252,13 @@ export function findConstantCallbacks( // entries actually used as indirect callees in each body. A module-wide // global/function cross product would dominate large-source emission. const globalTargets = new Map(); + const result = new Map>(); + // Only indirect callees get targets: without one, nothing resolves. + if (![...indirectCallees.values()].some((callees) => callees.size > 0)) return result; for (const [id, value] of globals) { const target = resolve(null, value); if (target) globalTargets.set(id, target); } - const result = new Map>(); for (const fn of mod.functions) { const needed = indirectCallees.get(fn.name)!; if (!needed.size) continue; diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index bfde59a30..d0cbe54e2 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -175,6 +175,7 @@ import { emitDynamicExpr } from "./expr-dynamic.js"; import { emitIntrinsicExpr, emitSerializationExpr, emitAsyncExpr } from "./expr-async.js"; import { PublishWalkers } from "./publish.js"; import { computeImmutableFields, stableGlobals, type ImmutableFields } from "../immutability.js"; +import { computeCycleMutability, hierarchyUnitOfClass } from "../cycle-mutability.js"; import { computePublishedTypes, type PublishedTypes } from "../publication.js"; import { emitExpr } from "./expr-dispatch.js"; @@ -1312,8 +1313,10 @@ export class LlEmitter { // and per async fiber; WASI, Windows and library builds have none. const triple = (options.targetTriple ?? "").toLowerCase(); const windows = triple === "" ? process.platform === "win32" : triple.includes("windows"); + this.publishedTypes = computePublishedTypes(mod); const mt = computeMayThrow(mod, { stackChecks: !this.wasi && mod.lib === undefined && !windows, + published: this.publishedTypes, }); this.mayThrow = mt.fns; this.workerEntryPolls = mt.workerEntryPolls ?? null; @@ -1322,11 +1325,12 @@ export class LlEmitter { // runtime moves deep prefix nests to a fresh stack using the guard, // which Windows lacks. this.stacklessAsyncAllowed = this.stacklessGenerators && mod.lib === undefined && !windows; - this.publishedTypes = computePublishedTypes(mod); this.unframedFunctions = this.stackTraces ? captureFreeFunctions(mod, this.stackChecks, this.publishedTypes) : new Set(); - this.immutableFields = computeImmutableFields(mod, dynamicStores); + // One field-mutability answer serves immutable fields and traced shapes. + const cycleMutability = computeCycleMutability(mod, hierarchyUnitOfClass(mod), dynamicStores); + this.immutableFields = computeImmutableFields(mod, dynamicStores, cycleMutability); this.constructorFunctions = new Set((mod.classes ?? []).map((c) => `%${c.name}.constructor`)); this.indirectMayThrow = mt.indirect || mod.workers === true; for (const cls of mod.classes ?? []) { @@ -1451,7 +1455,7 @@ export class LlEmitter { this.receiverReaders = new AmbientReceiverReaders(this.fnByName); this.stackCaptures = new StackCaptures(this.fnByName, this.callLifetimes, this.stackCallbacks); this.lazyCaptures = new LazyCaptures(this.fnByName); - const traced = computeTraced(mod, dynamicStores); + const traced = computeTraced(mod, dynamicStores, cycleMutability); this.tracedShapes = traced.shapes; this.tracedUnions = traced.unions; // A shape union's value is whichever record it holds, and the collector diff --git a/packages/compiler/src/backend/llvm/nullable-unions.ts b/packages/compiler/src/backend/llvm/nullable-unions.ts index 8729e9649..2c355f551 100644 --- a/packages/compiler/src/backend/llvm/nullable-unions.ts +++ b/packages/compiler/src/backend/llvm/nullable-unions.ts @@ -279,6 +279,8 @@ function shapeUnion( * tag and payload inside the C map. */ function boxedUnionIds(mod: IrModule): Set { const boxed = new Set(); + // Only the module's union definitions consult the result. + if (!mod.unions?.length) return boxed; const mark = (t: IrType): void => { if (t.kind === "union") boxed.add(t.unionId); }; diff --git a/packages/compiler/src/backend/may-throw.ts b/packages/compiler/src/backend/may-throw.ts index b72dc545d..a04cce856 100644 --- a/packages/compiler/src/backend/may-throw.ts +++ b/packages/compiler/src/backend/may-throw.ts @@ -35,7 +35,12 @@ export type MayThrowReason = * and a generator's `.return()` unwind through `catch`. */ export function computeMayThrow( mod: IrModule, - options: { stackChecks?: boolean; explain?: boolean } = {}, + options: { + stackChecks?: boolean; + explain?: boolean; + /** computePublishedTypes(mod), when the caller already has it. */ + published?: ReturnType; + } = {}, ): { fns: Set; /** With `explain`: the reason each may-throw function was marked. */ @@ -50,7 +55,8 @@ export function computeMayThrow( const terminations = mod.workers === true ? workerTerminationPolls(mod) : null; const stackChecks = options.stackChecks === true ? stackCheckedFunctions(mod) : null; // Stores that can reach a published object throw the frozen TypeError. - const published = computePublishedTypes(mod); + const published = + options.published !== undefined ? options.published : computePublishedTypes(mod); interface Facts { throws: boolean; callees: string[]; diff --git a/packages/compiler/src/backend/stack-observations.ts b/packages/compiler/src/backend/stack-observations.ts index fa67c071f..c448897f1 100644 --- a/packages/compiler/src/backend/stack-observations.ts +++ b/packages/compiler/src/backend/stack-observations.ts @@ -29,35 +29,50 @@ const NON_STACK_KEY_TYPES: ReadonlySet = new Set(["f64", "bigint * containers or calls; a key the program assembles from other strings or * from data it reads keeps the static answer. */ class StackKeys { - private readonly spelled: boolean; - private readonly calls = new Map(); - private readonly named: Set; + private spelled: boolean | undefined; + private census: + | { calls: Map; named: Set } + | undefined; private readonly inits = new Map>(); /** Settled answers by function and local: true where a search reached a * key that can be "stack", false where a search that found none passed. */ private readonly answers = new Map(); - constructor(mod: IrModule) { - this.spelled = spellsStack(mod); - this.named = new Set(moduleFunctionReferences(mod)); + // The module walks run on the first question that needs them: a program + // without a dynamic key read asks none. + constructor(private readonly mod: IrModule) {} + + private spellsStack(): boolean { + return (this.spelled ??= spellsStack(this.mod)); + } + + private callCensus(): { + calls: Map; + named: Set; + } { + if (this.census !== undefined) return this.census; + const mod = this.mod; + const calls = new Map(); + const named = new Set(moduleFunctionReferences(mod)); // Class methods, accessors and constructors are called through dispatch. const classNames = new Set((mod.classes ?? []).map((cls) => cls.name)); for (const fn of mod.functions) { - if (isClassMember(fn.name, classNames)) this.named.add(fn.name); + if (isClassMember(fn.name, classNames)) named.add(fn.name); everyStmtList(fn.body, { stmt: () => true, expr: (expr) => { if (expr.kind === "call") { - const sites = this.calls.get(expr.callee) ?? []; + const sites = calls.get(expr.callee) ?? []; sites.push({ caller: fn, args: expr.args }); - this.calls.set(expr.callee, sites); + calls.set(expr.callee, sites); } const referenced = exprFunctionReference(expr); - if (referenced !== undefined) this.named.add(referenced); + if (referenced !== undefined) named.add(referenced); return true; }, }); } + return (this.census = { calls, named }); } mayBeStack(fn: IrFunction, key: IrExpr): boolean { @@ -71,7 +86,7 @@ class StackKeys { private keyMayBeStack(fn: IrFunction, key: IrExpr, visited: Set): boolean { if (key.kind === "strLit") return key.value === "stack"; - if (!this.spelled) return false; + if (!this.spellsStack()) return false; if (NON_STACK_KEY_TYPES.has(key.type.kind)) return false; if (key.kind === "toString" && NON_STACK_KEY_TYPES.has(key.operand.type.kind)) return false; if (key.kind === "dynFrom" && NON_STACK_KEY_TYPES.has(key.value.type.kind)) return false; @@ -93,8 +108,9 @@ class StackKeys { if (local === undefined || local.mutable) return true; const index = fn.params.findIndex((param) => param.localId === localId); if (index !== -1) { - const sites = this.calls.get(fn.name); - if (this.named.has(fn.name) || sites === undefined) return true; + const { calls, named } = this.callCensus(); + const sites = calls.get(fn.name); + if (named.has(fn.name) || sites === undefined) return true; return sites.some( (site) => site.args.length !== fn.params.length || diff --git a/packages/compiler/src/ir/int32-slots.ts b/packages/compiler/src/ir/int32-slots.ts index 21223862a..ac0747276 100644 --- a/packages/compiler/src/ir/int32-slots.ts +++ b/packages/compiler/src/ir/int32-slots.ts @@ -508,7 +508,10 @@ export function dynamicFieldStores( }; const dynWritten = new Set(); let dynWritesAll = false; - for (const fn of mod.functions) + // A store whose key is not a literal names every field: later stores add + // nothing, so the walk stops there. + for (const fn of mod.functions) { + if (dynWritesAll) break; walkFunction(fn, { expr: (e) => { if (dynWritesAll) return; @@ -524,6 +527,7 @@ export function dynamicFieldStores( } }, }); + } return { all: dynWritesAll, names: dynWritten }; } @@ -536,6 +540,22 @@ export function dynamicallyStoredFamilies( slots: Int32Slots, stores?: DynamicFieldStores, ): Set { + const records = new Map((mod.records ?? []).map((r) => [r.id, r])); + const unions = new Map((mod.unions ?? []).map((u) => [u.id, u])); + // A live dyn view (typed-ref capsule) of an instance writes back into + // its fields only after a runtime store into the capsule; the commit then + // writes every field from the view, which holds the exact snapshot value + // of any property no store named. So reaching dynamic code disqualifies + // exactly the field names some dynamic store in the program can name. + const { all: dynWritesAll, names: dynWritten } = + stores ?? dynamicFieldStores(mod, records, unions); + const storable = (field: { name: string; type: IrType }): boolean => + field.type.kind === "f64" && + !field.name.startsWith("%") && + (dynWritesAll || dynWritten.has(field.name)); + const families = new Set(); + // Without a field some store can name, no class is worth exposing. + if (!(mod.classes ?? []).some((cls) => cls.fields.some(storable))) return families; const classes = new Map((mod.classes ?? []).map((c) => [c.name, c])); const children = new Map(); for (const cls of classes.values()) @@ -554,8 +574,6 @@ export function dynamicallyStoredFamilies( } return out; }; - const records = new Map((mod.records ?? []).map((r) => [r.id, r])); - const unions = new Map((mod.unions ?? []).map((u) => [u.id, u])); const exposedClasses = new Set(); const seenNamed = new Set(); let exposeAll = false; @@ -596,13 +614,6 @@ export function dynamicallyStoredFamilies( }); } }; - // A live dyn view (typed-ref capsule) of an instance writes back into - // its fields only after a runtime store into the capsule; the commit then - // writes every field from the view, which holds the exact snapshot value - // of any property no store named. So reaching dynamic code disqualifies - // exactly the field names some dynamic store in the program can name. - const { all: dynWritesAll, names: dynWritten } = - stores ?? dynamicFieldStores(mod, records, unions); for (const fn of mod.functions) walkFunction(fn, { expr: (e) => { @@ -628,16 +639,10 @@ export function dynamicallyStoredFamilies( if (s.kind === "throw") expose(s.value.type); }, }); - const families = new Set(); for (const cls of classes.values()) { if (!exposeAll && !exposedClasses.has(cls.name)) continue; for (const field of cls.fields) - if ( - field.type.kind === "f64" && - !field.name.startsWith("%") && - (dynWritesAll || dynWritten.has(field.name)) - ) - families.add(slots.family(cls.name, field.name)!); + if (storable(field)) families.add(slots.family(cls.name, field.name)!); } return families; } @@ -1058,12 +1063,18 @@ export function analyzeInt32Slots(mod: IrModule, inputs: Int32SlotInputs = {}): if (params) for (const id of params.keys()) reads(`param:${fn.name}\0${id}`, fn.name); // Range facts are consulted only at writes into candidate slots; slots // are only ever dropped, so a body without such a write needs none. - const ranges: IntegerRanges = hasCandidateWrite(fn) + const candidate = hasCandidateWrite(fn); + const ranges: IntegerRanges = candidate ? analyzeIntegerRanges(numeric(fn), slots.facts(fn.name)) : new Map(); if (ranges.size > 0) lastWrites.set(fn.name, slotWrites(fn, ranges, functions)); else lastWrites.delete(fn.name); const proven: IrExpr[] = []; + // The walk below acts only at writes into candidate slots. + if (!candidate) { + provenByFunction.set(fn.name, proven); + continue; + } const int32 = (e: IrExpr): boolean => withinInt32(ranges.get(e)); // An int32 written into a slot still assumed non-negative: widen the // assumption when the value may be negative, and reanalyze its readers. diff --git a/packages/compiler/src/ir/runtime-features.ts b/packages/compiler/src/ir/runtime-features.ts index 5b984f318..f8b5e8dcc 100644 --- a/packages/compiler/src/ir/runtime-features.ts +++ b/packages/compiler/src/ir/runtime-features.ts @@ -605,14 +605,23 @@ const STDIO_EXTENDED_NAMES: ReadonlySet = new Set(["setEncoding", "resiz /** The libCall nodes the rules below test. */ type LibCallNode = IrExpr & { kind: "libCall" }; +/** A condition over several nodes of the module (a WebCrypto call and an + * algorithm name): the scan's one walk shows it every expression, and the + * feature is set when `result` answers true after the walk. */ +interface RuleScan { + expr(node: IrExpr): void; + result(): boolean; +} + /** What makes a program need one optional runtime unit. The feature is set * when any condition holds: `initial` from the module's declarations, a * libCall named in `fns` or starting with one of `prefixes`, a libCall - * `libCall` accepts, another expression node `expr` accepts, or a type - * slot of one of the `types` kinds. A libCall or type that needs several - * units appears in each unit's rule. */ + * `libCall` accepts, another expression node `expr` accepts, a type slot of + * one of the `types` kinds, or a `scan` over the whole module. A libCall or + * type that needs several units appears in each unit's rule. */ interface RuntimeFeatureRule { initial?: (mod: IrModule) => boolean; + scan?: () => RuleScan; fns?: readonly IrLibFn[]; prefixes?: readonly string[]; libCall?: (node: LibCallNode) => boolean; @@ -650,17 +659,32 @@ const WEBCRYPTO_ASYMMETRIC_NAMES: ReadonlySet = new Set([ /** Whether a string literal in the module names Ed25519, without regard * to case ("ed25519" for node:crypto, "Ed25519" for WebCrypto): the * Ed25519 unit links then, alongside the pki unit. */ -export function moduleNamesEd25519(mod: IrModule): boolean { +function ed25519NameScan(): RuleScan { let named = false; - everyModuleNode(mod, { + return { expr: (node) => { - if (node.kind === "strLit" && node.value.toLowerCase() === "ed25519") named = true; - return !named; + if (!named && node.kind === "strLit" && node.value.toLowerCase() === "ed25519") named = true; }, - stmt: () => !named, - type: () => true, - }); - return named; + result: () => named, + }; +} + +/** moduleUsesWebCryptoAsymmetric's condition as a runtime-feature scan. */ +function webCryptoAsymmetricScan(): RuleScan { + let webcrypto = false; + let named = false; + return { + expr: (node) => { + if (node.kind === "libCall" && node.fn.startsWith("webcrypto.")) webcrypto = true; + else if ( + !named && + node.kind === "strLit" && + WEBCRYPTO_ASYMMETRIC_NAMES.has(node.value.toLowerCase()) + ) + named = true; + }, + result: () => webcrypto && named, + }; } /** Whether the module calls WebCrypto and a string literal in it names one @@ -906,10 +930,10 @@ const RUNTIME_FEATURE_RULES = { perfLoop: rule({ fns: ["perf.idleTime"], prefixes: ["perf.loop", "perf.eld"] }), /** scr_pki.c over the mbedTLS archive: asymmetric KeyObjects and * signatures. */ - pki: rule({ initial: moduleUsesWebCryptoAsymmetric, fns: [...PKI_LIB_FNS] }), + pki: rule({ scan: webCryptoAsymmetricScan, fns: [...PKI_LIB_FNS] }), /** scr_ed25519.c (the vendored libsodium Ed25519), linked with pki: a * string literal in the module names Ed25519. */ - ed25519: rule({ initial: moduleNamesEd25519 }), + ed25519: rule({ scan: ed25519NameScan }), processEvents: rule({ fns: [...PROCESS_EVENT_LIB_FNS, "util.styleText"], types: ["procStream"] }), /** scr_url_legacy.c parses and stringifies queries. */ qs: rule({ fns: [...QS_LIB_FNS, ...LEGACY_URL_LIB_FNS] }), @@ -1070,9 +1094,11 @@ for (const name of RUNTIME_FEATURE_NAMES) { function scanRuntimeFeatures(mod: IrModule, stopAt?: RuntimeFeatureName): RuntimeFeatures { const features = noRuntimeFeatures(); + const scans: { name: RuntimeFeatureName; scan: RuleScan }[] = []; for (const name of RUNTIME_FEATURE_NAMES) { - const initial = ruleOf(name).initial; + const { initial, scan } = ruleOf(name); if (initial && initial(mod)) features[name] = true; + if (scan) scans.push({ name, scan: scan() }); } const keepGoing = (): boolean => stopAt === undefined || !features[stopAt]; let tables: TypeTables | undefined; @@ -1083,6 +1109,7 @@ function scanRuntimeFeatures(mod: IrModule, stopAt?: RuntimeFeatureName): Runtim classes: new Map((mod.classes ?? []).map((c) => [c.name, c])), }); const expr = (node: IrExpr): boolean => { + for (let i = 0; i < scans.length; i++) scans[i]!.scan.expr(node); if (node.kind === "libCall") { const fn = node.fn; for (const name of FEATURES_BY_FN.get(fn) ?? []) features[name] = true; @@ -1101,6 +1128,7 @@ function scanRuntimeFeatures(mod: IrModule, stopAt?: RuntimeFeatureName): Runtim return keepGoing(); }; if (keepGoing()) everyModuleNode(mod, { expr, stmt: keepGoing, type }); + for (const { name, scan } of scans) if (scan.result()) features[name] = true; // Inline-cache sites of dynamic-tier functions use the tier's shapes // (scr_js_object.c, with the SCR_JS_TIER build of scr_json.c). if (!features.dynTier && mod.functions.some((fn) => fn.tier === "dynamic")) { diff --git a/packages/compiler/src/ir/traverse.ts b/packages/compiler/src/ir/traverse.ts index 11b20a210..251904351 100644 --- a/packages/compiler/src/ir/traverse.ts +++ b/packages/compiler/src/ir/traverse.ts @@ -109,8 +109,15 @@ export interface IrModuleVisitor extends IrVisitor { * source text, locations, labels, or other metadata. A false predicate stops * the whole walk. Results are never retained across mutable compiler passes. */ export function everyModuleNode(mod: IrModule, visitor: IrModuleVisitor): boolean { - const type = (node: IrType, loc: SrcLoc): boolean => - visitor.type(node, loc) && everyTypeChild(node, (child) => type(child, loc)); + // Every node of one type tree reports the tree's location: one shared + // child callback serves them all. + let typeLoc: SrcLoc; + const typeNode = (node: IrType): boolean => + visitor.type(node, typeLoc) && everyTypeChild(node, typeNode); + const type = (node: IrType, loc: SrcLoc): boolean => { + typeLoc = loc; + return typeNode(node); + }; const expr = (node: IrExpr): boolean => visitor.expr(node) && everyExprChild(node, expr, stmt) && type(node.type, node.loc); const stmt = (node: IrStmt): boolean => visitor.stmt(node) && everyStmtChild(node, expr, stmt); From 18c29009379adef0254dafcc7f51f81c2e4aa9b6 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 14:20:01 -0500 Subject: [PATCH 068/197] Read web ReadableStreams as async iterable stream sources --- .../builtin-apis/streams-and-readline.mdx | 3 +- .../src/frontend/lowering/lower-stream.ts | 282 +++++++++++++++++- .../corpus/7428-web-stream-sources.mjs.json | 6 + .../corpus/7429-web-stream-sources.ts.json | 6 + tests/corpus/7428-web-stream-sources.mjs | 41 +++ tests/corpus/7429-web-stream-sources.ts | 42 +++ .../7428-web-stream-sources.mjs.json | 1 + .../7429-web-stream-sources.ts.json | 1 + 8 files changed, 378 insertions(+), 4 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7428-web-stream-sources.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7429-web-stream-sources.ts.json create mode 100644 tests/corpus/7428-web-stream-sources.mjs create mode 100644 tests/corpus/7429-web-stream-sources.ts create mode 100644 tests/harness/corpus-costs/7428-web-stream-sources.mjs.json create mode 100644 tests/harness/corpus-costs/7429-web-stream-sources.ts.json diff --git a/docs/content/limitations/builtin-apis/streams-and-readline.mdx b/docs/content/limitations/builtin-apis/streams-and-readline.mdx index cfd52e054..eab139773 100644 --- a/docs/content/limitations/builtin-apis/streams-and-readline.mdx +++ b/docs/content/limitations/builtin-apis/streams-and-readline.mdx @@ -25,7 +25,8 @@ It also accepts a lone `Readable` or `Writable` and a `{ readable, writable }` pair, which the Duplex reads and writes through, a promise, whose value it reads, and an async function, which reads what is written and must resolve to `null` or `undefined`. A web stream and a body function that declares the options argument are unsupported in `Duplex.from`. `stream.compose` accepts streams and async generator functions in any position, an async function as the last stage, and an array, a string, or a generator as the first stage; one stage is `Duplex.from` of it. - Web streams as compose stages are unsupported, and the `ERR_INVALID_ARG_VALUE` message for a stream that is no longer readable or writable names the stream's class instead of printing the stream. + A web `ReadableStream` is read as an async iterable by `Readable.from` and as the first stage of `compose` or `pipeline`; a web stream in a later compose stage, and `Readable.fromWeb` and `Readable.toWeb`, are unsupported. + The `ERR_INVALID_ARG_VALUE` message for a stream that is no longer readable or writable names the stream's class instead of printing the stream. `readable.compose(stage, { signal })` is `stream.compose(readable, stage)` with the signal added; the options are an object literal whose `signal` is a variable or a property read. `pipeline` from `node:stream/promises` accepts a trailing `{ signal, end }` object, with or without function stages, and `finished` accepts `readable`, `writable`, `error`, `cleanup`, and `signal` options, with boolean literals for the flags. Stream constructors accept a `signal` option in an options object literal, and `stream.addAbortSignal` accepts any stream. diff --git a/packages/compiler/src/frontend/lowering/lower-stream.ts b/packages/compiler/src/frontend/lowering/lower-stream.ts index 7c0c2b46c..a7f1469a9 100644 --- a/packages/compiler/src/frontend/lowering/lower-stream.ts +++ b/packages/compiler/src/frontend/lowering/lower-stream.ts @@ -1864,6 +1864,42 @@ function lowerFromSource( // SCR_FROM_DUPLEX const duplexFlag = duplex ? 32 : 0; const js = isJsSourceFile(ts.getSourceFileOfNode(call)); + // A web ReadableStream is async iterable: from.js reads its iterator. + // (Duplex.from takes Readable.fromWeb of it instead.) + const webElem = srcNode ? staticReadableStreamElementType(lowerer, srcNode) : null; + if (srcNode && webElem !== null) { + if (duplex) { + lowerer.noLowering( + "Duplex.from over a web ReadableStream", + srcNode, + "Readable.from(stream) reads a web ReadableStream", + ); + } + const source = lowerWebStreamSource(lowerer, srcNode, webElem, loc); + const helpers = source ? fromGenHelpers(lowerer, source.out, loc) : null; + const opts = optsNode ? fromOptions(lowerer, optsNode) : { flags: 0, hwm: null }; + if (!source || !helpers || !opts || !fromGenChunk(lowerer, webElem)) { + lowerer.noLowering( + `Readable.from over a web ReadableStream of '${lowerer.fmt(webElem)}'`, + srcNode, + ); + } + const hwm = opts!.hwm ? lowerer.lowerExprExpecting(opts!.hwm, F64) : numLit(0, loc); + const gen: IrExpr = { + kind: "callValue", + callee: source!.closure, + args: [], + type: source!.out, + loc, + }; + return { + kind: "libCall", + fn: "readable.fromAsyncGen", + args: [gen, ...helpers!, numLit(opts!.flags, loc), hwm], + type, + loc, + }; + } const srcT = srcNode ? lowerer.mapTypeOf(lowerer.typeOf(srcNode)) : null; const typedElem = (t: IrType | null | undefined): "string" | "bytes" | null => t?.kind === "string" ? "string" : t?.kind === "bytes" && t.elem === "u8" ? "bytes" : null; @@ -3120,6 +3156,8 @@ type SyncGenType = IrType & { kind: "generator" }; * array, a string or a synchronous generator object), or null. */ function iterableSourceType(lowerer: Lowerer, node: ts.Expression): IrType | null { if (pipelineStageFunction(lowerer, node) !== null) return null; + const webElem = staticReadableStreamElementType(lowerer, node); + if (webElem !== null) return webElem; let t: IrType | null; try { t = lowerer.mapTypeOf(lowerer.typeOf(node)); @@ -3139,12 +3177,245 @@ function iterableSourceType(lowerer: Lowerer, node: ts.Expression): IrType | nul * through an async-from-sync iterator, and an async generator that * yields the same elements resumes its consumer on the same ticks. Null * when the source has no such lowering. */ +/** A web ReadableStream source, as an async generator closure over its + * reader: each pass awaits reader.read() and yields the chunk, as the + * stream's async iterator does; leaving early cancels the stream (its + * iterator's return()), and the reader's lock is released either way. */ +function lowerWebStreamSource( + lowerer: Lowerer, + node: ts.Expression, + elemT: IrType, + loc: SrcLoc, +): { closure: IrExpr; out: SyncGenType } | null { + const resultT = genResultRecord(elemT, VOID, lowerer.shapes, lowerer.unions); + if (!resultT) return null; + const valueField = lowerer.shapes.get(resultT.shapeId)?.fields.find((f) => f.name === "value"); + if (!valueField) return null; + const out: SyncGenType = { + kind: "generator", + async: true, + yieldT: elemT, + retT: VOID, + nextT: UNDEFINED_T, + }; + const genResultT = genResultRecord(elemT, VOID, lowerer.shapes, lowerer.unions); + if (!genResultT) return null; + const value = lowerer.lowerExprExpecting(node, DYN); + const key = `webStreamSource:${typeKey(elemT)}`; + let maker = lowerer.valueHelpers.get(key); + if (!maker) { + maker = `%stream.webStreamSource.${lowerer.valueHelpers.size}`; + const genName = `${maker}.gen`; + const frame = new LiftedFrame(loc); + const srcId = frame.local("src", DYN, false); + const readerId = frame.local("reader", DYN, false); + const activeId = frame.local("active", BOOL, true); + const promiseT: IrType = { kind: "promise", inner: resultT }; + const pId = frame.local("p", promiseT, false); + const rId = frame.local("r", resultT, false); + const reader = (): IrExpr => varRef(readerId, DYN, loc); + const r = (): IrExpr => varRef(rId, resultT, loc); + const setActive = (on: boolean): IrStmt => ({ + kind: "assign", + localId: activeId, + value: { kind: "boolLit", value: on, type: BOOL, loc }, + loc, + }); + const extracted = extractIteratorValue( + lowerer, + elemT, + valueField.type, + { + kind: "recordGet", + obj: r(), + shapeId: resultT.shapeId, + field: "value", + type: valueField.type, + loc, + }, + loc, + ); + if (!extracted) return null; + const invoke = (method: string, args: IrExpr[], calleeName: string): IrExpr => ({ + kind: "dynInvoke", + recv: reader(), + method, + calleeName, + args, + type: DYN, + loc, + }); + const body: IrStmt[] = [ + { + kind: "varDecl", + localId: readerId, + init: { + kind: "dynInvoke", + recv: varRef(srcId, DYN, loc), + method: "getReader", + calleeName: "ReadableStream async iterator", + args: [], + type: DYN, + loc, + }, + loc, + }, + { + kind: "varDecl", + localId: activeId, + init: { kind: "boolLit", value: false, type: BOOL, loc }, + loc, + }, + { + kind: "tryCatch", + tryBody: [ + { + kind: "while", + cond: { kind: "boolLit", value: true, type: BOOL, loc }, + body: [ + setActive(false), + { + kind: "varDecl", + localId: pId, + init: { + kind: "libCall", + fn: "fetch.readerRead", + args: [reader()], + type: promiseT, + loc, + }, + loc, + }, + { + kind: "varDecl", + localId: rId, + init: { kind: "awaitExpr", value: varRef(pId, promiseT, loc), type: resultT, loc }, + loc, + }, + { + kind: "if", + cond: { + kind: "recordGet", + obj: r(), + shapeId: resultT.shapeId, + field: "done", + type: BOOL, + loc, + }, + then: [{ kind: "break", loc }], + else_: null, + loc, + }, + setActive(true), + { + kind: "exprStmt", + expr: { kind: "yieldExpr", value: extracted, type: VOID, loc }, + loc, + }, + ], + loc, + }, + ], + catchBody: null, + catchLocalId: null, + finallyBody: [ + { + kind: "tryCatch", + tryBody: [ + { + kind: "if", + cond: varRef(activeId, BOOL, loc), + then: [ + { + kind: "exprStmt", + expr: { + kind: "libCall", + fn: "async.awaitDyn", + args: [ + invoke( + "cancel", + [dynUndefinedExpr(loc)], + "ReadableStream async iterator return", + ), + ], + type: DYN, + loc, + }, + loc, + }, + ], + else_: null, + loc, + }, + ], + catchBody: null, + catchLocalId: null, + finallyBody: [ + { + kind: "exprStmt", + expr: invoke("releaseLock", [], "ReadableStream async iterator releaseLock"), + loc, + }, + ], + loc, + }, + ], + loc, + }, + { kind: "return", value: null, loc }, + ]; + const srcLocal = frame.locals.find((l) => l.id === srcId)!; + srcLocal.boxed = true; + lowerer.valueHelpers.set(key, maker); + lowerer.liftedFns.push({ + name: genName, + params: [], + returnType: VOID, + async: true, + captures: [{ localId: srcId, name: "src", type: DYN }], + locals: frame.locals, + body, + generator: { yieldT: elemT, nextT: UNDEFINED_T, resultType: genResultT }, + loc, + }); + const makerSrc: IrLocal = { id: "src.0", name: "src", type: DYN, mutable: false, boxed: true }; + lowerer.liftedFns.push({ + name: maker, + params: [{ localId: makerSrc.id, name: makerSrc.name, type: DYN }], + returnType: funcOf([], out), + locals: [makerSrc], + body: [ + { + kind: "return", + value: { + kind: "closure", + fnName: genName, + captures: [makerSrc.id], + type: funcOf([], out), + loc, + }, + loc, + }, + ], + loc, + }); + } + return { + closure: { kind: "call", callee: maker, args: [value], type: funcOf([], out), loc }, + out, + }; +} + function lowerIterableSource( lowerer: Lowerer, node: ts.Expression, fn: StageFunction | null, loc: SrcLoc, ): { closure: IrExpr; out: SyncGenType } | null { + if (fn === null) { + const webElem = staticReadableStreamElementType(lowerer, node); + if (webElem !== null) return lowerWebStreamSource(lowerer, node, webElem, loc); + } let value: IrExpr; if (fn !== null) { if (fn.parameters.nodes.length > 0) { @@ -4004,15 +4275,20 @@ function lowerStreamCompose(lowerer: Lowerer, args: readonly ts.Expression[], lo loc, ); } - for (const a of args) { + // A web ReadableStream first stage reads as an async iterable (the + // pipeline's source); elsewhere it would be Duplex.fromWeb of it. + for (let i = args.length === 1 ? 0 : 1; i < args.length; i++) { + const a = args[i]!; // (typeOf on a stage function's name would report what collecting its // declared signature deferred) if (pipelineStageFunction(lowerer, a) !== null) continue; if (staticReadableStreamElementType(lowerer, a) !== null) { lowerer.noLowering( - "compose with a web ReadableStream", + i === 0 + ? "Duplex.from over a web ReadableStream" + : "compose with a web ReadableStream after the first stage", a, - "web streams do not convert to Node streams yet; compose Node streams, iterables or async generator functions", + "a web ReadableStream is supported as the first stage of compose or pipeline, and in Readable.from", ); } } diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7428-web-stream-sources.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7428-web-stream-sources.mjs.json new file mode 100644 index 000000000..e17735658 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7428-web-stream-sources.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7428-web-stream-sources.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7429-web-stream-sources.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7429-web-stream-sources.ts.json new file mode 100644 index 000000000..8707c2f2b --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7429-web-stream-sources.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7429-web-stream-sources.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7428-web-stream-sources.mjs b/tests/corpus/7428-web-stream-sources.mjs new file mode 100644 index 000000000..72d98e5f8 --- /dev/null +++ b/tests/corpus/7428-web-stream-sources.mjs @@ -0,0 +1,41 @@ +// A web ReadableStream as an async iterable source: Readable.from reads +// its chunks through the stream's iterator (pull-based and start-based +// sources, Uint8Array chunks in byte mode, an error from the controller, +// and an early destroy that cancels the stream), and it heads compose() +// and pipeline() like any async iterable. +import { Readable, Writable, compose, pipeline } from "node:stream"; +const log = []; +let n = 0; +const pulled = new ReadableStream({ + pull(c) { n++; log.push(`pull ${n}`); if (n <= 3) c.enqueue(`p${n}`); else c.close(); }, + cancel(reason) { log.push(`cancel ${reason}`); }, +}); +for await (const c of Readable.from(pulled)) log.push(`got ${c}`); +const bytes = new ReadableStream({ start(c) { c.enqueue(new TextEncoder().encode("hi")); c.enqueue(new TextEncoder().encode("!")); c.close(); } }); +const br = Readable.from(bytes, { objectMode: false }); +br.on("data", (b) => log.push(`bytes ${Buffer.isBuffer(b)} ${b.toString()}`)); +await new Promise((res) => br.on("end", res)); +const failing = new ReadableStream({ start(c) { c.enqueue("ok"); c.error(new Error("web failed")); } }); +const fr = Readable.from(failing); +fr.on("data", (c) => log.push(`failing data ${c}`)); +fr.on("error", (e) => log.push(`failing error ${e.message}`)); +await new Promise((res) => fr.on("close", res)); +let m = 0; +const endless = new ReadableStream({ + pull(c) { c.enqueue(`e${++m}`); }, + cancel(reason) { log.push(`endless cancel ${reason === undefined ? "undefined" : reason}`); }, +}); +const er = Readable.from(endless); +er.once("data", (c) => { log.push(`endless ${c}`); er.destroy(); }); +await new Promise((res) => er.on("close", res)); +await new Promise((res) => setTimeout(res, 5)); +const head = compose(new ReadableStream({ start(c) { c.enqueue("c1"); c.enqueue("c2"); c.close(); } }), async function* (s) { for await (const c of s) yield `${c}!`; }); +log.push(`compose ${(await head.toArray()).join(",")}`); +await new Promise((resolve) => + pipeline( + new ReadableStream({ start(c) { c.enqueue("q1"); c.enqueue("q2"); c.close(); } }), + new Writable({ objectMode: true, write(c, e, cb) { log.push(`pipeline ${c}`); cb(); } }), + (err) => { log.push(`pipeline callback ${err}`); resolve(undefined); }, + ), +); +console.log(log.join("\n")); diff --git a/tests/corpus/7429-web-stream-sources.ts b/tests/corpus/7429-web-stream-sources.ts new file mode 100644 index 000000000..8243b6873 --- /dev/null +++ b/tests/corpus/7429-web-stream-sources.ts @@ -0,0 +1,42 @@ +// Web ReadableStreams as sources in typed code: Readable.from over a +// ReadableStream, interleaved with nextTick callbacks, and a +// ReadableStream heading compose() into an async generator. +import { Readable, compose } from "node:stream"; + +const log: string[] = []; + +function words(): ReadableStream { + return new ReadableStream({ + start(controller): void { + controller.enqueue("typed"); + controller.enqueue("web"); + controller.close(); + }, + }); +} + +async function main(): Promise { + let n = 0; + const tick = (): void => { + log.push(`tick ${n}`); + if (++n < 4) process.nextTick(tick); + }; + process.nextTick(tick); + const r = Readable.from(words()); + log.push(`object mode ${r.readableObjectMode}`); + for await (const c of r) log.push(`word ${String(c)}`); + const bytes = new ReadableStream({ + start(controller): void { + controller.enqueue(new TextEncoder().encode("ab")); + controller.enqueue(new TextEncoder().encode("cd")); + controller.close(); + }, + }); + const lengths = compose(bytes, async function* (source: AsyncIterable): AsyncGenerator { + for await (const chunk of source) yield `${chunk.length}`; + }); + log.push(`lengths ${(await lengths.toArray()).join(",")}`); + for (const line of log) console.log(line); +} + +void main(); diff --git a/tests/harness/corpus-costs/7428-web-stream-sources.mjs.json b/tests/harness/corpus-costs/7428-web-stream-sources.mjs.json new file mode 100644 index 000000000..e95faaa64 --- /dev/null +++ b/tests/harness/corpus-costs/7428-web-stream-sources.mjs.json @@ -0,0 +1 @@ +{ "plain": 1829, "san": 2414 } diff --git a/tests/harness/corpus-costs/7429-web-stream-sources.ts.json b/tests/harness/corpus-costs/7429-web-stream-sources.ts.json new file mode 100644 index 000000000..6b14e36cc --- /dev/null +++ b/tests/harness/corpus-costs/7429-web-stream-sources.ts.json @@ -0,0 +1 @@ +{ "plain": 1250, "san": 3013 } From f5a3faf18d642a52ac2bf0464abfac117903593d Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 14:25:55 -0500 Subject: [PATCH 069/197] Bound the compiler's cyclic garbage to twice its live heap The cycle collector widens the growth fraction that schedules full passes up to 16 times the heap left by the last pass while passes find little. Lowering a large program leaves hundreds of megabytes of cyclic garbage, and whether a scheduled pass reclaims it before the build's peak depends on where the growth rule fires: small changes to the compiler moved the peak of a tsc-ts build between 1.66 and 2.05 GB. The native compiler now sets the growth cap to 2 at startup, as SCR_CYCLE_GROWTH_CAP=2 does, so a full pass runs once the heap has grown to three times its size after the last one. scr_cyc_set_growth_cap sets the cap for a program from its own code; the environment variable still takes precedence, and child processes are not affected. --- packages/compiler/native/host.c | 15 +++++++++++++++ packages/compiler/native/host.ffi.json | 3 ++- packages/compiler/src/native/cli.ts | 2 ++ packages/runtime/src/scr_cycle.c | 25 ++++++++++++++++++++----- packages/runtime/src/scr_rc.h | 4 ++++ packages/runtime/test/test_cycle.c | 22 ++++++++++++++++++++++ 6 files changed, 65 insertions(+), 6 deletions(-) diff --git a/packages/compiler/native/host.c b/packages/compiler/native/host.c index 6aa1b2a28..457774f81 100644 --- a/packages/compiler/native/host.c +++ b/packages/compiler/native/host.c @@ -56,3 +56,18 @@ void scr_collect_cycles_phase(void); void scriptc_native_collect_cycles(void) { scr_collect_cycles_phase(); } + +/* The runtime's growth cap for full cycle passes (scr_rc.h). */ +void scr_cyc_set_growth_cap(long cap); + +/* Bounds the cyclic garbage a build can hold to twice its live heap. + * Lowering a large program leaves hundreds of megabytes of cyclic garbage, + * and whether a scheduled full pass reclaims it before the build's peak + * depends on where the growth rule fires: with the default cap of 16, small + * changes to the compiler moved the peak of one build by 390 MB. With a cap + * of 2 a full pass runs once the heap has grown to three times its size + * after the last one; a cap of 1 bounds the garbage further but runs more + * passes over the live heap. SCR_CYCLE_GROWTH_CAP still overrides it. */ +void scriptc_native_bound_cycle_growth(void) { + scr_cyc_set_growth_cap(2); +} diff --git a/packages/compiler/native/host.ffi.json b/packages/compiler/native/host.ffi.json index d7e0e92aa..3d15a1e0e 100644 --- a/packages/compiler/native/host.ffi.json +++ b/packages/compiler/native/host.ffi.json @@ -2,6 +2,7 @@ "ffi_format": 6, "functions": [ { "name": "compilerNativePrivateDirectory", "symbol": "scriptc_native_private_directory", "params": ["string", "bool"], "returns": "bool" }, - { "name": "compilerNativeCollectCycles", "symbol": "scriptc_native_collect_cycles", "params": [], "returns": "void" } + { "name": "compilerNativeCollectCycles", "symbol": "scriptc_native_collect_cycles", "params": [], "returns": "void" }, + { "name": "compilerNativeBoundCycleGrowth", "symbol": "scriptc_native_bound_cycle_growth", "params": [], "returns": "void" } ] } diff --git a/packages/compiler/src/native/cli.ts b/packages/compiler/src/native/cli.ts index e2c3a895b..d073df5f4 100644 --- a/packages/compiler/src/native/cli.ts +++ b/packages/compiler/src/native/cli.ts @@ -23,8 +23,10 @@ import { runCompilerTask } from "./task.js"; declare function compilerNativePrivateDirectory(path: string, harden: boolean): boolean; declare function compilerNativeCollectCycles(): void; +declare function compilerNativeBoundCycleGrowth(): void; async function main(): Promise { + compilerNativeBoundCycleGrowth(); setFrontendDisposedHook(() => compilerNativeCollectCycles()); const manifestPath = process.env["SCRIPTC_TOOLCHAIN"] ?? process.execPath + ".json"; let compiler: NativeCompiler | null = null; diff --git a/packages/runtime/src/scr_cycle.c b/packages/runtime/src/scr_cycle.c index 5e5029e14..844c47001 100644 --- a/packages/runtime/src/scr_cycle.c +++ b/packages/runtime/src/scr_cycle.c @@ -529,7 +529,8 @@ static SCR_TL size_t scr_cyc_live_after[SCR_CYC_NGENS]; * large program, say) those walks dominate collection time. The cap trades * that walking for memory only when passes find nothing: while they find * garbage, the yield keeps the fraction narrow. SCR_CYCLE_GROWTH_CAP lowers - * the cap for memory-sensitive programs: a power of two from 1 to 16, with + * the cap for memory-sensitive programs, as scr_cyc_set_growth_cap does for + * a program that sets its own: a power of two from 1 to 16, with * other positive values rounded down to a power of two and larger ones read * as 16. At 1 the heap may at most double past its size after the last pass * at a level before the next. */ @@ -539,20 +540,34 @@ static SCR_TL unsigned char scr_cyc_growth_shift[SCR_CYC_NGENS]; static SCR_TL unsigned char scr_cyc_growth_streak[SCR_CYC_NGENS]; #endif +/* The growth cap a program sets for itself (scr_cyc_set_growth_cap), 0 for + * the default; SCR_CYCLE_GROWTH_CAP takes precedence over it. */ +static long scr_cyc_growth_cap_set = 0; +static SCR_TL unsigned char scr_cyc_growth_max_shift_cached = 0; + /* Shift of the configured growth cap: 2 + log2(cap), since the base * fraction is 1/4. */ static unsigned scr_cyc_growth_max_shift(void) { - static SCR_TL unsigned char cached = 0; - if (cached == 0) { + if (scr_cyc_growth_max_shift_cached == 0) { const char *env = scr_getenv("SCR_CYCLE_GROWTH_CAP"); long v = env ? strtol(env, NULL, 10) : 0; + if (v <= 0) v = scr_cyc_growth_cap_set; unsigned shift = SCR_CYC_GROWTH_MAX_SHIFT; if (v > 0) for (shift = 2; shift < SCR_CYC_GROWTH_MAX_SHIFT && (2L << (shift - 2)) <= v;) shift++; - cached = (unsigned char)shift; + scr_cyc_growth_max_shift_cached = (unsigned char)shift; } - return cached; + return scr_cyc_growth_max_shift_cached; +} + +void scr_cyc_set_growth_cap(long cap) { + scr_cyc_growth_cap_set = cap; + scr_cyc_growth_max_shift_cached = 0; + /* Fractions already widened past the new cap start at it. */ + unsigned max = scr_cyc_growth_max_shift(); + for (unsigned gen = 0; gen < SCR_CYC_NGENS; gen++) + if (scr_cyc_growth_shift[gen] > max) scr_cyc_growth_shift[gen] = (unsigned char)max; } /* Doublings of each generation's backlog threshold earned by consecutive diff --git a/packages/runtime/src/scr_rc.h b/packages/runtime/src/scr_rc.h index 7a921b451..18df3a237 100644 --- a/packages/runtime/src/scr_rc.h +++ b/packages/runtime/src/scr_rc.h @@ -670,6 +670,10 @@ static inline void scr_cyc_dead(void *obj) { * live heap — do not put it on a per-turn path. */ void scr_collect_cycles(void); void scr_collect_cycles_phase(void); +/* Sets the growth cap SCR_CYCLE_GROWTH_CAP sets (scr_cycle.c), for a program + * that knows its own memory profile; the environment variable takes + * precedence. Call it before the program allocates much. */ +void scr_cyc_set_growth_cap(long cap); void scr_cyc_context_cleanup(void); /* One pass on the normal generational schedule, for callers that reach a diff --git a/packages/runtime/test/test_cycle.c b/packages/runtime/test/test_cycle.c index 4e00b852e..160083d6b 100644 --- a/packages/runtime/test/test_cycle.c +++ b/packages/runtime/test/test_cycle.c @@ -536,6 +536,27 @@ static void check_mixed_dead_structure_narrows(void) { check(live <= narrowed, "a pass that freed a dead mature and old structure did not narrow the growth fraction"); } +/* scr_cyc_set_growth_cap sets the cap SCR_CYCLE_GROWTH_CAP sets, and a + * fraction already widened past it starts at it: after passes that found + * nothing widened the old level's fraction to the configured cap, a cap of + * 1 has the next dead cycle go once the heap has doubled past the floor. + * Restores the configured cap. */ +static void check_program_growth_cap(void) { + enum { FLOOR = 4096 }; + if (getenv("SCR_CYCLE_GROWTH_CAP") != NULL) return; /* it takes precedence */ + size_t slack = 2 * configured_nursery_threshold() + 2; + size_t live = 0; + for (int i = 0; i < 4; i++) live = grow_dead_cycle_until_full_pass(0, false); + check(live > FLOOR + FLOOR * configured_growth_cap(), + "passes that found nothing did not widen the growth fraction to the cap"); + scr_cyc_set_growth_cap(1); + for (int i = 0; i < 4; i++) { + live = grow_dead_cycle_until_full_pass(0, false); + check(live <= FLOOR + FLOOR + slack, "a program's growth cap of 1 did not bound the heap"); + } + scr_cyc_set_growth_cap(0); +} + /* The phase sweep (scr_collect_cycles_phase) is for a caller that knows a * phase's data has just died: it sweeps, and the growth schedule starts * over. After passes that find nothing have widened the old level's @@ -1500,6 +1521,7 @@ int main(void) { check_large_old_buffer_with_nursery_candidates(); check_unjudged_backlog_follows_growth(); check_acyclic_objects(); + check_program_growth_cap(); printf("cycle collection checks passed: threshold=%zu growth_cap=%zu\n", configured_nursery_threshold(), configured_growth_cap()); return 0; From f4365b7eea5dc2d7f0cb2901d0be7a1bd3bb08e7 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 15:02:41 -0500 Subject: [PATCH 070/197] Cache each constant method name's id in a slot A checked-dynamic method call looked its name up in the method table on every call: a strlen, a hash and a byte comparison of the name. The compiler now gives each method-name constant an i32 slot that starts at -1 and passes it with the call (scr_dyn_invoke_prepared_id, and the Buffer method entries, which forward it). The first call stores the name's id in the slot, and later calls read it. Ids stay the runtime's own, so the slot holds whatever this runtime's table says. Names are still matched by content on the first call, and the slot is written with a relaxed atomic store, since every thread writes the same id. The runtime ABI is now 28. --- packages/cli/test/native-link-info.test.ts | 4 +- .../compiler/src/backend/llvm/emit-module.ts | 2 + packages/compiler/src/backend/llvm/emitter.ts | 14 +++++++ .../compiler/src/backend/llvm/expr-dynamic.ts | 8 ++-- packages/compiler/src/backend/llvm/js-ic.ts | 2 +- packages/compiler/src/backend/runtime-abi.ts | 4 +- .../src/backend/runtime-pack-targets.test.ts | 2 +- .../compiler/src/backend/runtime-pack.test.ts | 2 +- .../test/native-codegen-integration.test.ts | 4 +- .../runtime-darwin-arm64/scripts/build.mjs | 2 +- .../runtime-pack-common/scripts/build.mjs | 2 +- packages/runtime/src/scr_bytes_dyn.c | 42 +++++++++++-------- packages/runtime/src/scr_console.c | 2 +- packages/runtime/src/scr_core.h | 2 +- packages/runtime/src/scr_dyn.h | 22 ++++++---- packages/runtime/src/scr_dyn_invoke.c | 23 +++++++--- .../self-hosting-native-toolchain.test.ts | 2 +- 17 files changed, 90 insertions(+), 49 deletions(-) diff --git a/packages/cli/test/native-link-info.test.ts b/packages/cli/test/native-link-info.test.ts index a7a5e718a..a1b673630 100644 --- a/packages/cli/test/native-link-info.test.ts +++ b/packages/cli/test/native-link-info.test.ts @@ -72,7 +72,7 @@ describe.runIf(supported)("macOS arm64 native link info", () => { minimum_os: "14.0", }, program: { object, entry_symbol: "main" }, - runtime_abi: { version: 27, marker: "scr_runtime_abi_v27" }, + runtime_abi: { version: 28, marker: "scr_runtime_abi_v28" }, runtime_pack: { kind: "precompiled", package: "@scriptc/runtime-darwin-arm64", @@ -165,6 +165,6 @@ describe.runIf(supported)("macOS arm64 native link info", () => { () => null, (failure: { stderr?: string }) => failure, ); - expect(error?.stderr).toContain("scr_runtime_abi_v27"); + expect(error?.stderr).toContain("scr_runtime_abi_v28"); }); }); diff --git a/packages/compiler/src/backend/llvm/emit-module.ts b/packages/compiler/src/backend/llvm/emit-module.ts index 3d9d41072..47d92e3bd 100644 --- a/packages/compiler/src/backend/llvm/emit-module.ts +++ b/packages/compiler/src/backend/llvm/emit-module.ts @@ -727,6 +727,8 @@ export function emitModuleParts(emitter: LlEmitter): string[] { out.push(`@${sym} = internal constant { ptr, ptr, ptr, ptr, ptr, ptr, ptr } { ${body} }`); } if (emitter.mapOpsConsts.size > 0) out.push(``); + for (const sym of emitter.methodIdSlots.values()) out.push(`@${sym} = internal global i32 -1`); + if (emitter.methodIdSlots.size > 0) out.push(``); for (const line of ffiCallbacks.globals) out.push(line); if (ffiCallbacks.globals.length > 0) out.push(``); diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index bfde59a30..ee8e69e94 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -782,6 +782,10 @@ export class LlEmitter { readonly cstrs = new Map(); /** Interned ScrMapOps descriptors (scr_map.h): initializer → symbol. */ readonly mapOpsConsts = new Map(); + /** Method-id slots of checked-dynamic method calls + * (scr_dyn_invoke_prepared_id): method name → symbol of an i32 the + * runtime fills with the name's id on the first call. */ + readonly methodIdSlots = new Map(); /** Type-directed walker functions (JSON serializers, the indent * rewriter, union ToString/join) — interned per typeKey/unionId, defs * flushed with the shape helpers. */ @@ -2400,6 +2404,16 @@ export class LlEmitter { return `@${c.sym}`; } + /** The method-id slot for a constant method name (methodIdSlots). */ + methodIdSlot(name: string): string { + let sym = this.methodIdSlots.get(name); + if (!sym) { + sym = this.contentSymbol("sc_mid_", name); + this.methodIdSlots.set(name, sym); + } + return `@${sym}`; + } + needBadTag(): void { this.needsBadTag = true; } diff --git a/packages/compiler/src/backend/llvm/expr-dynamic.ts b/packages/compiler/src/backend/llvm/expr-dynamic.ts index 58bb90569..e3d0dc8f2 100644 --- a/packages/compiler/src/backend/llvm/expr-dynamic.ts +++ b/packages/compiler/src/backend/llvm/expr-dynamic.ts @@ -250,8 +250,10 @@ export function emitDynamicExpr( } // Buffer.prototype method names take their scr_bytes_dyn.c entry, which // runs them on Buffer receivers and dispatches any other receiver. - const invoke = DYN_BUFFER_METHODS.get(e.method) ?? "scr_dyn_invoke_prepared"; - host.declare(`declare ptr @${invoke}(ptr, ptr, ptr, ptr, ${host.sizeType}, ptr)`); + // Each call passes its method name's id slot, so only the first call + // looks the name up. + const invoke = DYN_BUFFER_METHODS.get(e.method) ?? "scr_dyn_invoke_prepared_id"; + host.declare(`declare ptr @${invoke}(ptr, ptr, ptr, ptr, ptr, ${host.sizeType}, ptr)`); let name = host.cstr(e.calleeName); if (e.calleeNameValue !== undefined) { const value = emitBorrowedInput(host, e.calleeNameValue); @@ -260,7 +262,7 @@ export function emitDynamicExpr( } const t = B.tmp(); B.line( - `${t} = call ptr @${invoke}(ptr ${recv.name}, ptr ${prepared}, ptr ${host.cstr(e.method)}, ptr ${argsPtr}, ${host.sizeType} ${args.length}, ptr ${name})`, + `${t} = call ptr @${invoke}(ptr ${recv.name}, ptr ${prepared}, ptr ${host.cstr(e.method)}, ptr ${host.methodIdSlot(e.method)}, ptr ${argsPtr}, ${host.sizeType} ${args.length}, ptr ${name})`, ); const out = host.own({ name: t, type: e.type }); host.emitPendingCheck(); diff --git a/packages/compiler/src/backend/llvm/js-ic.ts b/packages/compiler/src/backend/llvm/js-ic.ts index 45c18b803..7516673cd 100644 --- a/packages/compiler/src/backend/llvm/js-ic.ts +++ b/packages/compiler/src/backend/llvm/js-ic.ts @@ -732,7 +732,7 @@ function emitKeyed(host: Host, e: ExprOf<"call">, callee: IrFunction): LlValue { /** dynInvoke's method read on an ordinary object through the site's * cache, else scr_dyn_prepare_method. Returns the owned method (or NULL, - * which scr_dyn_invoke_prepared answers with the full dispatch). */ + * which scr_dyn_invoke_prepared_id answers with the full dispatch). */ export function emitJsIcMethod(host: Host, e: ExprOf<"dynInvoke">, receiver: string): string { const B = host.B; const method = e.method; diff --git a/packages/compiler/src/backend/runtime-abi.ts b/packages/compiler/src/backend/runtime-abi.ts index 3a41d9176..a616dbec2 100644 --- a/packages/compiler/src/backend/runtime-abi.ts +++ b/packages/compiler/src/backend/runtime-abi.ts @@ -2,8 +2,8 @@ * of the link contract: a program object keeps this symbol undefined and the * matching runtime defines it, so incompatible manual links fail before the * program can start. */ -export const RUNTIME_ABI_VERSION = 27 as const; -export const RUNTIME_ABI_MARKER = "scr_runtime_abi_v27" as const; +export const RUNTIME_ABI_VERSION = 28 as const; +export const RUNTIME_ABI_MARKER = "scr_runtime_abi_v28" as const; /** Object consumption is useful today, but the complete scr_* surface may * still change before 1.0. The versioned marker prevents accidental mixing; diff --git a/packages/compiler/src/backend/runtime-pack-targets.test.ts b/packages/compiler/src/backend/runtime-pack-targets.test.ts index af61e5229..c2536ede8 100644 --- a/packages/compiler/src/backend/runtime-pack-targets.test.ts +++ b/packages/compiler/src/backend/runtime-pack-targets.test.ts @@ -52,7 +52,7 @@ async function fixture(target: NativeTargetSpec, workers = false) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 27, marker: "scr_runtime_abi_v27" }, + runtime_abi: { version: 28, marker: "scr_runtime_abi_v28" }, compiler: { command: "fixture", identity: "fixture", target: target.llvmTriple }, macros: { executable: [], excluded: ["SCR_LIB"], sanitizer: "external-toolchain-required" }, flavors: { diff --git a/packages/compiler/src/backend/runtime-pack.test.ts b/packages/compiler/src/backend/runtime-pack.test.ts index d09509911..33b1ca38e 100644 --- a/packages/compiler/src/backend/runtime-pack.test.ts +++ b/packages/compiler/src/backend/runtime-pack.test.ts @@ -127,7 +127,7 @@ async function fixture(target: NativeTargetSpec = MACOS_ARM64_TARGET) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 27, marker: "scr_runtime_abi_v27" }, + runtime_abi: { version: 28, marker: "scr_runtime_abi_v28" }, compiler: { command: "clang", identity: "fixture clang", diff --git a/packages/compiler/test/native-codegen-integration.test.ts b/packages/compiler/test/native-codegen-integration.test.ts index 35a747d60..71b9570b4 100644 --- a/packages/compiler/test/native-codegen-integration.test.ts +++ b/packages/compiler/test/native-codegen-integration.test.ts @@ -378,7 +378,7 @@ done: expect(await symbols(helperObject, ["-u"])).toEqual(await symbols(clangObject, ["-u"])); expect(await symbols(helperObject, ["-gU"])).toEqual(await symbols(clangObject, ["-gU"])); expect(await symbols(helperObject, ["-gU"])).toEqual(["0000000000000000 T _main"]); - expect(await symbols(helperObject, ["-u"])).toContain("_scr_runtime_abi_v27"); + expect(await symbols(helperObject, ["-u"])).toContain("_scr_runtime_abi_v28"); const clangExe = join(dir, "clang-program"); const linkDriver = join(dir, "clang-driver.c"); @@ -712,7 +712,7 @@ done: .split("\n"); expect(undefinedSymbols).toContain("_sf_scale"); expect(undefinedSymbols).toContain("_sf_callback_mix"); - expect(undefinedSymbols).toContain("_scr_runtime_abi_v27"); + expect(undefinedSymbols).toContain("_scr_runtime_abi_v28"); }); test("helper and clang object paths retain the same outbound FFI C ABI", async () => { diff --git a/packages/runtime-darwin-arm64/scripts/build.mjs b/packages/runtime-darwin-arm64/scripts/build.mjs index 631750cdf..3d22b2360 100644 --- a/packages/runtime-darwin-arm64/scripts/build.mjs +++ b/packages/runtime-darwin-arm64/scripts/build.mjs @@ -340,7 +340,7 @@ async function build() { package: packageManifest.name, version: packageManifest.version, target: RUNTIME_PACK_MATRIX.target, - runtime_abi: { version: 27, marker: "scr_runtime_abi_v27" }, + runtime_abi: { version: 28, marker: "scr_runtime_abi_v28" }, compiler: { command: compiler, identity: compilerVersion, diff --git a/packages/runtime-pack-common/scripts/build.mjs b/packages/runtime-pack-common/scripts/build.mjs index 0e81033ad..909412bc7 100644 --- a/packages/runtime-pack-common/scripts/build.mjs +++ b/packages/runtime-pack-common/scripts/build.mjs @@ -437,7 +437,7 @@ async function build() { package: packageManifest.name, version: packageManifest.version, target: matrix.target, - runtime_abi: { version: 27, marker: "scr_runtime_abi_v27" }, + runtime_abi: { version: 28, marker: "scr_runtime_abi_v28" }, compiler: { command: compiler, identity: compilerVersion, diff --git a/packages/runtime/src/scr_bytes_dyn.c b/packages/runtime/src/scr_bytes_dyn.c index 280aa423c..c492d43b1 100644 --- a/packages/runtime/src/scr_bytes_dyn.c +++ b/packages/runtime/src/scr_bytes_dyn.c @@ -509,8 +509,9 @@ static bool scr_bytes_dyn_buffer(const ScrDyn *recv) { /* One entry per method family, so a program links only the families it * names: each runs its methods on a Buffer receiver and gives any other * receiver the ordinary dispatch. */ -ScrDyn *scr_dyn_invoke_buffer_num(ScrDyn *recv, ScrDyn *callee, const char *method, ScrDyn *const *args, - size_t argc, const char *what) { +ScrDyn *scr_dyn_invoke_buffer_num(ScrDyn *recv, ScrDyn *callee, const char *method, + int32_t *mid_slot, ScrDyn *const *args, size_t argc, + const char *what) { if (scr_bytes_dyn_buffer(recv)) { bool read = !strncmp(method, "read", 4); const char *rest = method + (read ? 4 : 5); @@ -518,40 +519,45 @@ ScrDyn *scr_dyn_invoke_buffer_num(ScrDyn *recv, ScrDyn *callee, const char *meth if (!strcmp(rest, scr_bytes_dyn_names[at].name)) return scr_bytes_dyn_num_method(recv->v.bytes, read, at, args, argc); } } - return scr_dyn_invoke_prepared(recv, callee, method, args, argc, what); + return scr_dyn_invoke_prepared_id(recv, callee, method, mid_slot, args, argc, what); } -ScrDyn *scr_dyn_invoke_buffer_write(ScrDyn *recv, ScrDyn *callee, const char *method, ScrDyn *const *args, - size_t argc, const char *what) { +ScrDyn *scr_dyn_invoke_buffer_write(ScrDyn *recv, ScrDyn *callee, const char *method, + int32_t *mid_slot, ScrDyn *const *args, size_t argc, + const char *what) { if (scr_bytes_dyn_buffer(recv)) return scr_bytes_dyn_write(recv->v.bytes, args, argc); - return scr_dyn_invoke_prepared(recv, callee, method, args, argc, what); + return scr_dyn_invoke_prepared_id(recv, callee, method, mid_slot, args, argc, what); } -ScrDyn *scr_dyn_invoke_buffer_fill(ScrDyn *recv, ScrDyn *callee, const char *method, ScrDyn *const *args, - size_t argc, const char *what) { +ScrDyn *scr_dyn_invoke_buffer_fill(ScrDyn *recv, ScrDyn *callee, const char *method, + int32_t *mid_slot, ScrDyn *const *args, size_t argc, + const char *what) { if (scr_bytes_dyn_buffer(recv)) return scr_bytes_dyn_fill(recv, args, argc); - return scr_dyn_invoke_prepared(recv, callee, method, args, argc, what); + return scr_dyn_invoke_prepared_id(recv, callee, method, mid_slot, args, argc, what); } -ScrDyn *scr_dyn_invoke_buffer_to_string(ScrDyn *recv, ScrDyn *callee, const char *method, ScrDyn *const *args, - size_t argc, const char *what) { +ScrDyn *scr_dyn_invoke_buffer_to_string(ScrDyn *recv, ScrDyn *callee, const char *method, + int32_t *mid_slot, ScrDyn *const *args, size_t argc, + const char *what) { if (scr_bytes_dyn_buffer(recv)) return scr_bytes_dyn_to_string(recv->v.bytes, args, argc); - return scr_dyn_invoke_prepared(recv, callee, method, args, argc, what); + return scr_dyn_invoke_prepared_id(recv, callee, method, mid_slot, args, argc, what); } /* indexOf, lastIndexOf and includes. */ -ScrDyn *scr_dyn_invoke_buffer_search(ScrDyn *recv, ScrDyn *callee, const char *method, ScrDyn *const *args, - size_t argc, const char *what) { +ScrDyn *scr_dyn_invoke_buffer_search(ScrDyn *recv, ScrDyn *callee, const char *method, + int32_t *mid_slot, ScrDyn *const *args, size_t argc, + const char *what) { if (scr_bytes_dyn_buffer(recv)) { bool includes = !strcmp(method, "includes"); return scr_bytes_dyn_index_of(recv->v.bytes, args, argc, strcmp(method, "lastIndexOf") != 0, includes); } - return scr_dyn_invoke_prepared(recv, callee, method, args, argc, what); + return scr_dyn_invoke_prepared_id(recv, callee, method, mid_slot, args, argc, what); } /* toJSON, equals, compare, copy, swap16/32/64, and the refused members. */ -ScrDyn *scr_dyn_invoke_buffer(ScrDyn *recv, ScrDyn *callee, const char *method, ScrDyn *const *args, - size_t argc, const char *what) { +ScrDyn *scr_dyn_invoke_buffer(ScrDyn *recv, ScrDyn *callee, const char *method, + int32_t *mid_slot, ScrDyn *const *args, size_t argc, + const char *what) { if (scr_bytes_dyn_buffer(recv)) { ScrBytes *b = recv->v.bytes; if (!strcmp(method, "toJSON")) return scr_bytes_dyn_to_json(b); @@ -584,5 +590,5 @@ ScrDyn *scr_dyn_invoke_buffer(ScrDyn *recv, ScrDyn *callee, const char *method, return NULL; } } - return scr_dyn_invoke_prepared(recv, callee, method, args, argc, what); + return scr_dyn_invoke_prepared_id(recv, callee, method, mid_slot, args, argc, what); } diff --git a/packages/runtime/src/scr_console.c b/packages/runtime/src/scr_console.c index a3ac43cc0..9bd5abf8c 100644 --- a/packages/runtime/src/scr_console.c +++ b/packages/runtime/src/scr_console.c @@ -148,7 +148,7 @@ static void scr_enable_utf8_console_output(void) { } #endif -void scr_runtime_abi_v27(void) {} +void scr_runtime_abi_v28(void) {} void scr_init(void) { #ifdef SCR_WORKERS diff --git a/packages/runtime/src/scr_core.h b/packages/runtime/src/scr_core.h index fea020d16..35332d0ca 100644 --- a/packages/runtime/src/scr_core.h +++ b/packages/runtime/src/scr_core.h @@ -81,7 +81,7 @@ void scr_init(void); /* Program objects emitted by the bundled LLVM helper reference this symbol. * Its versioned spelling makes a mismatched manual runtime link fail before * the program can start. */ -void scr_runtime_abi_v27(void); +void scr_runtime_abi_v28(void); void scr_runtime_workers_v8(void); /* ── the trap funnel (scr_console.c; scr_library.c under -DSCR_LIB) ────── diff --git a/packages/runtime/src/scr_dyn.h b/packages/runtime/src/scr_dyn.h index bb3b06847..2bdf95562 100644 --- a/packages/runtime/src/scr_dyn.h +++ b/packages/runtime/src/scr_dyn.h @@ -1272,15 +1272,19 @@ bool scr_dyn_strict_eq(const ScrDyn *a, const ScrDyn *b); * result owned (+1). MAY THROW (NULL with the exception pending). */ ScrDyn *scr_dyn_prepare_method(ScrDyn *recv, const char *method); ScrDyn *scr_dyn_invoke_prepared(ScrDyn *recv, ScrDyn *callee, const char *method, ScrDyn *const *args, size_t argc, const char *what); -/* scr_dyn_invoke_prepared for Buffer.prototype method names (scr_bytes_dyn.c), - * one entry per method family: a Buffer receiver runs the method, any other - * takes the ordinary dispatch. */ -ScrDyn *scr_dyn_invoke_buffer_num(ScrDyn *recv, ScrDyn *callee, const char *method, ScrDyn *const *args, size_t argc, const char *what); -ScrDyn *scr_dyn_invoke_buffer_write(ScrDyn *recv, ScrDyn *callee, const char *method, ScrDyn *const *args, size_t argc, const char *what); -ScrDyn *scr_dyn_invoke_buffer_fill(ScrDyn *recv, ScrDyn *callee, const char *method, ScrDyn *const *args, size_t argc, const char *what); -ScrDyn *scr_dyn_invoke_buffer_to_string(ScrDyn *recv, ScrDyn *callee, const char *method, ScrDyn *const *args, size_t argc, const char *what); -ScrDyn *scr_dyn_invoke_buffer_search(ScrDyn *recv, ScrDyn *callee, const char *method, ScrDyn *const *args, size_t argc, const char *what); -ScrDyn *scr_dyn_invoke_buffer(ScrDyn *recv, ScrDyn *callee, const char *method, ScrDyn *const *args, size_t argc, const char *what); +/* scr_dyn_invoke_prepared for a call site whose method name is a constant: + * *mid_slot, the site's -1-initialized slot, caches the name's method id + * after the first call, so later calls skip the name lookup. */ +ScrDyn *scr_dyn_invoke_prepared_id(ScrDyn *recv, ScrDyn *callee, const char *method, int32_t *mid_slot, ScrDyn *const *args, size_t argc, const char *what); +/* scr_dyn_invoke_prepared_id for Buffer.prototype method names + * (scr_bytes_dyn.c), one entry per method family: a Buffer receiver runs + * the method, any other takes the ordinary dispatch. */ +ScrDyn *scr_dyn_invoke_buffer_num(ScrDyn *recv, ScrDyn *callee, const char *method, int32_t *mid_slot, ScrDyn *const *args, size_t argc, const char *what); +ScrDyn *scr_dyn_invoke_buffer_write(ScrDyn *recv, ScrDyn *callee, const char *method, int32_t *mid_slot, ScrDyn *const *args, size_t argc, const char *what); +ScrDyn *scr_dyn_invoke_buffer_fill(ScrDyn *recv, ScrDyn *callee, const char *method, int32_t *mid_slot, ScrDyn *const *args, size_t argc, const char *what); +ScrDyn *scr_dyn_invoke_buffer_to_string(ScrDyn *recv, ScrDyn *callee, const char *method, int32_t *mid_slot, ScrDyn *const *args, size_t argc, const char *what); +ScrDyn *scr_dyn_invoke_buffer_search(ScrDyn *recv, ScrDyn *callee, const char *method, int32_t *mid_slot, ScrDyn *const *args, size_t argc, const char *what); +ScrDyn *scr_dyn_invoke_buffer(ScrDyn *recv, ScrDyn *callee, const char *method, int32_t *mid_slot, ScrDyn *const *args, size_t argc, const char *what); ScrDyn *scr_dyn_invoke(ScrDyn *recv, const char *method, ScrDyn *const *args, size_t argc, const char *what); /* Keyed read on a FUNC node: the own-property table first (defineProperties * writes land there), then "name" (the box's best-effort static name; "" diff --git a/packages/runtime/src/scr_dyn_invoke.c b/packages/runtime/src/scr_dyn_invoke.c index b0f30ac74..d4189f070 100644 --- a/packages/runtime/src/scr_dyn_invoke.c +++ b/packages/runtime/src/scr_dyn_invoke.c @@ -724,7 +724,7 @@ static bool dyn_bytes_proto_real(int m) { } static ScrDyn *scr_dyn_invoke_impl( - ScrDyn *recv, const char *method, ScrDyn *const *args, size_t argc, + ScrDyn *recv, const char *method, int mid, ScrDyn *const *args, size_t argc, const char *what, ScrDyn *callback_recv); /* Live native arrays: the constant-time methods that loops call once per @@ -796,13 +796,26 @@ ScrDyn *scr_dyn_invoke_prepared(ScrDyn *recv, ScrDyn *callee, const char *method ScrDyn *scr_dyn_invoke(ScrDyn *recv, const char *method, ScrDyn *const *args, size_t argc, const char *what) { - return scr_dyn_invoke_impl(recv, method, args, argc, what, NULL); + return scr_dyn_invoke_impl(recv, method, dyn_method_of(method), args, argc, what, NULL); +} + +ScrDyn *scr_dyn_invoke_prepared_id(ScrDyn *recv, ScrDyn *callee, const char *method, + int32_t *mid_slot, ScrDyn *const *args, size_t argc, + const char *what) { + if (callee) return scr_dyn_invoke_prepared(recv, callee, method, args, argc, what); + /* The slot is the call site's, shared by every thread: each writes the + * same id. */ + int32_t mid = __atomic_load_n(mid_slot, __ATOMIC_RELAXED); + if (mid < 0) { + mid = dyn_method_of(method); + __atomic_store_n(mid_slot, mid, __ATOMIC_RELAXED); + } + return scr_dyn_invoke_impl(recv, method, mid, args, argc, what, NULL); } static ScrDyn *scr_dyn_invoke_impl( - ScrDyn *recv, const char *method, ScrDyn *const *args, size_t argc, + ScrDyn *recv, const char *method, int mid, ScrDyn *const *args, size_t argc, const char *what, ScrDyn *callback_recv) { - const int mid = dyn_method_of(method); if (recv->kind == SCR_DYN_UNDEF || recv->kind == SCR_DYN_NULL) { ScrJsonBuf b; scr_jb_init(&b); @@ -874,7 +887,7 @@ static ScrDyn *scr_dyn_invoke_impl( bool mutates = materialized->kind == SCR_DYN_ARR && dyn_arr_proto_mutates(mid); ScrDyn *result = scr_dyn_invoke_impl( - materialized, method, args, argc, what, recv); + materialized, method, mid, args, argc, what, recv); if (mutates && !scr_exc_pending()) scr_dyn_typed_ref_commit(recv); if (result == materialized) { scr_dyn_release(result); diff --git a/tests/harness/self-hosting-native-toolchain.test.ts b/tests/harness/self-hosting-native-toolchain.test.ts index c0176eb93..c4013d288 100644 --- a/tests/harness/self-hosting-native-toolchain.test.ts +++ b/tests/harness/self-hosting-native-toolchain.test.ts @@ -108,7 +108,7 @@ for (const backend of ["llvm"] as const) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 27, marker: "scr_runtime_abi_v27" }, + runtime_abi: { version: 28, marker: "scr_runtime_abi_v28" }, compiler: { command: "fixture", identity: "fixture", target: target.llvmTriple }, macros: { executable: [], excluded: [], sanitizer: "external-toolchain-required" }, flavors: { From 5c29845909601d6e8195e36ef048938209bbe358 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 15:23:28 -0500 Subject: [PATCH 071/197] Support Readable.fromWeb over web ReadableStreams --- .../builtin-apis/streams-and-readline.mdx | 3 +- packages/cli/test/native-link-info.test.ts | 4 +- .../ambient/scriptc-node-fallback.d.ts | 10 + .../compiler/src/backend/llvm/lib-shared.ts | 4 +- .../compiler/src/backend/native/executable.ts | 2 + packages/compiler/src/backend/runtime-abi.ts | 4 +- .../src/backend/runtime-pack-targets.test.ts | 2 +- .../compiler/src/backend/runtime-pack.test.ts | 2 +- .../src/frontend/lowering/lower-stream.ts | 140 +++++++++- packages/compiler/src/ir/builtin-effects.ts | 1 + .../compiler/src/ir/builtin-signatures.ts | 3 + packages/compiler/src/ir/runtime-features.ts | 1 + packages/compiler/src/ir/validate.ts | 16 ++ .../test/native-codegen-integration.test.ts | 4 +- .../corpus/7430-readable-from-web.mjs.json | 6 + .../corpus/7431-readable-from-web.ts.json | 6 + .../corpus/7432-readable-push-views.mjs.json | 6 + .../runtime-darwin-arm64/scripts/build.mjs | 2 +- .../runtime-pack-matrix.mjs | 1 + .../runtime-pack-common/scripts/build.mjs | 2 +- packages/runtime/src/scr_console.c | 2 +- packages/runtime/src/scr_core.h | 2 +- packages/runtime/src/scr_fetch.c | 48 +++- packages/runtime/src/scr_generator.h | 12 + packages/runtime/src/scr_stream.c | 19 +- packages/runtime/src/scr_stream.h | 7 + packages/runtime/src/scr_stream_from_web.c | 247 ++++++++++++++++++ tests/corpus/7430-readable-from-web.mjs | 42 +++ tests/corpus/7431-readable-from-web.ts | 47 ++++ tests/corpus/7432-readable-push-views.mjs | 16 ++ .../7430-readable-from-web.mjs.json | 1 + .../7431-readable-from-web.ts.json | 1 + .../7432-readable-push-views.mjs.json | 1 + .../self-hosting-native-toolchain.test.ts | 2 +- 34 files changed, 631 insertions(+), 35 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7430-readable-from-web.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7431-readable-from-web.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7432-readable-push-views.mjs.json create mode 100644 packages/runtime/src/scr_stream_from_web.c create mode 100644 tests/corpus/7430-readable-from-web.mjs create mode 100644 tests/corpus/7431-readable-from-web.ts create mode 100644 tests/corpus/7432-readable-push-views.mjs create mode 100644 tests/harness/corpus-costs/7430-readable-from-web.mjs.json create mode 100644 tests/harness/corpus-costs/7431-readable-from-web.ts.json create mode 100644 tests/harness/corpus-costs/7432-readable-push-views.mjs.json diff --git a/docs/content/limitations/builtin-apis/streams-and-readline.mdx b/docs/content/limitations/builtin-apis/streams-and-readline.mdx index eab139773..aa0afd352 100644 --- a/docs/content/limitations/builtin-apis/streams-and-readline.mdx +++ b/docs/content/limitations/builtin-apis/streams-and-readline.mdx @@ -25,7 +25,8 @@ It also accepts a lone `Readable` or `Writable` and a `{ readable, writable }` pair, which the Duplex reads and writes through, a promise, whose value it reads, and an async function, which reads what is written and must resolve to `null` or `undefined`. A web stream and a body function that declares the options argument are unsupported in `Duplex.from`. `stream.compose` accepts streams and async generator functions in any position, an async function as the last stage, and an array, a string, or a generator as the first stage; one stage is `Duplex.from` of it. - A web `ReadableStream` is read as an async iterable by `Readable.from` and as the first stage of `compose` or `pipeline`; a web stream in a later compose stage, and `Readable.fromWeb` and `Readable.toWeb`, are unsupported. + A web `ReadableStream` is read as an async iterable by `Readable.from` and as the first stage of `compose` or `pipeline`, and through its reader by `Readable.fromWeb` (with `objectMode`, `highWaterMark`, `encoding`, and `signal` in an options object literal) and `Duplex.from`. + A web stream in a later compose stage, `Readable.toWeb`, and web `WritableStream` and `TransformStream` interop are unsupported. The `ERR_INVALID_ARG_VALUE` message for a stream that is no longer readable or writable names the stream's class instead of printing the stream. `readable.compose(stage, { signal })` is `stream.compose(readable, stage)` with the signal added; the options are an object literal whose `signal` is a variable or a property read. `pipeline` from `node:stream/promises` accepts a trailing `{ signal, end }` object, with or without function stages, and `finished` accepts `readable`, `writable`, `error`, `cleanup`, and `signal` options, with boolean literals for the flags. diff --git a/packages/cli/test/native-link-info.test.ts b/packages/cli/test/native-link-info.test.ts index 1f2fb241a..5b917c63b 100644 --- a/packages/cli/test/native-link-info.test.ts +++ b/packages/cli/test/native-link-info.test.ts @@ -72,7 +72,7 @@ describe.runIf(supported)("macOS arm64 native link info", () => { minimum_os: "14.0", }, program: { object, entry_symbol: "main" }, - runtime_abi: { version: 30, marker: "scr_runtime_abi_v30" }, + runtime_abi: { version: 31, marker: "scr_runtime_abi_v31" }, runtime_pack: { kind: "precompiled", package: "@scriptc/runtime-darwin-arm64", @@ -165,6 +165,6 @@ describe.runIf(supported)("macOS arm64 native link info", () => { () => null, (failure: { stderr?: string }) => failure, ); - expect(error?.stderr).toContain("scr_runtime_abi_v30"); + expect(error?.stderr).toContain("scr_runtime_abi_v31"); }); }); diff --git a/packages/compiler/ambient/scriptc-node-fallback.d.ts b/packages/compiler/ambient/scriptc-node-fallback.d.ts index 380f5d814..bd8f78105 100644 --- a/packages/compiler/ambient/scriptc-node-fallback.d.ts +++ b/packages/compiler/ambient/scriptc-node-fallback.d.ts @@ -7290,6 +7290,16 @@ declare module "stream" { readonly destroyed: boolean; readonly errored: Error | null; static from(iterable: unknown, options?: ReadableOptions): Readable; + /* A Readable over a web ReadableStream's reader. */ + static fromWeb( + readableStream: globalThis.ReadableStream, + options?: { + encoding?: string; + highWaterMark?: number; + objectMode?: boolean; + signal?: AbortSignal; + }, + ): Readable; /* The Readable operators: chunks are strings or Buffers, map and * flatMap callbacks return strings or Buffers (flatMap: arrays of * them, or strings), and callbacks that declare the options argument diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index a790ad197..8e433832a 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -1121,7 +1121,7 @@ export const LIB_FN_SYMS: Record = { // chaining forms answer the receiver +1. The throwing members // (listeners and option callbacks run synchronously inside) ride the // generic pending check; loop liveness rides USES_TIMERS_LIB_FNS. - "readable.push": "scr_stream_push", + "readable.push": "scr_stream_push_view", "readable.pushStr": "scr_stream_push_str", "readable.pushStrEnc": "scr_stream_push_str_enc", "readable.pushEncoding": "scr_stream_set_push_encoding", @@ -1134,6 +1134,7 @@ export const LIB_FN_SYMS: Record = { "stream.duplexifyPair": "scr_stream_duplexify_pair", "stream.duplexFromPromise": "scr_stream_duplex_from_promise", "stream.duplexFromAsyncFn": "scr_stream_duplex_from_async_fn", + "fetch.readableFromWeb": "scr_stream_readable_from_web", "writable.writeEncoding": "scr_stream_write_encoding", "writable.setDefaultEncoding": "scr_stream_set_default_encoding", "stream.chunkDone": "scr_stream_chunk_done", @@ -1747,6 +1748,7 @@ export const USES_TIMERS_LIB_FNS = new Set([ "stream.duplexifyPair", "stream.duplexFromPromise", "stream.duplexFromAsyncFn", + "fetch.readableFromWeb", "stream.setWritev", "stream.setWritevDefault", "process.activeResources", diff --git a/packages/compiler/src/backend/native/executable.ts b/packages/compiler/src/backend/native/executable.ts index ad318e623..1b90bb1f1 100644 --- a/packages/compiler/src/backend/native/executable.ts +++ b/packages/compiler/src/backend/native/executable.ts @@ -337,6 +337,8 @@ const RUNTIME_UNITS: readonly RuntimeUnit[] = [ unitArgs((u) => (u.tlsCa && targetPlatform(u.driver) === "win32" ? ["-lcrypt32"] : [])), // Fetch uses the native network, TLS, and HTTP runtime. unit((u) => u.nativeFetch, "scr_fetch.c"), + // Readable.fromWeb reads a web ReadableStream through its reader. + unit((u) => u.nativeFetch && u.opts.stream, "scr_stream_from_web.c"), ]; /** The arguments of the optional runtime inputs, in link order. */ diff --git a/packages/compiler/src/backend/runtime-abi.ts b/packages/compiler/src/backend/runtime-abi.ts index fcc8d160e..462f15a9e 100644 --- a/packages/compiler/src/backend/runtime-abi.ts +++ b/packages/compiler/src/backend/runtime-abi.ts @@ -2,8 +2,8 @@ * of the link contract: a program object keeps this symbol undefined and the * matching runtime defines it, so incompatible manual links fail before the * program can start. */ -export const RUNTIME_ABI_VERSION = 30 as const; -export const RUNTIME_ABI_MARKER = "scr_runtime_abi_v30" as const; +export const RUNTIME_ABI_VERSION = 31 as const; +export const RUNTIME_ABI_MARKER = "scr_runtime_abi_v31" as const; /** Object consumption is useful today, but the complete scr_* surface may * still change before 1.0. The versioned marker prevents accidental mixing; diff --git a/packages/compiler/src/backend/runtime-pack-targets.test.ts b/packages/compiler/src/backend/runtime-pack-targets.test.ts index 58e2b4855..4292b93b2 100644 --- a/packages/compiler/src/backend/runtime-pack-targets.test.ts +++ b/packages/compiler/src/backend/runtime-pack-targets.test.ts @@ -52,7 +52,7 @@ async function fixture(target: NativeTargetSpec, workers = false) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 30, marker: "scr_runtime_abi_v30" }, + runtime_abi: { version: 31, marker: "scr_runtime_abi_v31" }, compiler: { command: "fixture", identity: "fixture", target: target.llvmTriple }, macros: { executable: [], excluded: ["SCR_LIB"], sanitizer: "external-toolchain-required" }, flavors: { diff --git a/packages/compiler/src/backend/runtime-pack.test.ts b/packages/compiler/src/backend/runtime-pack.test.ts index b898a3e62..a5cb5dabe 100644 --- a/packages/compiler/src/backend/runtime-pack.test.ts +++ b/packages/compiler/src/backend/runtime-pack.test.ts @@ -127,7 +127,7 @@ async function fixture(target: NativeTargetSpec = MACOS_ARM64_TARGET) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 30, marker: "scr_runtime_abi_v30" }, + runtime_abi: { version: 31, marker: "scr_runtime_abi_v31" }, compiler: { command: "clang", identity: "fixture clang", diff --git a/packages/compiler/src/frontend/lowering/lower-stream.ts b/packages/compiler/src/frontend/lowering/lower-stream.ts index a7f1469a9..7d15e82bb 100644 --- a/packages/compiler/src/frontend/lowering/lower-stream.ts +++ b/packages/compiler/src/frontend/lowering/lower-stream.ts @@ -1814,6 +1814,7 @@ export function lowerStreamStaticCall( } } } + if (member === "fromWeb" && cls === "%Readable") return lowerReadableFromWeb(lowerer, call, loc); if (member !== "from") { lowerer.noLowering(`${cls.slice(1)}.${member}`, call); } @@ -1846,6 +1847,112 @@ export function lowerStreamStaticCall( return lowerFromSource(lowerer, call, args[0], args[1], false); } +/** Readable.fromWeb(readableStream, options): Node's adapter over the web + * stream's reader (scr_stream_from_web.c). The options are an object + * literal: objectMode a boolean literal, highWaterMark a number, encoding + * a literal name, and signal a variable or property read, applied with + * addAbortSignal as the constructors do. */ +function lowerReadableFromWeb(lowerer: Lowerer, call: ts.CallExpression, loc: SrcLoc): IrExpr { + const args = call.arguments.nodes; + const readable: IrType = { kind: "object", className: "%Readable" }; + if (args.length === 0 || args.length > 2 || args.some((a) => ts.isSpreadElement(a))) { + lowerer.noLowering( + `Readable.fromWeb with ${args.length} arguments`, + call, + "the supported form is Readable.fromWeb(readableStream, options)", + ); + } + const web = lowerer.lowerExprExpecting(args[0]!, DYN); + let objectMode = false; + let hwm: IrExpr = numLit(-1, loc); + let encoding = ""; + let signal: IrExpr | null = null; + const options = args[1]; + if (options !== undefined) { + if (!ts.isObjectLiteralExpression(options)) { + lowerer.noLowering( + "Readable.fromWeb options that are not an object literal", + options, + "pass { objectMode, highWaterMark, encoding, signal } as an object literal", + ); + } + for (const p of options.properties.nodes) { + const key = + (ts.isPropertyAssignment(p) || ts.isShorthandPropertyAssignment(p)) && + ts.isIdentifier(p.name) + ? p.name.text + : null; + const value = + key === null + ? null + : ts.isShorthandPropertyAssignment(p) + ? (p.name as ts.Identifier) + : (p as ts.PropertyAssignment).initializer; + if (key === "objectMode" && value) { + if (value.kind === ts.SyntaxKind.TrueKeyword) objectMode = true; + else if (value.kind !== ts.SyntaxKind.FalseKeyword) + lowerer.noLowering( + "the Readable.fromWeb option 'objectMode' with this value", + p, + "write true or false", + ); + continue; + } + if (key === "highWaterMark" && value) { + hwm = lowerer.lowerExprExpecting(value, F64); + continue; + } + if (key === "encoding" && value) { + const t = lowerer.typeOf(value); + if (t.isStringLiteralType() && knownBufEncoding(t.value) === undefined) { + return nodeThrowExpr( + 1, + "ERR_INVALID_ARG_VALUE", + `The property 'options.encoding' is invalid. Received '${t.value}'`, + readable, + loc, + ); + } + encoding = bufEncoding(lowerer, "the Readable.fromWeb option 'encoding'", value); + continue; + } + if (key === "signal" && value) { + if (!sideEffectFreeOptionValue(value) && !pureRead(value)) { + lowerer.noLowering( + "the Readable.fromWeb option 'signal' with this value", + p, + "pass a variable or property read (const signal = controller.signal)", + ); + } + signal = lowerer.lowerExprExpecting(value, DYN); + continue; + } + lowerer.noLowering( + key === null + ? "Readable.fromWeb options with this member" + : `the Readable.fromWeb option '${key}'`, + p, + "objectMode, highWaterMark, encoding and signal are the Readable.fromWeb options", + ); + } + } + const built: IrExpr = { + kind: "libCall", + fn: "fetch.readableFromWeb", + args: [web, boolLit(objectMode, loc), hwm, strLit(encoding, loc)], + type: readable, + loc, + }; + if (signal === null) return built; + return { + kind: "libCall", + fn: "stream.addAbortSignal", + args: [built, signal], + type: readable, + loc, + }; +} + /** Readable.from(source, options), and for Duplex.from over an iterable * (duplex) duplexify.js's from(Duplexify, body, { objectMode: true, * writable: false }): the same builders with SCR_FROM_DUPLEX, answering a @@ -1865,16 +1972,24 @@ function lowerFromSource( const duplexFlag = duplex ? 32 : 0; const js = isJsSourceFile(ts.getSourceFileOfNode(call)); // A web ReadableStream is async iterable: from.js reads its iterator. - // (Duplex.from takes Readable.fromWeb of it instead.) + // Duplex.from takes _duplexify({ readable: Readable.fromWeb(body) }). const webElem = srcNode ? staticReadableStreamElementType(lowerer, srcNode) : null; + if (srcNode && webElem !== null && duplex) { + const fromWeb: IrExpr = { + kind: "libCall", + fn: "fetch.readableFromWeb", + args: [ + lowerer.lowerExprExpecting(srcNode, DYN), + boolLit(false, loc), + numLit(-1, loc), + strLit("", loc), + ], + type: { kind: "object", className: "%Readable" }, + loc, + }; + return { kind: "libCall", fn: "stream.duplexifyReadable", args: [fromWeb], type, loc }; + } if (srcNode && webElem !== null) { - if (duplex) { - lowerer.noLowering( - "Duplex.from over a web ReadableStream", - srcNode, - "Readable.from(stream) reads a web ReadableStream", - ); - } const source = lowerWebStreamSource(lowerer, srcNode, webElem, loc); const helpers = source ? fromGenHelpers(lowerer, source.out, loc) : null; const opts = optsNode ? fromOptions(lowerer, optsNode) : { flags: 0, hwm: null }; @@ -4276,17 +4391,16 @@ function lowerStreamCompose(lowerer: Lowerer, args: readonly ts.Expression[], lo ); } // A web ReadableStream first stage reads as an async iterable (the - // pipeline's source); elsewhere it would be Duplex.fromWeb of it. - for (let i = args.length === 1 ? 0 : 1; i < args.length; i++) { + // pipeline's source), and alone it is Duplex.from of it; a later stage + // would be Duplex.fromWeb of it. + for (let i = 1; i < args.length; i++) { const a = args[i]!; // (typeOf on a stage function's name would report what collecting its // declared signature deferred) if (pipelineStageFunction(lowerer, a) !== null) continue; if (staticReadableStreamElementType(lowerer, a) !== null) { lowerer.noLowering( - i === 0 - ? "Duplex.from over a web ReadableStream" - : "compose with a web ReadableStream after the first stage", + "compose with a web ReadableStream after the first stage", a, "a web ReadableStream is supported as the first stage of compose or pipeline, and in Readable.from", ); diff --git a/packages/compiler/src/ir/builtin-effects.ts b/packages/compiler/src/ir/builtin-effects.ts index 734f9ee80..af2f2a7cc 100644 --- a/packages/compiler/src/ir/builtin-effects.ts +++ b/packages/compiler/src/ir/builtin-effects.ts @@ -386,6 +386,7 @@ export const MAY_THROW_LIB_FNS: ReadonlySet = new Set([ "stream.duplexFromDyn", "stream.duplexFromPromise", "stream.duplexFromAsyncFn", + "fetch.readableFromWeb", "stream.destroy", "stream.destroyErr", "stream.iteratorClose", diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index ad8b907da..9b23a40da 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -170,6 +170,9 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ "fetch.streamFrom": { argTypes: [null], result: DYN }, // The chunk/result record depends on ReadableStream; validated by validate.ts. "fetch.readerRead": { argTypes: [DYN], result: VOID }, + /** Readable.fromWeb (scr_stream_from_web.c): the stream, objectMode, + * highWaterMark (< 0: the mode's default) and encoding ("": none). */ + "fetch.readableFromWeb": { argTypes: [DYN, BOOL, F64, STRING], result: VOID }, "json.parse": { argTypes: [STRING], result: DYN }, "json.parseReviver": { argTypes: [STRING, DYN], result: DYN }, "json.stringifyReplacer": { argTypes: [DYN, DYN, STRING], result: DYN }, diff --git a/packages/compiler/src/ir/runtime-features.ts b/packages/compiler/src/ir/runtime-features.ts index 84b767fab..12baeea57 100644 --- a/packages/compiler/src/ir/runtime-features.ts +++ b/packages/compiler/src/ir/runtime-features.ts @@ -223,6 +223,7 @@ const STREAM_LIB_FNS: ReadonlySet = new Set([ "stream.duplexFromDyn", "stream.duplexFromPromise", "stream.duplexFromAsyncFn", + "fetch.readableFromWeb", "stream.duplexifyReadable", "stream.duplexifyWritable", "stream.duplexifyPair", diff --git a/packages/compiler/src/ir/validate.ts b/packages/compiler/src/ir/validate.ts index d08d0084f..8c9a0432e 100644 --- a/packages/compiler/src/ir/validate.ts +++ b/packages/compiler/src/ir/validate.ts @@ -4288,6 +4288,22 @@ function validateFunction( } return true; } + if (e.fn === "fetch.readableFromWeb") { + // Readable.fromWeb: (stream, objectMode, highWaterMark, encoding) → a Readable. + if ( + e.args.length !== 4 || + e.args[0]?.type.kind !== "dyn" || + e.args[1]?.type.kind !== "bool" || + e.args[2]?.type.kind !== "f64" || + e.args[3]?.type.kind !== "string" || + e.type.kind !== "object" + ) + err( + `libCall ${e.fn} must take (stream, objectMode, highWaterMark, encoding) and return a Readable`, + e.loc, + ); + return true; + } if (e.fn === "fetch.readerRead") { if (e.type.kind !== "promise" || e.type.inner.kind !== "record") { err(`libCall fetch.readerRead must return a promise of a read-result record`, e.loc); diff --git a/packages/compiler/test/native-codegen-integration.test.ts b/packages/compiler/test/native-codegen-integration.test.ts index 169e89c2e..7f819bdad 100644 --- a/packages/compiler/test/native-codegen-integration.test.ts +++ b/packages/compiler/test/native-codegen-integration.test.ts @@ -378,7 +378,7 @@ done: expect(await symbols(helperObject, ["-u"])).toEqual(await symbols(clangObject, ["-u"])); expect(await symbols(helperObject, ["-gU"])).toEqual(await symbols(clangObject, ["-gU"])); expect(await symbols(helperObject, ["-gU"])).toEqual(["0000000000000000 T _main"]); - expect(await symbols(helperObject, ["-u"])).toContain("_scr_runtime_abi_v30"); + expect(await symbols(helperObject, ["-u"])).toContain("_scr_runtime_abi_v31"); const clangExe = join(dir, "clang-program"); const linkDriver = join(dir, "clang-driver.c"); @@ -712,7 +712,7 @@ done: .split("\n"); expect(undefinedSymbols).toContain("_sf_scale"); expect(undefinedSymbols).toContain("_sf_callback_mix"); - expect(undefinedSymbols).toContain("_scr_runtime_abi_v30"); + expect(undefinedSymbols).toContain("_scr_runtime_abi_v31"); }); test("helper and clang object paths retain the same outbound FFI C ABI", async () => { diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7430-readable-from-web.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7430-readable-from-web.mjs.json new file mode 100644 index 000000000..3e26c3ba0 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7430-readable-from-web.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7430-readable-from-web.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7431-readable-from-web.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7431-readable-from-web.ts.json new file mode 100644 index 000000000..6461c53af --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7431-readable-from-web.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7431-readable-from-web.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7432-readable-push-views.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7432-readable-push-views.mjs.json new file mode 100644 index 000000000..ffde7918d --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7432-readable-push-views.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7432-readable-push-views.mjs" + ], + "diags": [] +} diff --git a/packages/runtime-darwin-arm64/scripts/build.mjs b/packages/runtime-darwin-arm64/scripts/build.mjs index aa251dc34..e04a0dfcf 100644 --- a/packages/runtime-darwin-arm64/scripts/build.mjs +++ b/packages/runtime-darwin-arm64/scripts/build.mjs @@ -340,7 +340,7 @@ async function build() { package: packageManifest.name, version: packageManifest.version, target: RUNTIME_PACK_MATRIX.target, - runtime_abi: { version: 30, marker: "scr_runtime_abi_v30" }, + runtime_abi: { version: 31, marker: "scr_runtime_abi_v31" }, compiler: { command: compiler, identity: compilerVersion, diff --git a/packages/runtime-pack-common/runtime-pack-matrix.mjs b/packages/runtime-pack-common/runtime-pack-matrix.mjs index 1c0458d7e..d0194be05 100644 --- a/packages/runtime-pack-common/runtime-pack-matrix.mjs +++ b/packages/runtime-pack-common/runtime-pack-matrix.mjs @@ -161,6 +161,7 @@ export const OPTIONAL_RUNTIME_SOURCES = [ // WebCrypto's AES ciphers run over the cipher feature's mbedTLS archive. ["scr_webcrypto_aes.c", all("cipher", "webcrypto")], ["scr_fetch.c", "nativeFetch"], + ["scr_stream_from_web.c", all("stream", "nativeFetch")], ]; export function variantsFor(source) { diff --git a/packages/runtime-pack-common/scripts/build.mjs b/packages/runtime-pack-common/scripts/build.mjs index acf463824..dd34cf10b 100644 --- a/packages/runtime-pack-common/scripts/build.mjs +++ b/packages/runtime-pack-common/scripts/build.mjs @@ -437,7 +437,7 @@ async function build() { package: packageManifest.name, version: packageManifest.version, target: matrix.target, - runtime_abi: { version: 30, marker: "scr_runtime_abi_v30" }, + runtime_abi: { version: 31, marker: "scr_runtime_abi_v31" }, compiler: { command: compiler, identity: compilerVersion, diff --git a/packages/runtime/src/scr_console.c b/packages/runtime/src/scr_console.c index 5b7f8da0f..194a14b26 100644 --- a/packages/runtime/src/scr_console.c +++ b/packages/runtime/src/scr_console.c @@ -148,7 +148,7 @@ static void scr_enable_utf8_console_output(void) { } #endif -void scr_runtime_abi_v30(void) {} +void scr_runtime_abi_v31(void) {} void scr_init(void) { #ifdef SCR_WORKERS diff --git a/packages/runtime/src/scr_core.h b/packages/runtime/src/scr_core.h index 57928cfb5..9f27a15b1 100644 --- a/packages/runtime/src/scr_core.h +++ b/packages/runtime/src/scr_core.h @@ -81,7 +81,7 @@ void scr_init(void); /* Program objects emitted by the bundled LLVM helper reference this symbol. * Its versioned spelling makes a mismatched manual runtime link fail before * the program can start. */ -void scr_runtime_abi_v30(void); +void scr_runtime_abi_v31(void); void scr_runtime_workers_v8(void); /* ── the trap funnel (scr_console.c; scr_library.c under -DSCR_LIB) ────── diff --git a/packages/runtime/src/scr_fetch.c b/packages/runtime/src/scr_fetch.c index 735183b89..96e90e47c 100644 --- a/packages/runtime/src/scr_fetch.c +++ b/packages/runtime/src/scr_fetch.c @@ -44,7 +44,9 @@ typedef struct SfChunk { } SfChunk; struct SfReadRequest { - ScrPromise *promise; + ScrPromise *promise; /* or NULL for a sink request */ + const ScrFetchReadSink *sink; /* readableStreamDefaultReaderRead's read request */ + void *sink_ctx; SfReadRequest *next; }; @@ -776,6 +778,11 @@ static SfReadRequest *sf_reader_take_request(SfReader *r) { static void sf_reader_reject_message_all(SfReader *r, const char *message) { SfReadRequest *request; while ((request = sf_reader_take_request(r)) != NULL) { + if (request->sink) { + request->sink->error(request->sink_ctx, NULL); + free(request); + continue; + } sf_type_error(message); scr_promise_reject_pending(request->promise); scr_promise_release(request->promise); @@ -958,6 +965,12 @@ static void sf_reader_fulfill_one(SfReader *r, bool done, ScrDyn *value) { if (!r) return; SfReadRequest *request = sf_reader_take_request(r); if (!request) return; + if (request->sink) { + if (done) request->sink->close(request->sink_ctx); + else request->sink->chunk(request->sink_ctx, value); + free(request); + return; + } scr_promise_fulfill_ref(request->promise, sf_read_result(done, value), &scr_dyn_retain_v, &scr_dyn_release_v, NULL); scr_promise_release(request->promise); @@ -968,6 +981,11 @@ static void sf_reader_reject_all(SfReader *r, ScrDyn *reason) { if (!r) return; SfReadRequest *request; while ((request = sf_reader_take_request(r)) != NULL) { + if (request->sink) { + request->sink->error(request->sink_ctx, reason); + free(request); + continue; + } sf_reject_promise_reason(request->promise, reason); scr_promise_release(request->promise); free(request); @@ -1654,18 +1672,21 @@ static SfReader *sf_stream_get_reader(SfStream *s) { return r; } -static ScrPromise *sf_reader_read(SfReader *r) { - ScrPromise *p = scr_promise_new(); +/* readableStreamDefaultReaderRead: queue the request, then serve it from + * the queue or a pull. false (TypeError pending) for a released reader. */ +static bool sf_reader_read_request(SfReader *r, ScrPromise *p, const ScrFetchReadSink *sink, + void *sink_ctx) { SfStream *s = r->stream; if (!s || s->reader != r) { sf_type_error("This reader has been released"); - scr_promise_reject_pending(p); - return p; + return false; } s->disturbed = true; SfReadRequest *request = calloc(1, sizeof *request); if (!request) sf_oom(); - request->promise = scr_promise_retain(p); + request->promise = p ? scr_promise_retain(p) : NULL; + request->sink = sink; + request->sink_ctx = sink_ctx; if (r->pending_tail) r->pending_tail->next = request; else r->pending_head = request; r->pending_tail = request; @@ -1674,6 +1695,12 @@ static ScrPromise *sf_reader_read(SfReader *r) { sf_stream_pull(s); sf_stream_drain(s); } + return true; +} + +static ScrPromise *sf_reader_read(SfReader *r) { + ScrPromise *p = scr_promise_new(); + if (!sf_reader_read_request(r, p, NULL, NULL)) scr_promise_reject_pending(p); return p; } @@ -1960,6 +1987,15 @@ ScrDyn *scr_fetch_stream_from_string(ScrStr *iterable) { return out; } +bool scr_fetch_reader_read_into(ScrDyn *reader, const ScrFetchReadSink *sink, void *ctx) { + if (!reader || reader->kind != SCR_DYN_HANDLE || + reader->v.handle.tag != SCR_DYNH_WEB_READER) { + sf_type_error("Expected a ReadableStreamDefaultReader"); + return false; + } + return sf_reader_read_request((SfReader *)reader->v.handle.ptr, NULL, sink, ctx); +} + ScrPromise *scr_fetch_reader_read(ScrDyn *reader) { if (!reader || reader->kind != SCR_DYN_HANDLE || reader->v.handle.tag != SCR_DYNH_WEB_READER) { diff --git a/packages/runtime/src/scr_generator.h b/packages/runtime/src/scr_generator.h index c4397e021..c3dac861d 100644 --- a/packages/runtime/src/scr_generator.h +++ b/packages/runtime/src/scr_generator.h @@ -200,5 +200,17 @@ ScrDyn *scr_fetch_stream_from_bytes( ScrDyn *scr_fetch_stream_from_string( ScrStr *iterable); /* borrowed string; +1 handle */ ScrPromise *scr_fetch_reader_read(ScrDyn *reader); /* borrowed handle; +1 */ +/* A read request served by callbacks instead of a promise (the streams + * spec's readableStreamDefaultReaderRead with a read request; Node's + * Readable.fromWeb adapter uses one). chunk and error borrow their value; + * error's reason is NULL when the reader was released. */ +typedef struct ScrFetchReadSink { + void (*chunk)(void *ctx, ScrDyn *value); + void (*close)(void *ctx); + void (*error)(void *ctx, ScrDyn *reason); +} ScrFetchReadSink; +/* Queue a sink read request on a reader handle (borrowed). false with a + * TypeError pending for a value that is not a reader or a released one. */ +bool scr_fetch_reader_read_into(ScrDyn *reader, const ScrFetchReadSink *sink, void *ctx); #endif /* SCR_GENERATOR_H */ diff --git a/packages/runtime/src/scr_stream.c b/packages/runtime/src/scr_stream.c index 992203eb4..c4910af6a 100644 --- a/packages/runtime/src/scr_stream.c +++ b/packages/runtime/src/scr_stream.c @@ -1129,7 +1129,7 @@ bool scr_stream_push_dyn(ScrStream *s, const ScrDyn *d) { switch (d->kind) { case SCR_DYN_BYTES: { ScrBytes *b = scr_bytes_raw_view(d->v.bytes); - bool r = scr_stream_push(s, b); + bool r = scr_stream_push_view(s, b); scr_bytes_release(b); return r; } @@ -6348,6 +6348,23 @@ bool scr_stream_push(ScrStream *s, ScrBytes *chunk) { return scr_stream_add_chunk(s, chunk, false); } +/* readable.push(chunk) from a program: in byte mode another + * ArrayBufferView becomes a Buffer over the same bytes + * (Stream._uint8ArrayToBuffer); the runtime's own producers push Buffers + * through scr_stream_push. */ +bool scr_stream_push_view(ScrStream *s, ScrBytes *chunk) { + if (chunk && !chunk->is_buffer && !(s->st->has_r && s->st->r.object_entries)) { + ScrBytes *b = scr_bytes_buffer_view(chunk, SCR_BYTES_U8, scr_bytes_byte_offset(chunk), true, + scr_bytes_byte_len(chunk)); + if (!b) return false; + b->is_buffer = true; + bool r = scr_stream_add_chunk(s, b, false); + scr_bytes_release(b); + return r; + } + return scr_stream_add_chunk(s, chunk, false); +} + bool scr_stream_push_str(ScrStream *s, ScrStr *str) { /* object mode: the string is one whole entry, not converted */ if (s->st->has_r && s->st->r.object_entries) diff --git a/packages/runtime/src/scr_stream.h b/packages/runtime/src/scr_stream.h index d276700c8..cdc51466e 100644 --- a/packages/runtime/src/scr_stream.h +++ b/packages/runtime/src/scr_stream.h @@ -266,6 +266,9 @@ ScrDyn *scr_stream_done_dyn_l(ScrClosure *clo, ScrDyn *const *args, size_t argc) /* The readable surface. push/unshift borrow their chunk (strings convert * utf8, Node's decodeStrings default); push answers the below-hwm bool. */ bool scr_stream_push(ScrStream *s, ScrBytes *chunk); +/* readable.push(chunk) from a program: a non-Buffer view becomes a Buffer + * over the same bytes in byte mode. */ +bool scr_stream_push_view(ScrStream *s, ScrBytes *chunk); bool scr_stream_push_str(ScrStream *s, ScrStr *str); /* push(chunk, enc): the per-call literal encoding (canonical); overrides * the stream's defaultEncoding. Borrows both. */ @@ -545,6 +548,10 @@ void scr_stream_set_writev_default(ScrStream *s, ScrClosure *cb, ScrStreamWritev ScrStream *scr_stream_duplex_from_gen(ScrClosure *stage, ScrClosure *source, ScrClosure *next, ScrClosure *read, ScrClosure *ret, ScrClosure *thr); ScrStream *scr_stream_duplex_from_dyn(struct ScrDyn *body); +/* Readable.fromWeb(readableStream, { objectMode, highWaterMark, encoding }) + * (scr_stream_from_web.c): hwm < 0 is the mode's default, encoding NULL or + * a known name; web and encoding borrowed. +1, or NULL with the error. */ +ScrStream *scr_stream_readable_from_web(struct ScrDyn *web, bool object_mode, double hwm, ScrStr *encoding); /* Duplex.from over an async function (the compiled stage takes the source * generator and answers its promise; source and the promise's box helper * borrowed): a writable Duplexify whose final waits for the promise. +1. */ diff --git a/packages/runtime/src/scr_stream_from_web.c b/packages/runtime/src/scr_stream_from_web.c new file mode 100644 index 000000000..f88826335 --- /dev/null +++ b/packages/runtime/src/scr_stream_from_web.c @@ -0,0 +1,247 @@ +/* Readable.fromWeb (Node's newStreamReadableFromReadableStream in + * internal/webstreams/adapters.js): a Readable over a web ReadableStream's + * reader. read() queues one read request (scr_fetch_reader_read_into); a + * chunk is pushed in a microtask, the end pushes null and an error + * destroys the Readable. destroy() cancels the reader unless it has + * closed, then completes. reader.closed's settlement marks it closed (a + * rejection also destroys the Readable). Linked only by programs that + * use both streams and the native web streams. */ +#include "scr_runtime.h" +#include "scr_stream.h" + +#include +#include + +typedef struct SfwCtx SfwCtx; +struct SfwCtx { + size_t rc; + ScrStream *readable; /* +1 until teardown */ + ScrDyn *reader; /* +1 */ + ScrDyn *pending; /* +1: pendingChunk, or NULL */ + bool closed; + SfwCtx *live_prev, *live_next; +}; + +static SCR_TL SfwCtx *sfw_live = NULL; + +static SfwCtx *sfw_ref(SfwCtx *c) { + c->rc++; + return c; +} + +static void sfw_unref(SfwCtx *c) { + if (--c->rc > 0) return; + if (c->readable) scr_stream_release(c->readable); + if (c->reader) scr_dyn_release(c->reader); + if (c->pending) scr_dyn_release(c->pending); + free(c); +} + +static void *sfw_retain_v(void *p) { return sfw_ref(p); } +static void sfw_release_v(void *p) { sfw_unref(p); } + +/* Drop the Readable (the cycle through its callbacks). */ +static void sfw_teardown(SfwCtx *c) { + if (c->live_prev) c->live_prev->live_next = c->live_next; + else if (sfw_live == c) sfw_live = c->live_next; + else return; /* already torn down */ + if (c->live_next) c->live_next->live_prev = c->live_prev; + c->live_prev = c->live_next = NULL; + ScrStream *r = c->readable; + c->readable = NULL; + if (r) scr_stream_release(r); + sfw_unref(c); /* the live list's reference */ +} + +static void sfw_atexit(void) { + while (sfw_live) sfw_teardown(sfw_live); +} + +static ScrClosure *sfw_closure(void *fn, SfwCtx *c) { + ScrClosure *cl = scr_closure_new(fn, 1); + cl->caps[0] = scr_box_new_obj(&sfw_retain_v, &sfw_release_v, NULL); + scr_box_set_ref(cl->caps[0], sfw_ref(c)); + return cl; +} + +static SfwCtx *sfw_of(ScrClosure *cl) { return scr_box_get_ref(cl->caps[0]); /* +1 */ } + +/* destroyer(readable, error) */ +static void sfw_destroyer(SfwCtx *c, ScrDyn *reason) { + ScrStream *r = c->readable; + if (!r || scr_stream_prop(r, "destroyed")) return; + ScrError *err = reason ? scr_error_from_dyn(reason) : NULL; + if (scr_exc_pending()) scr_exc_clear(); + scr_stream_release(scr_stream_destroy(r, err)); + if (err) scr_error_release(err); +} + +/* forwardChunk: readable.push(pendingChunk) */ +static void sfw_forward(ScrClosure *job) { + SfwCtx *c = sfw_of(job); + ScrDyn *chunk = c->pending; + c->pending = NULL; + if (chunk && c->readable) { + scr_stream_push_dyn(c->readable, chunk); + if (scr_exc_pending()) scr_exc_clear(); + } + if (chunk) scr_dyn_release(chunk); + sfw_unref(c); +} + +static void sfw_sink_chunk(void *ctx, ScrDyn *value) { + SfwCtx *c = ctx; + if (c->pending) scr_dyn_release(c->pending); + c->pending = scr_dyn_retain(value); + scr_queue_microtask(sfw_closure((void *)&sfw_forward, c)); + sfw_unref(c); /* the request's reference */ +} + +static void sfw_sink_close(void *ctx) { + SfwCtx *c = ctx; + if (c->readable) scr_stream_push_null(c->readable); + if (scr_exc_pending()) scr_exc_clear(); + sfw_unref(c); +} + +static void sfw_sink_error(void *ctx, ScrDyn *reason) { + SfwCtx *c = ctx; + sfw_destroyer(c, reason); + sfw_unref(c); +} + +static const ScrFetchReadSink sfw_sink = {&sfw_sink_chunk, &sfw_sink_close, &sfw_sink_error}; + +/* read(): readableStreamDefaultReaderRead(reader, readRequest) */ +static void sfw_read_inv(ScrClosure *cb, ScrStream *s, double size) { + (void)s; + (void)size; + SfwCtx *c = sfw_of(cb); + if (c->reader) { + if (!scr_fetch_reader_read_into(c->reader, &sfw_sink, sfw_ref(c))) { + sfw_unref(c); + if (scr_exc_pending()) scr_exc_clear(); + } + } + sfw_unref(c); +} + +/* destroy's done(): callback(error) */ +typedef struct { + SfwCtx *c; /* +1 */ + ScrStream *s; /* +1 */ + ScrError *err; /* +1 or NULL */ + ScrDyn *cancel; /* +1: reader.cancel(error)'s promise */ +} SfwDestroy; + +static void sfw_destroy_done(SfwCtx *c, ScrStream *s, ScrError *err) { + scr_stream_destroy_done(s, err ? scr_error_retain(err) : NULL); + sfw_teardown(c); +} + +static void sfw_cancel_entry(ScrFiber *fiber, void *arg) { + (void)fiber; + SfwDestroy d = *(SfwDestroy *)arg; + free(arg); + ScrDyn *v = scr_await_ref(d.cancel->v.promise); /* then(done, done) */ + if (scr_exc_pending()) scr_exc_clear(); + if (v) scr_dyn_release(v); + sfw_destroy_done(d.c, d.s, d.err); + if (d.err) scr_error_release(d.err); + scr_stream_release(d.s); + scr_dyn_release(d.cancel); + sfw_unref(d.c); +} + +static void sfw_destroy_inv(ScrClosure *cb, ScrStream *s, ScrError *err) { + SfwCtx *c = sfw_of(cb); + if (!c->closed && c->reader) { + /* _destroy(err, cb) gets null for a destroy without an error */ + ScrDyn *reason = err ? scr_dyn_from_error(err) : scr_dyn_new_null(); + ScrDyn *cancel = scr_dyn_invoke(c->reader, "cancel", &reason, 1, "reader.cancel"); + scr_dyn_release(reason); + if (cancel && cancel->kind == SCR_DYN_PROMISE) { + SfwDestroy *d = malloc(sizeof *d); + if (!d) abort(); + d->c = sfw_ref(c); + d->s = scr_stream_retain(s); + d->err = err ? scr_error_retain(err) : NULL; + d->cancel = cancel; + scr_promise_release(scr_async_spawn_after(cancel->v.promise, &sfw_cancel_entry, d)); + sfw_unref(c); + return; + } + if (cancel) scr_dyn_release(cancel); + if (scr_exc_pending()) scr_exc_clear(); + } + sfw_destroy_done(c, s, err); + sfw_unref(c); +} + +/* reader.closed.then(() => closed = true, (e) => { closed = true; destroy(readable, e) }) */ +typedef struct { + SfwCtx *c; /* +1 */ + ScrDyn *closed; /* +1, the promise */ +} SfwClosed; + +static void sfw_closed_entry(ScrFiber *fiber, void *arg) { + (void)fiber; + SfwClosed w = *(SfwClosed *)arg; + free(arg); + ScrDyn *v = scr_await_ref(w.closed->v.promise); + w.c->closed = true; + if (scr_exc_pending()) { + ScrCaught *caught = scr_exc_take(); + ScrDyn *reason = scr_caught_to_dyn(caught); + scr_caught_release(caught); + sfw_destroyer(w.c, reason); + if (reason) scr_dyn_release(reason); + } + if (v) scr_dyn_release(v); + scr_dyn_release(w.closed); + sfw_unref(w.c); +} + +/* web: the ReadableStream (borrowed); hwm < 0 = the mode's default; + * encoding NULL or a known name (borrowed). +1, or NULL with the error + * pending. */ +ScrStream *scr_stream_readable_from_web(ScrDyn *web, bool object_mode, double hwm, ScrStr *encoding) { + if (encoding && encoding->len == 0) encoding = NULL; + if (!web || web->kind != SCR_DYN_HANDLE || web->v.handle.tag != SCR_DYNH_WEB_STREAM) { + scr_dyn_arg_type_fail("readableStream", "an instance of ReadableStream", web); + return NULL; + } + ScrDyn *reader = scr_dyn_invoke(web, "getReader", NULL, 0, "readableStream.getReader"); + if (!reader) return NULL; + SfwCtx *c = calloc(1, sizeof *c); + if (!c) abort(); + c->rc = 1; + c->reader = reader; + ScrStream *r = scr_stream_new_readable(hwm < 0 ? (object_mode ? 16 : -1) : hwm, true, true, + sfw_closure((void *)&sfw_read_inv, c), &sfw_read_inv, + sfw_closure((void *)&sfw_destroy_inv, c), &sfw_destroy_inv); + if (object_mode) scr_stream_set_object_mode(r, true, false); + if (encoding) scr_stream_release(scr_stream_set_encoding(r, encoding)); + c->readable = scr_stream_retain(r); + static SCR_TL bool registered = false; + if (!registered) { + registered = true; + scr_atexit(sfw_atexit); + } + c->live_next = sfw_live; + if (sfw_live) sfw_live->live_prev = c; + sfw_live = c; /* the reference c was created with */ + ScrStr *key = scr_str_new("closed", 6); + ScrDyn *closed = scr_dyn_handle_key_get(reader, key); + scr_str_release(key); + if (closed && closed->kind == SCR_DYN_PROMISE) { + SfwClosed *w = malloc(sizeof *w); + if (!w) abort(); + w->c = sfw_ref(c); + w->closed = closed; + scr_promise_release(scr_async_spawn_after(closed->v.promise, &sfw_closed_entry, w)); + } else if (closed) { + scr_dyn_release(closed); + } + return r; +} diff --git a/tests/corpus/7430-readable-from-web.mjs b/tests/corpus/7430-readable-from-web.mjs new file mode 100644 index 000000000..017556f9d --- /dev/null +++ b/tests/corpus/7430-readable-from-web.mjs @@ -0,0 +1,42 @@ +// Readable.fromWeb: a Readable over a web ReadableStream's reader, in +// byte mode by default (Uint8Array chunks arrive as Buffers), in object +// mode, and with an encoding; an early destroy cancels the stream with +// null, and an error from the stream destroys the Readable. Duplex.from +// and compose() over a lone web ReadableStream read it the same way. +import { Duplex, Readable, compose } from "node:stream"; +const log = []; +const tick = (n) => { if (n < 6) { log.push(`tick ${n}`); process.nextTick(() => tick(n + 1)); } }; +const web = new ReadableStream({ start(c) { c.enqueue(new TextEncoder().encode("ab")); c.enqueue(new TextEncoder().encode("cd")); c.close(); } }); +const r = Readable.fromWeb(web); +log.push(`r ${r.readableObjectMode} ${r.readableHighWaterMark}`); +r.on("data", (b) => log.push(`data ${Buffer.isBuffer(b)} ${b.toString()}`)); +r.on("end", () => log.push("end")); +r.on("close", () => log.push("close")); +tick(0); +await new Promise((res) => r.on("close", res)); +const words = new ReadableStream({ start(c) { c.enqueue("x"); c.enqueue("y"); c.close(); } }); +const o = Readable.fromWeb(words, { objectMode: true, highWaterMark: 1 }); +for await (const v of o) log.push(`obj ${v}`); +const text = new ReadableStream({ start(c) { c.enqueue(new TextEncoder().encode("héllo")); c.close(); } }); +const t = Readable.fromWeb(text, { encoding: "utf8" }); +for await (const v of t) log.push(`text ${typeof v} ${v}`); +let m = 0; +const endless = new ReadableStream({ pull(c) { c.enqueue(new Uint8Array([++m])); }, cancel(reason) { log.push(`cancel ${reason && reason.name}`); } }); +const e = Readable.fromWeb(endless); +e.once("data", (b) => { log.push(`endless ${b[0]}`); e.destroy(); }); +e.on("close", () => log.push("endless close")); +await new Promise((res) => e.on("close", res)); +await new Promise((res) => setTimeout(res, 5)); +const failing = new ReadableStream({ start(c) { c.error(new Error("web broke")); } }); +const f = Readable.fromWeb(failing); +f.on("error", (err) => log.push(`failing error ${err.message}`)); +f.on("close", () => log.push("failing close")); +f.resume(); +await new Promise((res) => f.on("close", res)); +const d = Duplex.from(new ReadableStream({ start(c) { c.enqueue(new TextEncoder().encode("dx")); c.close(); } })); +log.push(`d ${d.readable} ${d.writable} ${d.readableObjectMode}`); +d.on("data", (b) => log.push(`d data ${b.toString()}`)); +await new Promise((res) => d.on("close", res)); +const one = compose(new ReadableStream({ start(c) { c.enqueue(new TextEncoder().encode("one")); c.close(); } })); +for await (const b of one) log.push(`one ${b.toString()}`); +console.log(log.join("\n")); diff --git a/tests/corpus/7431-readable-from-web.ts b/tests/corpus/7431-readable-from-web.ts new file mode 100644 index 000000000..23e7d2871 --- /dev/null +++ b/tests/corpus/7431-readable-from-web.ts @@ -0,0 +1,47 @@ +// Readable.fromWeb in typed code: a ReadableStream read as +// Buffers, a ReadableStream in object mode with a highWaterMark of +// 1, and an AbortSignal that destroys the Readable and cancels the stream. +import { Readable } from "node:stream"; + +const log: string[] = []; + +async function main(): Promise { + const bytes = new ReadableStream({ + start(controller): void { + controller.enqueue(new TextEncoder().encode("web")); + controller.enqueue(new TextEncoder().encode("bytes")); + controller.close(); + }, + }); + const r = Readable.fromWeb(bytes); + for await (const chunk of r) log.push(`bytes ${Buffer.isBuffer(chunk)} ${String(chunk)}`); + const words = new ReadableStream({ + start(controller): void { + controller.enqueue("alpha"); + controller.enqueue("beta"); + controller.close(); + }, + }); + const o = Readable.fromWeb(words, { objectMode: true, highWaterMark: 1 }); + log.push(`object mode ${o.readableObjectMode} ${o.readableHighWaterMark}`); + for await (const word of o) log.push(`word ${String(word)}`); + const ac = new AbortController(); + const signal = ac.signal; + let n = 0; + const endless = new ReadableStream({ + pull(controller): void { + controller.enqueue(new Uint8Array([++n])); + }, + cancel(reason: unknown): void { + log.push(`cancelled ${(reason as Error).name}`); + }, + }); + const a = Readable.fromWeb(endless, { signal }); + a.on("error", (e: Error) => log.push(`aborted ${e.name}`)); + a.once("data", () => ac.abort()); + await new Promise((resolve) => a.on("close", () => resolve())); + await new Promise((resolve) => setTimeout(resolve, 5)); + for (const line of log) console.log(line); +} + +void main(); diff --git a/tests/corpus/7432-readable-push-views.mjs b/tests/corpus/7432-readable-push-views.mjs new file mode 100644 index 000000000..9aca435b6 --- /dev/null +++ b/tests/corpus/7432-readable-push-views.mjs @@ -0,0 +1,16 @@ +// readable.push() in byte mode turns a Uint8Array or another typed array +// into a Buffer over the same bytes (Stream._uint8ArrayToBuffer); the +// pushed array itself is left as it was. +import { Readable } from "node:stream"; +const u = new Uint8Array([100, 101]); +const r = new Readable({ read() {} }); +r.push(u); r.push(new Uint16Array([0x6766])); r.push(null); +for await (const b of r) console.log(Buffer.isBuffer(b), b.toString(), b.length); +console.log("source still", Buffer.isBuffer(u)); +const r1 = new Readable({ read() {} }); +r1.push(new Uint8Array([97, 98])); +r1.push(null); +for await (const b of r1) console.log(Buffer.isBuffer(b), b.toString()); +const r2 = new Readable({ read() {} }); +r2.on("data", (b) => console.log("data", Buffer.isBuffer(b), b.toString())); +r2.push(new Uint8Array([99])); r2.push(null); diff --git a/tests/harness/corpus-costs/7430-readable-from-web.mjs.json b/tests/harness/corpus-costs/7430-readable-from-web.mjs.json new file mode 100644 index 000000000..2266382b4 --- /dev/null +++ b/tests/harness/corpus-costs/7430-readable-from-web.mjs.json @@ -0,0 +1 @@ +{ "plain": 3342, "san": 4411 } diff --git a/tests/harness/corpus-costs/7431-readable-from-web.ts.json b/tests/harness/corpus-costs/7431-readable-from-web.ts.json new file mode 100644 index 000000000..05fab8e59 --- /dev/null +++ b/tests/harness/corpus-costs/7431-readable-from-web.ts.json @@ -0,0 +1 @@ +{ "plain": 1956, "san": 2929 } diff --git a/tests/harness/corpus-costs/7432-readable-push-views.mjs.json b/tests/harness/corpus-costs/7432-readable-push-views.mjs.json new file mode 100644 index 000000000..8a2f5280e --- /dev/null +++ b/tests/harness/corpus-costs/7432-readable-push-views.mjs.json @@ -0,0 +1 @@ +{ "plain": 1582, "san": 2088 } diff --git a/tests/harness/self-hosting-native-toolchain.test.ts b/tests/harness/self-hosting-native-toolchain.test.ts index 81ff5b293..e74d9f60a 100644 --- a/tests/harness/self-hosting-native-toolchain.test.ts +++ b/tests/harness/self-hosting-native-toolchain.test.ts @@ -108,7 +108,7 @@ for (const backend of ["llvm"] as const) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 30, marker: "scr_runtime_abi_v30" }, + runtime_abi: { version: 31, marker: "scr_runtime_abi_v31" }, compiler: { command: "fixture", identity: "fixture", target: target.llvmTriple }, macros: { executable: [], excluded: [], sanitizer: "external-toolchain-required" }, flavors: { From fa62abce4dd178d0d2d2acf022ac72ddbdc793c1 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 16:45:06 -0500 Subject: [PATCH 072/197] Keep record types acyclic when only result-slot stores write their fields The backend gives every function that returns a complete record literal a result-slot entry, `%into.(...params, dst)`, which stores the literal's fields into caller frame storage. The cycle analysis counted those stores as writes after construction, so a record type that refers to itself and is never written after construction carried the cycle collector's header, its live count and its candidate checks in release builds, even when nothing called the entry. Dev builds, which add no entries, left it acyclic. The destination is frame storage of a caller that only projects the value (stack-objects.ts): no object can point to it, so its fields cannot lie on a cycle. computeCycleMutability now skips those stores, under the same condition emit-statements.ts already uses to emit them as initializations (resultSlotStore in the new result-slots.ts). Every other store still makes its field mutable. The same facts mark such fields as never replaced, so reads of them borrow instead of retaining. A program that rebuilds a tree of 2^21 - 1 object literals 12 times runs in 1.19 s instead of 1.35 s and peaks at 68 MB instead of 84 MB: each node is 32 bytes instead of 40. --- .../src/backend/cycle-analysis.test.ts | 56 ++++++++++++ .../compiler/src/backend/cycle-mutability.ts | 8 +- .../src/backend/llvm/emit-statements.ts | 10 +-- .../src/backend/llvm/record-results.ts | 15 +--- packages/compiler/src/backend/result-slots.ts | 29 +++++++ .../7261-acyclic-result-builders.ts.json | 6 ++ tests/corpus/7261-acyclic-result-builders.ts | 86 +++++++++++++++++++ .../7261-acyclic-result-builders.ts.json | 1 + 8 files changed, 189 insertions(+), 22 deletions(-) create mode 100644 packages/compiler/src/backend/result-slots.ts create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7261-acyclic-result-builders.ts.json create mode 100644 tests/corpus/7261-acyclic-result-builders.ts create mode 100644 tests/harness/corpus-costs/7261-acyclic-result-builders.ts.json diff --git a/packages/compiler/src/backend/cycle-analysis.test.ts b/packages/compiler/src/backend/cycle-analysis.test.ts index c78420f73..340ffa8f9 100644 --- a/packages/compiler/src/backend/cycle-analysis.test.ts +++ b/packages/compiler/src/backend/cycle-analysis.test.ts @@ -17,6 +17,7 @@ import { } from "../ir/ir.js"; import { computeTraced, explainCycle } from "./cycle-analysis.js"; import { computeTraced as llvmTraced } from "./llvm/shapes.js"; +import { addRecordResultEntries } from "./llvm/record-results.js"; const loc = { file: "cycles.ts", start: 0, end: 1 }; function module(): IrModule { @@ -404,6 +405,61 @@ test("a later write through any alias makes the edge mutable", () => { checkExact(record, ["record:list"]); }); +test("a result-slot entry's stores into its destination are not writes", () => { + const next: IrType = { kind: "union", unionId: "next" }; + const listModule = (): IrModule => { + const mod = module(); + mod.records = [shape("list", [next, F64])]; + mod.unions = [{ id: "next", arms: [ref("list"), { kind: "nullT" }] }]; + mod.functions.push({ + name: "cell", + params: [{ localId: "rest", name: "rest", type: next }], + locals: [], + returnType: ref("list"), + body: [ + { + kind: "return", + value: { + kind: "recordLit", + fields: [ + { name: "field0", value: { kind: "varRef", localId: "rest", type: next, loc } }, + { name: "field1", value: { kind: "numLit", value: 1, type: F64, loc } }, + ], + type: ref("list"), + loc, + }, + loc, + }, + ], + loc, + }); + return addRecordResultEntries(mod); + }; + // The entry stores the literal's fields into caller frame storage that + // nothing can point to: the list stays acyclic. + const entries = listModule(); + expect(entries.functions.map((fn) => fn.name)).toContain("%into.cell"); + checkExact(entries, []); + // The same store into the destination's local id outside an entry, and a + // store into any other binding of the shape inside one, are writes. + const store = (localId: string): IrStmt => ({ + kind: "recordSet", + obj: { kind: "varRef", localId, type: ref("list"), loc }, + shapeId: "list", + field: "field0", + value: value(next), + loc, + }); + for (const [fn, localId] of [ + ["main", "%into.dst"], + ["%into.cell", "v"], + ] as const) { + const written = listModule(); + written.functions.find((f) => f.name === fn)!.body.unshift(store(localId)); + checkExact(written, ["record:list"], ["next"]); + } +}); + test("field increments count as writes", () => { const mod = tree(); mod.functions[0]!.body.push({ diff --git a/packages/compiler/src/backend/cycle-mutability.ts b/packages/compiler/src/backend/cycle-mutability.ts index 78daa6204..7ae451780 100644 --- a/packages/compiler/src/backend/cycle-mutability.ts +++ b/packages/compiler/src/backend/cycle-mutability.ts @@ -2,6 +2,7 @@ import type { IrExpr, IrFunction, IrModule, IrStmt, IrType } from "../ir/ir.js"; import { RUNTIME_EMITTER_CLASS, RUNTIME_STREAM_CLASSES, typeKey } from "../ir/ir.js"; import { dynamicFieldStores, type DynamicFieldStores } from "../ir/int32-slots.js"; import { everyExprChild, everyStmtChild, everyTypeChild } from "../ir/traverse.js"; +import { resultSlotStore } from "./result-slots.js"; /** Which shape fields can be written after their object escapes. * @@ -17,6 +18,10 @@ import { everyExprChild, everyStmtChild, everyTypeChild } from "../ir/traverse.j * - every `fieldSet` outside a constructor's own `this`, `fieldIncDec`, and * `recordSet` makes its (unit, field) mutable; `recordKeySet` on declared * fields and `recordKeyDelete` make the whole record mutable; + * - a result-slot entry's stores into its destination (result-slots.ts) are + * not writes: the destination is frame storage of a caller that only + * projects the value, so no object can point to it and its fields cannot + * lie on a cycle; * - a constructor's stores into its own `this` are construction-only up to * the first top-level statement in which `this` escapes (any use other * than a field store target, a field read, or the receiver of the direct @@ -296,7 +301,8 @@ export function computeCycleMutability( result.fields.add(fieldKey(unitOfClass(s.className), s.field)); break; case "recordSet": - result.fields.add(fieldKey(`record:${s.shapeId}`, s.field)); + if (!resultSlotStore(fn.name, s)) + result.fields.add(fieldKey(`record:${s.shapeId}`, s.field)); break; case "recordKeySet": if (s.overflowOnly !== true) result.units.add(`record:${s.shapeId}`); diff --git a/packages/compiler/src/backend/llvm/emit-statements.ts b/packages/compiler/src/backend/llvm/emit-statements.ts index 39ff5dcd3..c5043aa3f 100644 --- a/packages/compiler/src/backend/llvm/emit-statements.ts +++ b/packages/compiler/src/backend/llvm/emit-statements.ts @@ -37,7 +37,7 @@ import { InternalCompilerError } from "../../errors.js"; import { emitDenseArrayGet, emitDenseArrayPop, emitDenseArraySet } from "./dense-array-access.js"; import { emitBorrowedInput } from "./borrowed-inputs.js"; import { f64Lit, llvmCommentText, loadArrCount, sizeToDouble } from "./common.js"; -import { RESULT_DST_ID, RESULT_ENTRY_PREFIX } from "./record-results.js"; +import { resultSlotStore } from "../result-slots.js"; import { matchIntegerArrayForLoop, matchIntegerBytesForLoop, @@ -573,13 +573,7 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { ? emitter.classFieldPtr(obj.name, s.className, s.field, "compressed") : emitter.recordFieldPtr(obj.name, s.shapeId, s.field); // A result-slot entry fills zeroed caller storage, each field once. - if ( - s.kind === "recordSet" && - s.obj.kind === "varRef" && - s.obj.localId === RESULT_DST_ID && - emitter.currentFunctionName.startsWith(RESULT_ENTRY_PREFIX) - ) - emitter.initField(ptr, type, inline, v); + if (resultSlotStore(emitter.currentFunctionName, s)) emitter.initField(ptr, type, inline, v); else emitter.replaceField(ptr, type, inline, v); if ( s.kind === "fieldSet" && diff --git a/packages/compiler/src/backend/llvm/record-results.ts b/packages/compiler/src/backend/llvm/record-results.ts index 47a7983d2..db25016cd 100644 --- a/packages/compiler/src/backend/llvm/record-results.ts +++ b/packages/compiler/src/backend/llvm/record-results.ts @@ -12,20 +12,9 @@ import { import { dynamicFieldStores } from "../../ir/int32-slots.js"; import { everyExpr, everyStmtList, transformStmtList } from "../../ir/traverse.js"; import { InlineUnionFields } from "./inline-unions.js"; +import { RESULT_DST_ID, resultEntryName } from "../result-slots.js"; -/** Prefix of a function's result-slot entry: `%into.(...params, dst)` - * writes the function's fresh result object into `dst` and returns nothing. */ -export const RESULT_ENTRY_PREFIX = "%into."; - -export function resultEntryName(name: string): string { - return `${RESULT_ENTRY_PREFIX}${name}`; -} - -/** The local id of a result-slot entry's destination parameter. The - * caller passes zeroed frame storage, and on every path the entry stores - * each of its fields at most once, so a store into it is an - * initialization: there is no old value to release. */ -export const RESULT_DST_ID = "%into.dst"; +export { RESULT_DST_ID, RESULT_ENTRY_PREFIX, resultEntryName } from "../result-slots.js"; function distinctNames(names: readonly string[]): boolean { return new Set(names).size === names.length; diff --git a/packages/compiler/src/backend/result-slots.ts b/packages/compiler/src/backend/result-slots.ts new file mode 100644 index 000000000..608877f06 --- /dev/null +++ b/packages/compiler/src/backend/result-slots.ts @@ -0,0 +1,29 @@ +import type { IrStmt } from "../ir/ir.js"; + +/** Prefix of a function's result-slot entry: `%into.(...params, dst)` + * writes the function's fresh result object into `dst` and returns nothing + * (llvm/record-results.ts). */ +export const RESULT_ENTRY_PREFIX = "%into."; + +export function resultEntryName(name: string): string { + return `${RESULT_ENTRY_PREFIX}${name}`; +} + +/** The local id of a result-slot entry's destination parameter. The + * caller passes zeroed frame storage, and on every path the entry stores + * each of its fields at most once, so a store into it is an + * initialization: there is no old value to release. */ +export const RESULT_DST_ID = "%into.dst"; + +/** Whether `s`, a statement of the function `fnName`, is a result-slot + * entry's store into its destination. The destination is frame storage of + * a caller that only projects the value (llvm/stack-objects.ts), so the + * store initializes a field of an object no other object can point to. */ +export function resultSlotStore(fnName: string, s: IrStmt): boolean { + return ( + s.kind === "recordSet" && + s.obj.kind === "varRef" && + s.obj.localId === RESULT_DST_ID && + fnName.startsWith(RESULT_ENTRY_PREFIX) + ); +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7261-acyclic-result-builders.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7261-acyclic-result-builders.ts.json new file mode 100644 index 000000000..4f4afd821 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7261-acyclic-result-builders.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7261-acyclic-result-builders.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7261-acyclic-result-builders.ts b/tests/corpus/7261-acyclic-result-builders.ts new file mode 100644 index 000000000..b6a98fce4 --- /dev/null +++ b/tests/corpus/7261-acyclic-result-builders.ts @@ -0,0 +1,86 @@ +// Records built by functions that return object literals, as tree and list +// builders are written. The backend gives such a function a result-slot +// entry that writes the literal's fields into a caller's frame storage, and +// those stores do not make the record's fields mutable: a type no store +// changes after construction stays acyclic and needs no cycle-collector +// header. This pins what such programs print: rebuilt trees, a shared +// subtree (a DAG), a builder whose result the caller only projects (so the +// entry runs), a list longer than the runtime's teardown depth budget, and +// a record type the program writes after construction, whose rings only +// the collector frees. The sanitized lane asserts every object is freed. + +interface Node { + value: number; + left: Node | null; + right: Node | null; +} + +function leaf(value: number): Node { + return { value, left: null, right: null }; +} + +function branch(value: number, left: Node | null, right: Node | null): Node { + return { value, left, right }; +} + +function tree(depth: number, value: number): Node { + return depth === 0 ? leaf(value) : branch(value, tree(depth - 1, value * 2), tree(depth - 1, value * 2 + 1)); +} + +function sum(node: Node | null): number { + return node === null ? 0 : node.value + sum(node.left) + sum(node.right); +} + +let total = 0; +for (let generation = 0; generation < 8; generation++) total += sum(tree(8, generation + 1)); +console.log("trees", total); + +// A subtree shared by both branches is released once. +const shared = tree(3, 1); +const dag = branch(0, shared, shared); +console.log("dag", sum(dag), JSON.stringify(branch(1, leaf(2), null))); + +// Only projected: the result lives in the caller's frame, which releases +// the subtree it holds. +const top = branch(5, tree(2, 1), null); +console.log("top", top.value, sum(top.left), top.right === null); + +interface Cell { + n: number; + next: Cell | null; +} + +function cons(n: number, next: Cell | null): Cell { + return { n, next }; +} + +let list: Cell | null = null; +for (let i = 0; i < 5000; i++) list = cons(i, list); +let length = 0; +let last = -1; +for (let c = list; c !== null; c = c.next) { + length++; + last = c.n; +} +console.log("list", length, last); +list = null; + +// A record type written after construction keeps its cycle-collector +// header: these rings are garbage only the collector can free. +interface Ring { + id: number; + next: Ring | null; +} + +function ring(id: number, next: Ring | null): Ring { + return { id, next }; +} + +let ids = 0; +for (let i = 0; i < 200; i++) { + const a = ring(i, null); + const b = ring(i + 1, a); + a.next = b; + ids += a.next.id + b.next!.id; +} +console.log("rings", ids); diff --git a/tests/harness/corpus-costs/7261-acyclic-result-builders.ts.json b/tests/harness/corpus-costs/7261-acyclic-result-builders.ts.json new file mode 100644 index 000000000..30b7c116f --- /dev/null +++ b/tests/harness/corpus-costs/7261-acyclic-result-builders.ts.json @@ -0,0 +1 @@ +{ "plain": 3336, "san": 4712 } From 7a5fdae1bd8fc800b76fe24619b73db07329b8cd Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 16:45:59 -0500 Subject: [PATCH 073/197] Send an http client request's head and body in one write request.end(body) on a request whose head was not sent yet wrote the head and then the body. On a socket that was already connected (a reused keep-alive connection), each write is sent at once, so the two went out as separate segments, and the server could handle the head and answer before the body arrived. Node sends the head together with the body. When the client and the server run in one program, the server's 'end' and 'close' for the request and the client's reading of the response then depended on which socket the poller reported first: corpus 5409-http-held-request-body printed the client's line before the request's 'end' and 'close' in a macOS dev build. The head and a body known at end() now go out as one write, as Node sends them (the server's response already did). --- packages/runtime/src/scr_http.c | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/packages/runtime/src/scr_http.c b/packages/runtime/src/scr_http.c index a5847e59b..cb88297bf 100644 --- a/packages/runtime/src/scr_http.c +++ b/packages/runtime/src/scr_http.c @@ -4323,14 +4323,32 @@ static void scr_http_client_end_raw(ScrHttpClientReq *c, const char *data, size_ } if (!c->head_sent) { /* whole body known NOW: Content-Length framing (or none — the head - * serializer's empty-body method split) */ + * serializer's empty-body method split). Head and body go out as one + * write, as Node sends them: on a connected (reused) socket two writes + * would be two segments, and the server could answer before the body + * arrives. */ + c->hold_head = true; scr_http_client_send_head(c, (long long)len); + c->hold_head = false; if (scr_exc_pending()) return; if (c->chunked) { scr_http_client_write_raw(c, data, len); if (c->sock) scr_http_send_chunk_end(c->sock, &c->trailers); - } else if (c->sock && len > 0) { - scr_net_sock_write_native(c->sock, data, len); + } else if (c->held_head != NULL) { + char *head = c->held_head; + size_t head_len = c->held_head_len; + c->held_head = NULL; + c->held_head_len = c->held_head_cap = 0; + if (c->sock) { + if (len > 0) { + char *joined = realloc(head, head_len + len); + if (!joined) scr_http_oom(); + head = joined; + memcpy(head + head_len, data, len); + } + scr_net_sock_write_native(c->sock, head, head_len + len); + } + free(head); } } else { scr_http_client_write_raw(c, data, len); From 8bc4586b9ae864f693d0f761bd23f93a7c7f3775 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 16:56:13 -0500 Subject: [PATCH 074/197] Select AST children by wire slot in the generated getters --- packages/compiler/native/cli.profdata.gz | Bin 1634955 -> 1634316 bytes .../src/frontend/ts7/ast-node.test.ts | 24 + .../compiler/src/frontend/ts7/ast-node.ts | 1127 +++++++++++++++-- .../compiler/src/frontend/ts7/ast-wire.ts | 4 +- scripts/generate-ts7-ast-schema.mjs | 36 +- 5 files changed, 1084 insertions(+), 107 deletions(-) diff --git a/packages/compiler/native/cli.profdata.gz b/packages/compiler/native/cli.profdata.gz index e2f344a4d7f73f6e5b70e119a45c1e9db5fdf6d3..aa0e9c36beac0cdc3528321a8a654ede3118b4b4 100644 GIT binary patch literal 1634316 zcmV)yK$5>7iwFP!000021MD4loD)TNcXOSi!_kor3c~gRNUuuss8WnuX72*IT_%|%S2-~6pO@L}PJQ#{ zy*F>(oO9-S+@@nrRpLC51Oj8;elIAU{r1kFbgq?7veGpy>~scjR&wq0xQgqWfkp9e zCx5@fkLmO(15^FqwIzbygB4jlKk)0#3E!N5Khb*LqpIyZ9#O4fT|cg7J^yxT!g*Zp zh3C@!xT*DgcEdmoH_IK9{g}6&H+%SelDO%MPUk5+foAQ#a{>Tfu4cqyRB*rzbjB7u~*=E9qV~&zw^x?x?9h!>Jbxu zp{#+Yh%VFG`~H{E+WP0PHKbnMu9ifo@qoJE@r5O;B*U97>s^LA?Xx^F4B?_8w)rQgn3fMthgkRMBIJsmk+1Z zY)%kCLTi!Qs~(RXOaY#JyD6J2!%95RG-J_67RYin{tOc6VyaJ8K!Tj%_~+2ixAa?3A4ge&D2fuxax5>&yb?mYxt7>?4Q6mshz`D!`tyq} zv(m@Na55xgp9-{0pc<;?1jC%FtNphA43l|pwaynAw@8ZeJSomlF=TS{CpEThYTV39 zuTKdj6ON+;szx7|x3}taYN|fz(Avuue7`FpxfU7k%ZpiiiuY?A;^SZO`KOfd|CCbJ zBPBk+E{Xcb`C4&xm0H9j#=>Wg)qyI#$e>Y&o$X*<7ly4XC$NAQI2Op9eCW`aQ=J;TlUTKc{ghD5 z=lHtUA!<%(55!N~geo?L5~bbY?GJoCbsc!UZciVeJ5`VBbGThu+4nu&yNH@Iy4l7c z99mvo;_p}1PuZB<(t5I-!f4PQ;8U~gst$Ng5hcywwMXSd^Xs`g5{8zr)2dZ+;kW*e z;h#6R>b=cMx2+y*a(SwP>)8kZ~a6?<(IUL-vt_;Uj5SHd3ExceYb2dk(Dr#=>bmajKnTg^zBTInSNf zJ#lym8^*^M$8v#omw5ZqG0z*vyh1i0w#}RWWbfs0#X1%`x&2mC)9vZ2>e+TvXBCw( zfh2HR(-u2kK05mcTYV(+Dwsm6l(3Ppepk{>P4qjwo%AxI(?=igZP5g8ffpEsR%AvI zd4=U98a0&eqcc z>of#(YqGqBX^Ln@?AmyX{N@!6kx7rNPpm{UvSH{v$DTWP?zd0U2B*Q%t(_V^?`Fi} zmwvmc#}oBKhr{ES;2eHI4k!#zI35U+j1GqT?i8$Iqx7R^jaq1Gsz+Uw)>)%n@pqH);2{x%Jnrs1CV(py(m{GYen)WB%OsYP{>fbM9C-Gu%qG$ zI$!DJ@c-lTy;SG>rPi)EeZ|M2G~+-Z7UJ!7t^Tzdc$wvfu#gOHM=~a#(U(YnTr3dx z(Jf!Nww&`X>v?ynRMF(|b!vf4J>vlHds8 zo|TvX))I24B5p7q7no}IFf40os>m9Qr2Nq$^R^pKZ>j>-wc{pk(A_b%9+ z4zq9P-`UbtMwB#_AJhNjnaB>LG!RwAkQtL^xQC8Cd+m>jD-x1RsOF&1 zuE;l3nC}b!>rEW5x_pD;Z5KMG1=Cez?H=GVfM*!+m~Kyw>e9hz)#GxwLU&@5Vx-vK z1lhW8M)UdALPywT(UKSVLbjAU2Lz;_*k+i%??Y^Ed`H&bN$~%Dq3!Xz%y`{7z|Ja$ zX3~tr>9jg?Vz--x{bSNlf@#(`7^Bb23erpYKAsWvC!XHF=EHYPNR7k7QlqIdqNodu zK$~*yHbdjAr+~}#Q#b<_8P#vxzR+@M2_bgT&pdz%^QRPWtAfbbSgq*^C#OPay>k zAwCEkyLb*B@km%^i_5| z@p8Y1CvPe6ANe-SSy_(d?VdcB&yfQ%!Q?#P^$pEEFXNb-7v+3tTNpx;B?a3B{vMZK zO^mIryvn`MP)HsVh&gM;MrZ$wC{u{EVe+&la2yEnb!(I!K5L4w6HDj*Tyyfx5woUQ zlh!y^Rb-JyvwmtxO4zLJ#F4$XcR9H&l29k}IxUN;%&Mw{3U!SzIJ9>F@zVJFltm}P zumitixXcvw3%6&hUX&dyu!Mz)6t0)Dh}W*ZF@4oGEXUy)T9hPF>57gJdgB#AY20{|iShF2fyv1=kxzX-eZ!wPR;WbQ&5lQuc|=BOm%Ma7CNw0*8HsR~qf`0J{gM#vtIz@?~>(-q%Fz`fw5yGX8Q!wu4^ZF+8tZU9EgL z4Mli}Kc5{7&Q`$^IwSBLCo_Vq@I1@Y3`*!p7$yyi@DOd@`?2lmZukONk>_be5d>M5 zcsazIdf4v3AiizGjggl~{<7+ivnND$4Q4eRD5k{9JR8E(ezcL8ynRH=IZZk(e7_#H z`V7a5G9!pA!|);M<4UX2S7Qlm3Q=#*gZ-8_h$sorf?dtg9Khup=`w&hO7&#A-M*pb zO&wF}X1 z!#_W?H!{5qa)f_=)(ZM@@O?4}qouqU<5Is5Gv7OIjg1jd#VoJ?ntl?Q zwpCiE1xe*hQDRWqPQd0pgV5Wj{^c?8#t}hb6^Ru^jum81;Zf@Tz}i)M8e7CHV$u9h zm+0HDVztQ_K+`mXRuvN!tN*o(g+EV=wYL@#L*MS+edK05v*8(85?P61WtwBSkokJe z%K--6ovex};&x)+ti-KrJ2%8H2&T&}DL|wdpvu6Y&pdz6k~6zUlCf&$Wkq|a?FKd8 z*b4r;n>fr(6h6D6h4o}(_@GLr!|4MayP`0>LGuDF(&FCL2@mF9|63y5X6n?dbamcs zEhpD^sn`iOFEM<2UNQa77sYt?aZou4UugL_OUm8E=$EhQG;AY0oiRIf#EYKJ;CYsn z6-nb&1H8KZ-~R6!i>VIQr?dW7yehV`X7z5nwNrFmplm23O+gsegBFuH-6B}8xYZlL zk6KkrQp70Ec?kkD_?l@cjXHg*OpNs3tCFh{OX74D& z6}DK*x`((d;qG^vxa#BA2ww0n5sV^o94n#>b%j;ARV1#2dx%3KvDnOO%Iaa(u6o;0yf;d?7b?3(iBPawSOA8Fo z3amd&oPv&c1B_w0u!xn!)wd~IKCe+Pj8Ne7s5-EFG@UUNi54YJGP1Jo8(%a11choG zNii#u4-$^tAAkF!N2K0UB+2A-MNlLoWWT8z28TB7AtG&a)i=)WsTtaPMO9WQ{v+c?M6thLY!RU@H9D+$k@KqyMgIwx82weWvC3A?bP?xcm8+c< zm0x_`Us(QLvyA64dZuXTMK<2HIA8cr@Y_pGF1%RzQE}IQTbynym;82-@t5OqK7B#u zF$fJWu-sGL{J*T`+rj_nI$2iqaQq_S1C?EWD{J~aSjOW?RFZa8G`$@!VgL3lr5qM( zz7-9&61FQXM*debO8yB^_@)iPxccGhvj~GXAsClm8m4`)9#b&SA+Cw(hrKbb7qHz^ zN;tl0G5!zh5vvnzLX`2PkXQt0IkpnkH`aERHUCJ%v=4=*WmFz?7Hf~86I&Gi6{{R_ zvF?vtXejG-D;j;_o|kmK9K1;S`Cy#X{!{bkVv!>+RhV*-);oTa=%vXI;`~1jafrjE z3SP`}3hU$i2+^;1kIz0I4$qc^xr8j>=*)=r;UGtK`5gL){9Isnct_;rI)VRp$3I?J zHFoX$g$Gct2@Zy1J65vww6KhKI5MSe8-wjecdt6TX6pyto>{vOl!iH`6l5WGpO~Qnb15j21k6H;6Lg;%;V1T zs5$u*-i zRf~CSJ|rGF*r!|nUt*l2iU0rySWXm02|bW9(TeCjttYOqkBD}E{4hP~>3SqwRnHQKt6ZZ&4%-9t=Ty!PGR-?xHnrDMoQ=5R6{I9k*z-v_*d zfiK%__zxE~`SOw44^Fr;^*w8u}r9O4j%N(66w z6u>k0SR&_#!T&!}N_hZ^C9^ENBEBA03aCW{%O}PCzpTX0$^r@hv2y<_kW$I>i$eff z#JeC4aj3MQkr!vZxQqM(m1`{AgeYD+o5YxJtqUY~#vu-60ZlIP_EDJjHmRU;>&K`E zqEHu;u3OCS%DX=+JCs*{w2HCZx)^;mn%;U%G3zr6!GjSTt(;kieio~q*5V@P2W8#9 zS(nKCnH6JuDteqQHvUuDAt_#;I9$RIw-1*M-)wV%$BU{Y<4jdhzfq;^w}fKKn`Kph z>LTL{38j2q+32J2BvkUN;F>#OK7x?>;?#TT0|hyiCQAXo2HZjiWce zz;wT7z?qa9NVQ2!v(N)0=oya2NSzv#uzL${ay&r1x8wTW9n)c(ZV1~Z5$Gn(iK-^k z=Gzn6a4UyipNg7dzANUMhda4&fVj=oe5zwNJdG0Oj??qkin%>~RXy7-=^D$K0M7|e9qV@`-5ihokjGaSXG)YD$7^C;&aN z14%O#Ln2=hUp)BI(XJ=qaP+7VQi5BXXa_Ez#{s-{fs+K?WJH~3Bw%ZYzs&7A?QZ##cU%(@VU6;_#m{(v;OY z5jSqd<(fFeAr5gU4=A*_{f}s|bl|zstPAx~>xp7<^@Ym5{P>7+$e^eX8aJgWT@iQK~Z!oEkdmY{!}161$afA;Wz(m5NQE zSlp~^$~{uOWc93Q=v+d#WD3Es+aNKQm6uL>Ik&%}POmSGa$KgyCCXqSJzVI}wiV@- zHTplkpjN4%a$do#wkOyC?qcz^X>>6Ehx~4o^*#(xHwsxNbPGGJ=QV2wuMf)*KHuGX z|3LNN_eUZ1`=NSpzI`x!t`m~3h2U-0kI$=z=Oft`?p4fwMe7SAAb+L-wSp?Z`wx;su}VNivuU(5>ZpTwfg)89{g08;9{!m^AUJ0l8q&28krNDp(^zIiCA9+FSv zC<4+Ts%+C`l~3VVl4QSq{|;((RXr>niZsj9G)+sK97=_!*OOrvuq${i^Z--Kj!{u{BMOu*U zAjjbwo~?R-G03eNJ$-K;Y1Gf%X?8Uj!}%twr8q|(1Nk4O+mqqnLxX2;u<(1ul=!{a zK1d`xz0!Q~5qnESRtt(PEvg0w6qaLk4Zzsab_0zrU2jNi3OCOE#YUu%NA6m6@XhO? zqPwwXczcHKc6oiO%QxHyayvRW_NsTszFv=6n#9GoGU5=20$6CBXErOSyv%r9s+bHl zX90k%l=bv zq1Me&mxw1FoOd0Oew_3lr7s84FD;RNDF{18zSH=Q_RqNsF8B()Jjis}S;pXKpzE5% z%UN3%RNZ!`yIQOSdji2Y_zm-%!n~JI%zTJq*0;FuE0Qj1V=KB1kwT7rbn$`B6QPA| zbzzTN9d6{i?7CC+dhNL$cP{Yw^6iYswO)C2V4oMYBxvwmy1aOsh{b(B4zWR@A-4+o zzCGjXA2!3ebU5r>qJcz9BFD0dAy1mLxa-$df2#uNW1l77ZHrONi?)4u2$?)jBiz0@ zl3s#*KNm@lK+?mI^wmguAd>Emq&r7Rqi#zqlBSXL+7rS$1&VUx{%371@9;*hO^s?5x0{SHQxPB%ejSmNbLBcFd!l`kjD0qFwkY z;C1H!I|Bq(5>1vjc;DrpnoUhu-g`uRMn4y>6hFsyyJ!w33+ zT;MW*OLqXTz%Bgg>sQ_>pzN1y;W30U$T(1ltPssr2A3v@mgKNohc0)ka5WBd#vq5| z?YXKxNzDSovmH5ofbLX1s?XtejR4bpM;hFfv8_|bRCGCAzcTW2eBngDR^-G+e1oxT z>W4BQkN=KWhySWqkuyw2&U6 zUhBiVYdjYVE9Fd4r8UN+B|}76X-QXITPeSfHUIL?7tcyNkRPVxPrQn55GrmKN}GaV z3Jjykw89CfwE4Jx{WGYhTI62nZZ(5Tx-lx7 z=tRC&&3J6fxhRfuk|ODbDlw8yqmFXvF|#C^eEO@D1OHTkI*{Bvbesg94Oki+X8wq5 zzbx2yS}xFiz|gO{z1{OuyQQ`);faD+?=?@Tps7!@lVbdy z#OU`4n0_y*T_CBL{a9Z1hPWLg4snP>9O4j%IK&|iafm}S{R%>%-vvuHU-a@VKTFWZ z<4Mb<0lCvkGZY!ek`RcF1NrOnz%&=>CTD+-#4ecUyRug$G(0 z9z>MpLCAO05I6p}EPSY8@bi||0h9YJH}a>fPJw>Ke_@pK0~lTGPQ-m*v)uPzw9=Nh zC-IVnRZKs;jd4n-RZ1Nm1ji;^lJZFvny_T`S`9lD& zd^Aw-_q_st46Pez=ttJ!vvYr1^koqu3=THm@`}y8I`H01%cE(I)s8-W%4g0O&TWq5 zhc9%u-wMyZJxPVk?Ho>K?S4pJ?{3SpC-ORnm2>kj8m()0SGZ59#>RL<+}9 zyvXySEJ-xOqW-_x2o5bmAP*gCr_jq`)VHD$&nzc=E7F)~9}aR*^Mq6MX3Rz_p;a z>zEpz{}txnizKT#?@H|&(bXI<9Bpz8Cuy35x|&nitb3EIx8B*> zx7b?ei5lV034kfUlW&(aL6T@))D%IydcJh-)yXT98e?b!Qh${w|HldqWBjgm8FOtk zKm5U>0pxvc4y4Sw6?Wwf;RukH7+TjkM$u&z2AM=dOb!-;F+qe+N1J|E6#REBdGN~m zzna_OnWr$90Oz?JZvP1ypPFTt4MkFQnO0;$dGq-NL(Q&J65%=98kKG}9Zp{H#@S!K zTM=<~NSrD&v}Ed{4$#@r|L*3P6>B8<(vn>@lm80Cww#8?&T{q~)dxHd)#c?VRFh+|QNt(aRU3W|GNQatOkzoYY5IIpd&_hjw$0Ucvp}jPU%=zi? zq5%`J0g4$#pe2E2Sb-K8hDW*7$nAmd`m`Qy9LA74_ju-ZnOdNg*9f=nDWaxHy3Dc$ zFCSVnr~kOW>QXhS*d)DZ=Qk4&)(?j*8I$jV64D-O4fv#ll5$Ep@hBXy#mu*)Cq*jz zI9#gjy4dLJVy|y)1lJ+T7B1HOZN;<<#Non1Wf_<1m82eBE4V&rc~@4l{`8NsPYI@bA5m6G0~eOJAN^B?y@=Dva0;uKz*GQgr|Hu@JPs{MBX3_@}1>>V=#c$cLngb z$op2ah>7EZHFT%7aSr1n!5x+7tquL3p+7*U;0$409XA>LU44WFE{wvuS|R^`i`l;L z)sLNuz)57sXGXko z%jt&r0!^3Q&{dtIHICL)8a2*NirR@T@_zS`{p-&{Hb5~J3b_9ZZRAtQ&f^X* z`t7HP=68ixIo(t_RpB_){JtF%C2J~qcI69iywAh*n8J*mMxLE(`6*dv<4>d&$wiyn6kJADZJAB@fW;Qyqpc+s=!ut}Cpnv$DMGi_Cr5 zX?3bXN~#jGQ1SrTui4D0gGM$8Gg}JeJwQt;0~nyQvbk&7L(0T)-efpq8jKI%?>hz; zfu;+oU?ut5J^x%Yq(5xk00Ofdva)@IR~jA=<%fCPSspdV?+!8Co#)X(KZg?xQFDNI z<+(PW&Oa?xhe}j&g=1K=mE@B5=h(-+3_VH|HhwsW@Tv|Z$!E^8j$@~WC&vxqo<-;cn3B-R>v=X=}`mtMoZ_*{_ zy>xViN~mP(J4rl7W-Z!u;@7WX;MHA8p-L>niguUkkLo_$>Gox+t}Ni)S3M(t%WYLs z;70v)3-^ap%;JXQ;7D7X=1a?xKFzY2zXBH;EB3-)XM<}BpYOBGF+&yvrw7@BF-}J~ zJHs+uTe_1S<17A?9>8l5HU%YH7~uF-T*&crL+uPfiQ}{I|N{zdMFtwEnMJ>CdW%&$pxi>k09zQR)fdgw=2NTl6xyR`~i3 zDd#(+{#Q5rIKqA!9>tvZ5oR0`ql^!V(G3+?pIGUucR6Wf11?cb??`+vBM6!mS;@mm zpcJu=T=aeY)i-x4IEEEo#4;UDAMn^k&QL^AWmI0^8Yc~|vhdBe$?znd+D)0c0-#-}KWE zXnx5JrS_u!@rq(-tO5jDWCY{mFm}C{)3`7cFYn#m4F3V7)|6 zd}4Er+n#}L=tWOi1nzI~aE8WcA}2CB!)ImRH@;^2i2~LBio1d_QAB|++I1++PR67u z;uZ3hjxU3DEudC8Ep#5`@RDc;F5T(&I$T-3-7c@slc)RKp4B^9>Yf2-Qs5x|vp=8{ z{fXe~UJ2k;gM)`Hsb0xC@fC8D{BO^>mq#{{m=a(mg<}oLkWdrJk=D_4sm&zWG~vFv zdRx4Sr0AT|G~hpWz{@;sc64y;-u<2H7B%8WQxyseewWBHCgo+``&#cx{Vq5Es5?KG(~X9baAM=_t8TnaTJG^yN!I|iHBNxTEg*2NeD-$vi&@3_qjcpz?lH`Rc}AioF2tc=r)A-8j@|A5 znymW^JK~$qp(H*#vedwt94m=}B4`@>_`74KO+UOb>itb6g?>?I#y8~Rr&HGN+=AWE zluVr$8H1r2+4!SH=503=x$5e$wWGCnWMqb0SlmzKnzkDo-jxDreM#6bNW3H(tZB-; zDT>!VGo#M(X9rYw7O?{qWo9T-LKndgV*IWu=KHKtet&H#zZWj>_ianq?`RmJUKais zdG4}s2J#+SuQAB;UdaDlG3jcUe9`h? zW(iR_X#%yXb?u}76!_k;Aig z5AeF3Q@}8f+waa7aIT*+t^U(1I{lpn<6oRcw!Bt86~KKl;>XUE@?kB7M$GEeYBOQj2fy`*MAsZsCp@$)6 zC5FeL<5h!N;YnQm!Tk-1;Dr;hQ}MdNi-Ms@GAlCsUDw_FMA|=?t?xEO9O6)Up-n0O zuPo5w0+n0sBI%!uIH|`W4i`O?wGU%u3#CvkUT&r2=a+hUmnxK1h^4aT6O}!`IQ%!| zE~D0pW+$knb~M%&!iTPjN$NMXP_T`B0_o z$2i0x4snP>9O4j%;sW0PwGQ>w$xib>xF34v6bTzv=;NUXcCTszXa4!L#?c(BGdhrX zEot|-Py3x+lnng`^wyCpg1dy}?sw;W>vPf*Yz6;iP%C>Jc~VxPUsjFKo9(OvcMNSm zAC_xc(czxy&~z$dc802k_w#&H(5m}mW=tBRfVn?cMj-nk!Ag_ zlflo&T50Hp@#aNB5`Pux{e<&(PLe^6Y)g>$7b9uZIWYy(Z?UG)dkE1xKhvV5lQ1d- zghI+{Kt0~EZBx26_HGxMRYZ6^%nZn-|46fQ7D1O{obp|2RJ6a!BG~u22h{K6V*)KntoNnE+pC4rmdkSX%29LCAXbO33IuyOkd}bSw>_2Evi4b5tB6ez z73Tcm4nyKpv?;39F*~jFQX)7VTI7{f6S~0(V>>*m8}6hg%hL9#4wvDc>K&e^dGozK zkTb~Pngl#OJz3sYRyFx{(Mq-|^w%!$$asZyXQ!HXvv|N_DFMa2XZS|qY??y6Axh%&i0xhbFWEOb*)@pw3PoF;1IvEmHXEGS0fIgo_ z22TNWO7lp%1CmBh#-ew^AQ4DHVzgH)W*S{RP&ZK2#XmL5Y7;G;T+DQ|)ub>)07o^? zTeYb6bnI#(vMejJ0>ksXByk~EHNIwrznga+}V{?fvK+hY3;_dOaGMTEA;OAFSWj+06>gappZvM!G z({M>*DC-hf*6#2Q%*g?UL-m0`?6speoqb6Bd#>%-i(KCC6r*o)G&oI0o+gluTyufi z35B}Lt$NlagGsBn&;|3nRuM_#eE(tcf3u8NxCSx4=PoIol<+&0Jzf)~zUT~Rqbx75 z8+(>fKVBV$zTtdoO9n4{N!EJ{k^JC^H+Wnf22;2kvy9*XvRr2|+m#&S zcvW^NWaXWy>Z4Y*UtSF^GTBtv!@mdrhwsDpAV%{k zJKF3*NlAYX``Qa**(?udJ1aaNrRtN^EHE%9*X{9nNAzC!tY1irHI!pQ1<-^Fy7BiJIk|=S8&MGwS%JR>is~v<1)*BQ-nkAm0k#Vt0mSRh0d`IVfYAk@X^Q;1@9L$0UcSBRWf(y%Rw(bg$to&eT=+kd z=t(_!+oU(%xHIBmZcSzwMv@gx0s`(Zw+*|V^r5<6Q)iK$963~s${VUC%93I*A)#WX zMqRV|QEThex+Up`$mfMcM&u?C2JQ+HmKW`VoTF9eB>#0^{@X`J`p?CW_V@#MRcYE^ zylm2`0@0~w8LJ<$!a`)eCx@ZVqEIJ__@9VTm!qWzP?xos@cF+yB$6VRC7r*SLE?gL zX2=4*eeIKS{fDKYqH$%D$T%8zA3zPMb8hw*f7c5$+jOVf4 z%QmC;I}g_U3%fA`cu5vbS!G0CLGRG;z8UBrn^nr7`D9q-s<5y+H~!UNV!>5U1Hu}i zd70M@5%_nU7C!p)kH^>5tx{kFw#H)y8EjoZ)1fll)ztVmX1#RaS;*Gu2qFJ7XC*>Dhtnr+PVDhQpO5?V0g+rv~pN!iID!qlyipPN&^aYy54HA%zLhlEW~9Jx6tA zIYCcvzDw^pRdx7!t4=3+GuN(|HPaE)&g;buZ1;l59ih=HgCg{rD$58uuPd^$DKlmL zqp$y+4E2gzIun-!>@22fHrf@FqbynFTp5`aYL)W)n=$R%+OCZH?*fnC4b_P_{D+~C zxs9Z*eP+gv2~+SFH7cCIN(v*fjKr|4h;F$awIX*lOUD&4n#$#DL#{ZC58cReoXE?( zASt}8NFkvcPg!|w#rffHsIq6Mxjwrl8#*VkJS&O3-I0^)T=qp~ zO759Pg;$il5WdrJ1_bZWqX*n;f61>=@ZSV#@gVcD2X4o1V+)L^&;m{Sts4w2qYb{- zTG*ChoucfNr{=8LgKw@ID$@Iv3YjgcQ=kHUS8TR7P`=OFoqAwpM5R;UIYDDs!C*}h zRXVo{0kw~PB9NYcN7bNoXpuTaZSqe4`9Q2#<407behwE{~oi$zx*?ggk^S4 zEFRHQ#3X9@poDjSS&JPXMG<9IrbU_;M2_Xr@kzs|gs_B3)DwGZjlXh!^HA}rh_Ygc zyaq%;V+020xnjfUmN=agCtFj5lT!Y;aeK3sFq~p8fzj<5(LNmHs4kyFw}Oz_Jz#Pk z@cPdGFY%&kDiWh}DqnSc!_K#^WUCfjs*qm3+{#HcX#dWpVKZBXl8V&?Bj07$ovPPs z&-J){Zr$y)GqgXpI@r7_tEl=pp*>klMwvrhlXl(P`A`!pKUw`)jD}wm| zx6|;BbQwT*svcnYf0+&!FaTMdny}xUc6~9$s6#!%WT)%E|#`U3z3TT2@ff{1eKV&M(#Rzu3mLxbvN_q~lxJ>X{HFZAMAMbu}H+ zzJZ7<8~Psl{)Y76?@&74fXwgCG2Xr!?Y(I01^oY(GNRY=o@cmmtt{=Z6_d{XNg(17 zR=S={wR@=5gwNJT-ri9}T9i0}2cm**@BA|glm~7bN4N#QF;m~ru)PNsm$@fjTEL4RrvfV+;x@rQEF=^edx$|5Gs?1+6;R^6k z${*u;G)`(BY7{AKn5>EnWAF?MM1b?K7>Hm3lg4YGc~tLvn)Ud>2bFv|tdegkjH)rT zDYB-3n(Ob0(Fr?``m6qqhmRT&-AW9G=XHaXRTW?x&6B~HAi}4kxs?<;Kun>Q(X;LM z@_195EXo|s39QI7j4X$kg>o!w55U-OrcghB(|zck_O-D3vBTx?*;!89_l$gH{UX%6 z*6QZZV(ev#m`YuBEUR|?n>&O~Sj1OAQ0#-8eRCXswb{E>OkEu?)>({rjZDp=UKoqqN*v3aefBZ`MxE$WZYRB+&#Mu!6@7o zm6POCw%voKKia0f^<;T$ed2IA9WIcmk5C=XbxocayY8xq>Cisk7s1hD@0azdu)+bq zk?JAeolqwwj7)A|FK-3xq~AnWbcou{|il#&XSZRi)Wu#{E0k*fRX%$aR& zjOaroGOQ@kCc|Y>qw`zR&^3J?FzkW0w`d{2c1>3G()HCl*zVoBL zcta}7)1oB#-Ai~`lu(YgB_lY;&=O`+qdG6_@>2C$p=4K-rwKgsdhe|n8-H3;KxdGb zeMXj+^yg60t;c8VZ*ZmcWEzh2i(Xed)#1x7F!=TKe+6(}>9p{_@8| zt5Q&ogPymi5Kh&K2ru;vhczUFS1RZ}2MHQ3<&R=kls1_1A;2pzVLfMC6=_(^6#>l` zw`&3Og;vzjh-ldX^&86sxARU*dweyQFv(o;YRLvq%Kp|KTE3lo$Su`FETstRqPLjJ-oNy3X(ra>!ikE)&@4+c49oFsaedcFVhPo?&Z&7#UacQ$ za`9y8rU&@#YJ#LQyk?4;s-sS8(c(&H#JFuh5=*Iisl&VV{JCkET_MZDSH$KBvcR-n zs>6wH4aIs;E3_x>p&qZ6nvwQQ!TG+hw_IyL6+~9xRZe5SmZxbiJTWxIj^z0aQiuZz zotB}uv$aogL!00wu)Wn}yLA?C+j9Zj!QwtYTo*h>J4OZ*4itsfBC;kt%mArjxiu&s zFR+%pe*y(8wq%oO*1*6MXZJM>?z>Y~y)&!=J6YBE1L2a@F(U5}$AJe3*^+mk`Z3AD*Bp*;i>_@ga2=}D)76NK4#T3{~l{8x2z-A zbqW@}7o>$}VXLkBcPiuM0zbF!rB02%y?NqUEFY6NmQ!efV`X0C#Sp*qj3~_5M%+&w zedFIzNhc#8DB}dvFfx?<82ZG?ENr#GpB0H#1$zsY#opP$dE(H~}<8!lsBkvI`(nksSnTmN|PnSOX< z0pClHQH4!MRD;+>f(NMNx}973vcv4c7e; z`q$7nlR~1bI_U65F7WvB?V|WW-PD?g-~G%61x*mM|I8~ zrkhYcBD{&T4o1RBjPe}!U7Z+B!gXRa8OI8-=95@X`4VN^o;dusp{QP#6r_=?qn>(l z_A4)MiY53J!von+fvTz;g9g8z z%VIB|VL12;)S*XKm>*ozBGhtc8SY?=49Pwu&*@Y(CpaIu$&1?XK1ND-tAu(!MZ8FP zSFO!okk>9$_rp6+Ia~uAz@vKf>=19gr$>Zu`-{|;Z9}V{nu+yuQfUJ)Dv)Va)loku z_|Y3N`oEIIOVq|6hI+2LH=;_*=nTUv3a@LNfVN{~9h}`_12yvQgB|3pk)5Rt)gNGl z<2XUpP-p3AH~GP`Hc+<_eZJoPAUrm|iwHZ{{dZjqbQxfpT~h(A0Mk%4L%m^{^7-uj zWL4af^zl=c>O8u56A6BY!nMOoHZjrfaHP6{x^cYuOd~cH#;2)_W-36Z0mq_@ucU&e zt(4!#ntyrci;w#EGWB!sJ&TVOoB}u&)*vUbK-B@Q8KS6d_~hV~OOxuQ!YT07j({aB z<0X=a!=)j91TPn_Q{(qt{`cv{siAz;$nhVCLHX!i7~b|erEWU?=gdEAg>uoLRhDJv zWhOl-q;07N2|3zfh{8qS4eIVK@&}KehPJ8Mk!@2tt+S@Cv4SWnkH0%+n%@wKGdLc} z2sYJ17JXHep8Y7y%nM)ncc`ohQo{u=HNbX|e+APxlQjf`Q56ndWui^N!lJfNatCr% zgEu1{(9lIu7BwL0w62Fepdlg{1u<$p-lc3ekn({y5fxtML$=!&SQ5`bj6nSXRrmU>z1E0$n+q@Uvdjt`M+-79C};tlPX+Mu z1RPCnr`C1Izj<5D2mzR6USK4SWoVw3LvK)9b~5uk?)pkIOgHVIZolsIrbXYvb@$uw z)@;U9BuNuRURGaug#{oU9oDNUX@C_Rk5(9ZCz6qw$^(sV~N__}`>! z;LHdUfX4%Lx5u!1+<87=S2HCxzG*uQI<@-vsU;bzMvK@c(?jyHDdR$ zn!$67$}luA(e8PkVlmzGCH4E2XU8o60+K>DKEf`;Ypfu!em2FV_f2{0nYT8iq1*L? zin+cas?o!PFC?u$+U` znogrWUA+W~(iKNcihx>RmTMA&6k7`gT91uTP@0UT%hPmg>-+2 zdbpo{_}{e=r1{+YVwiuU>2AA)mZKU#Z)l-tVdRPK*k$q5w^1fTkIS ziBo_fNw2hiKo7iF6<{eLnBi=~deR=J#^)T^HS^oZc2?Omc^VjsV4D2g+6PAZr9jAF zPFtqU8>})8V1=D?)RkR6+4;;-*iq}FPyww9tS$%kW~g-X|v;vjf{YIyyd%7cN=_4(41@tVeJkpY{3`>_`C+T9*MS|Szz^VUqlQ_qR=uY@S+4{UBU$=sn#wS6GZrQ zw4fw?3jRyB)wp@jk~w=}Wv2z(+oXEtnl6d;hozur+G>nR55|9E=+d54taM?grP$WL zwqjg&Gh$pGQ8hTAupFyvAZ%QodPB&FIJDgAwlS-0Z*3VHK~`sYQK4l~5EV|6Llknt z5~LR=9Dfo2NV2lEMo{harJj3?- z(Q#ReH^ck-!o;h|VR;D#s2T0{7~X_iuejCkfeABxBqf*TWFaqKkOfq_^L6cqQ4pR&3W$X3kf6}{y|5p8XY`aScfCmnl` zNET>bmKB<3MN#2Hg89Gwd>~(&Ej+EZwawbw@7ML$Lw4N^$F4m-y9`u=6**QkWqtWi zN9UXchpRUuE-oM|h~H5}d)wCI$xUwmy! zjf@E*@^e^`_O{E$xSRa_5)@Ih!!aYjV#X^P4;VS1CB8lAn|8lkHlr%6O?r%&z@+iq zxRdQV`@DBs`63)D<^+!C6_N38A_yVe_@&g~2BRI&$@c!D=6zmgBAOvZP8M{UmL)~i zQQ`C{8%{XoY~z1_BH#WOJfdAA%>Bm#%h(;~Z$CC2z|&jJ^8&j|^*N@1A$k6TP(vSR zvQ~NVi3Hp&Vzpv?j!WadPb_A6M2z{66*SQ~u=i}oFGqJgWWz-TX;H#vw5TNA3A;&<3R z_dHx9obCw>UImJ-%M7D^Y^(A~jo*c&O=Q`u7&pgQGQ) z#W*jh3$!j#G16^+#PU`L;j8H8L(MYc`?I6WFI+!C6|yl>f1-@mvuN|bG4dSli`0lQ zpIFx+oR56tV)7}ngyqK}4wn-6BR7*pmF}d3#Kgozf+TEy2zej%GDF=QZBgWzXnem3 zl1_>-|7Lo>)K=fj7?>R8`)JpLk>!*h?t3aqd#DP|4{d)+l<`g|;rqD%$tdaadY>e& zvK?Qwci0O@@V*`lEAk>O@f@SD3Kw!U^@oTGh8kZ;qPOkfJbQ!oFG0^QYz#wzQS80_ z;ZKHHSJ!&cK!$_o=;!gRF9uV9mx^owB8k4X(GMKm@@f+pJvEH0TR4v_|7|9{@_oQN z2)MF**}KO!SlhJ9wFMl)7xQW9{x(yau=@}XCG3S^yiqw;3>fUL0r>%&U2oOtob>Cr z)vu}jWEwQ}K^MqpKM!Ov2vdjW&jqL7veNx9b+db)UKDg?*1i_5l|d zLuVMq6gWXVckbNKMQev8Lh}4GB^aY+475G-$`hyamtl|aE0Q3vEH6tUOSk!W)XbvZ zf!nP)0sTXdMx7K_+wSf6xnav`fp3son`qBgy+fzE?2-vMj^-tXRT;-?=Xna|rQ4m~i1Ne`;c!#>2`NQ*bG`ABiIVDQTLv82wp5V8b4MVw4kwIgCK+BxU zi9lgaeAO_|?|KE9jtjxD5)4TUx4oG8>HP!n5gvI#R(Ot6Xqr)YA>_KV&D#SwXC|(M zk+!e%UR1AYi)G&&FG;K@vOF(xv?PR_Qabr^fPFlLb=)0m+q$9Oux)QNM-~rc_ZLh7 zo_xC?^O7v9KooU((TeXj-@j{XGA>4B9|Yt6FD~G|>sdw&p=TK5WB7f`akdSY_q+KN z4HuhzSej>e`w-RD+id_N)9(aqcewia{XOz@U!KR|${L}1vVd>W1Fyfi>+4&p*F(g@ zBqWUsq!8^n5##f+X079FuU@}D|LU}ehY=N>Qx#d(WuOY^VZ{Eo1?V}&>Y0Bm1^lKI z997?7Te4xqnP=xjbU;-kL6`rgL_0N9y@;C31waS@>dY2ImlEg%tctojp%lP`CBBIBC zD8b>fdrX5-1;Z2-UDv+*cf_2cQ4?xY;c;k5PFvf**R*l;i0EzrG**;ljx|+2u8{E;$U#+8y5W&VB*0cBh$hYD2?=39pv%*w`!B+&wDF`<3PL z;X`dpXU2ul4KX)X^d;m;Ry*d$_+Dj$?yW0FJ&^mgu;$~Ym1G|qjwif^<)^m6&u5mB zUSfq>F^=c8#q75!m$)iR5^meZUu|=nOR&Sj9**T`|K+a0Yn|Hr*gMES0w*gYAgKKOtOxwd%=M$R8S74paIFftY^ zNOseq0gn@Sy*<78F1=r#OZWL>7QRLdy>8WEF#)=DRzolZ^iJfe#io}$8`pn+|7X8X z4wa33hFxl{*ZHQXT@1qP#Aq2$u{A%kdO^lsD8X*55>A3OIbP8eps=P!pW8R{<92)R zPo7-H%rI78nKNZn-o^@YdZ<>xOeR{pDvPhj4>8iu#a7>om2NWs~@c*@tb@nVmUo;hr&%{W7H^lh7 zwDTsRjK4cu#`)ps=TLlp#RT|c-#6Kg+Ky$q_P2+W8b;gU><(|=X}KQY^*Y?H(GFj> zfB&x_JX1(K_|0iSI8?!yV{3P1)lVP!2PzmJ5aARIL-EHPFh$j1)Ts}GdbiI%m0lM! zaZ0+o676|e?Y0!Q{brl<`}e=YW7&| z*Mp%uz0V#Xz)fQV`1qUF*slo2JC~HMhv(YrRc|!E*3roElO$j=s%D5huh0`}a&Ofu z5<&Q`n8)&V+e@!CxO3Nn7*1u=636O-$YSBW<+D;R9Yfn3n7L$9$>zmn#odgH-*1 zb%FD}YwDLtQ_y@0r{hxFwfQ$ZDN%4gVHd9I3}^4@^$l?wz@7zs_Gz4I&s9BY4)6hw zcj3HWQvAW&p!0aEB3)K=0X}T?TP`15Kj>tiVD&G%$HH$0zT9I?q8SK(+1M>P2(Mp5 zmI*&yYJ0eOZr12`BOemhWKPsM-qcJL-L;di#!c>fZ42c48?4#@&9QPgbF9b)yu*5{ zd0E*$|FY;dM*y#H_>%X&s@3E76ljiZ^=RPhyhq5Obp`E(C!)0Xj@@WA!{>9cL1BB= z*!p%~Z64YNN3^JCVqw4$ATx?AX|!pus6S#2YixCkMxU|%*lZme`L1!Gr2oFH^mqoE)I)nhH!4;+Zctf1ezw@olu8Hp}|m z#VR+}?YmUKib-S5u*P~EaAbI_9PF4b@pV+u>$I32dcpG6ft?RF_yMcQ#gHsdk~~S0 z97WJvke1~~bYZ#yQ^gAw=chA21WUt`7BZk&z$-G(vO)ew{z%3-b9}{elVxt=56i>& zu@_j85*eOhIi3)L+TK$SnmG*X%yE1k)DFkhSY(jvUuCJmaUivit@Mq^ybz%`2Qt2_2JOvAnqBwPeVFyxFB}jV|l= zI9NiPBl_LwJrt@%C{d}JT)Ogbyx$hf_G)y!-fuPzBj@7$DtLo{SMzwY{W=wX)n8P# zJ;jjV*=AW>r~NbTwl#wBMhTyuq*In$St!b|U{~CmmcQ$~cD(uk$c>z^;AztWM=}Kw z6m{zK=;GtfK5wiyOe&;5K<(&({2WgDL(~5xd&TS7C{x@k?VfvCTrKWXh>PuB{_-Uu* zWg@Nr3&XHxHM~gp%q;8{RC^1Fv;s=+szb_>Se^qUZmtGb_A2nFTz_3M9|)D*sofDVs|wUR5*TS&9d7 zmnEbAeWU*VGK2%tl4ONTojB0z)F__j08nI^CEl4owDa&j$IR%$9IpkY(1!$?8ZE3{ zmT%6^uKn5my2y6{w6FkJNuhzCB32tuVt+b3vTf0h@LB_R?hr@lo!kUFHoGm8YqtHX z+PyFc*IcaVmLZ*kC-wqvXO0(Wu8D)PBzNw+|7*8<)eU1)paKmV{k5&0)Xsyk%je;I7+IgUo?-9$CiKY*?jEUgp zju$jaWdxaFMGALw*VsTF)i^?VRpTh-T8(2Q2m1gcUwXHk@%KO6 zBAhga@rS%rIL(iDoL(@&e{VO$m+eClz=T$~Q|24Xo1ZRvjdm9q?^#0OHBulnftEFq zBE_}Gj*qYMBa`Yh#UT#G%Xa!zD&Z};g-eq1-&p>a^VN3Swt9Hl)*q~o^qbzZ1SJx3 zhlGE6`>7pBL%x-#%T}hm=L`jMY1AIb39FuT#zS&I}<}oM~caj zUh zy5I4R0=~C288zj%E#E*HZ-@|i1rj$exIGW#`qJ~2EYLM42e{Q)bGEFhaeG=4WHuiB zQa`P~46OSf47oG*=FrbqmSHQFL@(c4W;^oET)PgQf;r(?sX>xA7kHhD-wRMzpC_Al z{{DW+xWCgd(vPGvVv^!@c*2s_w_4yN+zEJ= zAq85X04Jr681~h{M{Y@lLXi(2h;D?VeLqsD-llZ#;mN<>Eswl5X^k+oUhu4xSOF42 zi<-=-gdl6x_n%9UN=(f{OJHhz$@7Qhi4IfJx~(trfDu9+gr!(#j@Om#@&^XE(Ryb=B32>r3}@+-{fYoh1SA0@RpC))br81b zGnTnI315$WrdqHNPA}T!cKMt%Pj5}%S!?6TWl3;{?l%Xfr9Ya_T80g~>DQkYwltz8 zNKSC3OF6zgFX)-&nc#G(xqaN7*5;%QAJDC+yyiM8x+{iENFdd0vA$QG`#O14c<($K zuL&H>P?{`K=zN?zk2^3DtvBXxT#(!`JeeXAEHAN)qyUme$&~h5WaFM_y?#?m+kbb3 z?*$=oiYAK+NAW>=uM;h}UL;$;T798o)2(>-WjyC6@>yP>M4lB`l&^TSa`?t4TivI# z>6GS-(UPHArqK6f_k6dM?NYkCa&!E)5d|@MyGa?nrChFS=3i9r!ByZ9g*ux~jS>&T z@2g-Jd=+c|f$M&_!`vXOAW9Tvnvw~T)C8v@2dyD1w)kq+4pka{(B!l5=t2{LER!lp z2?Sd7C~VQytvj9CjcxP8FB+cL2!*92o~46O(U{7(vTIn)`OkG8c@`ss7lR^)p2N!E zTUsmME>D>MSQx&Rpczr%7@FW{S`2d0Rn-!j|0u4at@YvWuKT^-cg=(O{y;**EqT43 z2~I{*1(p_QUgQ*9gz!jVwY97$x1)6@vCUWKC5ZNWherF9!2jAmt*)O2hF2>qDo2O* zi|eGUt3Eir{aGmzbuu9MBZT|IB(lWitWIXCFdb>ey07DwrR{b=oYFf)JTj6boUYvN zE-!EwD)>yZZu`4&wR+Yb`bWo0*^II(EyVi!Ka&FcGf^L>o6Kb^H`dA-_Hrt>yxKbv z>vxf}jy^vzf8#uO`Cheh(ef))tvOXXbtsu zrRY)!#}guZZ%JzLru3iqb^K89H<U5@$rw3e|-Go;~yXYt8@Gv z26_7xp0^p~t>v7Lum3pyiI0DL{3|+dHVM{N>W057Z|G&~a{ z!e0BUjHWQENQ)w;q1XPA$^wgp?A@)L@3y~x8VgzOI^hC%FcKkg8pm*gB5oNl{fCbh zJnDd=q9sVUz&T~SVO&y9;Rmm+9-HrQDgxcD3-7M{)?*(v3NN7aEA$EiM`;4dF@!{| zt=nVLzjS*m8ceQNQf#|#R9>|8_0>s=VWpH3q2J*z%z9W8(`vs6IILk0MbWHC2?8xr z1W5+PB1?X;u)L430{`CD{F}~wmheu)5L>TE3y)D%UZhBkVQ8|z7UJZHS)cp8Hd!&^ zIGlK{xZXb2WqWTuHLM!E4;d7UtNa$;1WO7EL(?)vD$ks1_{OIDiZl{$F6Cw`#sht= zO8r$kH*5`mWvG!V%aXhZ2$jjmoS#zf?|*IZ{w_<`92!?WFh%R?YrXT8A7483P57E5 zp5-N4Wdug!(aXsXXGPkpe%7>MLptV6#v)T*zhAid~%>G9Ltw&p~jK`8cz$NDvOfv_POz6 z?%q1TC}2w*;t+>87zGW*y&vg!7409)e38G87I1V=e`}9V|JNt!LF@^}(IO*A8qX3; z5IgWOn-EcFib;{d*2ikR^XxLVRWOkiO7l^W8Cj7Tg5^c2gNvq653FBXj85nw*8NZS ze$6e051XkK05GkhIGtONe@JvMDIr0Oz59m@hokc36mC{;vvzuG=X=-OgAX++(u7EJ z1jln6PX&4Tnes|u9sgp3mW{Cv93alS|H}?ig@I+3tbE&a&4F7}>LNYb6lERx7A|>mhN-Y-ig3qThauA zCN!F$RN2U2`{<7Fc_T1-Kicw8cudEdXPT6)9Q1oUy7pQ7HhuLYk;iML3W7*71S`-a zNzx3DBGXNq3Ja`;Rf8FCedbgLzH4uMpl5}kB#q%1nP zNTn_tI8|9>2u)RJz-yZa9&G+pm=L~p1C&}?y&~Lqv2u|fx#i;&>!!+m{%xEP*3FGd z5R9f3_?{3^H#exGIHP>=q}^?;lQwnZgdz_{CF7A6h9lK=rFg>G)8&@3db&WCnu~@d z-K8JY8;~@(!fDobKY6zP#=C0a#Wn@bDCGPBQZ_(EHcQeXU5)o!&;EM*M++x~&^yjj zy-tDkyQK$Smz3qY1NftlqvS(NtSiEqR(@!QQFU{mi9pqmj+EI#N6NijxxNA&vq4@@ z4)FT&-OrqzcMH=b0hQ1~zs*CZB?RQJR45+LS=Q;buRn6|yCNqgAw23bBwsk519SYu zgOoKO&wsDd9cWTsme0>KeAILqTYcs2)_bn7aT*<=OtxAn@nEb&aN(Le2Ejp14a(_y zk;mPW7~S~18zWZVW{N|_;-A^pAMfuoQ6!h zyvb$+!8yU@%Y+eKySdz|%bk&1@GGDNf&m%>1W{W3*BBdMGaS{+x%s`Le6M!|G8jt$ z=9_Qr()Z`t>pBpUAgefoC`tjyOrGNKqS8d+(jkIq{D@1OXn2$V>R6e|(}$B2v&2ur60Px!Oem+=ydV4~vrzb+gG`P!M##%-df9sZXq|4m*M+~<5`iW% ztf+~ArqOGyhZBtISZ_Tv{_kJj#cK-6q^tl=C1^oqTkg4K^6#r_7KsVpiO}&vadlg* zd6oK(JGMN$uuBzqK=B%_GCV5miq^+N@g?DQYu5HB_a5z!x5VZch7>54p#8p`sUVr_ z=?4p?GI!jS7_y4()=s~@*!<3nCU_C37bwmNF4dRmWF?X3fGBHX!NN^Ix z-C5m|dD~0l6CZTO6ot?;VIDzZX{)Z6G|kOpJ@_pFXhcQ!d|s}@WQl{|1sPC&J>a0hO2 z+3#Dub4I=Y_0I5Ii%J7g=1EOa*&wcE&PsEHhxkX<70l7*V|U|qvq_PZ7={-(Nh|v; z_a1AV#wVJecsQ&;fsj~^70EX+ z2Y4m7I&}4i$69VmOhO%XQuPAU((a7C)*h`_))k)!tBp?k!_rBDtk4vPYNH$3b_Nge zeruQOCR>kw0|gW%+`#!Giqk}4MDaznTlbnf^1KD^vpUEbo`j?q^xqrlsRP@91J+J` zPmZtreOhpzOKuH@P|`uel0G?GG>VoR`pjw>H_S0U9nX?5B*l|F$r3a#Fno|s-D52Z z>l#uESH&0B|LQh>FS|bUO*$iZS%vV9tGaT>>Y7@W8+~Bzl>JT$^gV{&!NY-s^EJ7Q zNZEoNt;5zaEgnBTHWl($hfEyTOxgISrDTde zj}>fvv~2I=y*eMUp8VjAUk~2Y94{frrk$?bfwDiUy?zHXNb>oB*Nr0f>!;NUNYM#6 zVtt}=*Xa`{HwaS~Nea$fpQMZ{h}YK5GZ7U?9MhHB#$!*9FRgQK{$6R-49@b$<`5`{ zkbz1Gswi4h?T=boF)t;xF(}o(za7b1}9r=}3hVz~E+tkb#pRQ3W_@wr0dC?qEQqAjd zt;4ncs&ur7G&x-(bm?X+mvF+mdhQo5FHUL~iZm(2Oi*7=aCXb`DC4^2xw2IMc?vbi zt2Md#M&Fh|P9D+~_Xd27W&H<9&zjwN%WCE04n^HA! zD%vF(QTsny|C#sM7n|=0a|8u=pX3x3P^?C10;Qqk%v?ip^-tCdb1EHevlyNZ=RzDp z6weg+RQ0&C@(WDyhvw&iYkSvw{oui#Hh2?|?=D<)AKF4=pGdJ(tU`B}A8-4!b?==+ z?|*y^i)Y95abrbizpNhFERvnqqmDLw8Vr-Nt`mmP^WA_r*TP@uO)8(aPMGwutOG z%;VBn?~mWD$0wayYp!SDK^v%mA}Cb?3W>7x=`Y{{qo1>;o&TtF=P9+YoMZvlm79|# z`AF{6Po3Vs?;pNMs54VPNPFoIvdu<2{U7U*?XypUA6g(clZa7#fdQ2!vMk^jN>*j5 zz<{cH%U?P!JMSnWpX)?8kVa7#YhV!PKh~V*zwKk`8G*4EU=@)Bl19<85YpI-#BQj6 zts_PzJEtGM7B5x71&8U8t4KLu2=G0(>(&qau^wTmFplvAJw-%+L0USAn4_Y&l^3k{ zwx!zoUc+yDlA-}iJM1d1ooZ9iNeiEFV{*lg8*KJK>Xg_Usf3_}YfCozCyPzmSj z?F-WddUsps*lndXFdtmC-F9c6pVytf9V?@gG?h~oP0$D}$P#6*UTo8dacH8=G`Pyi zu0_FBehaw^L=F%nAu_ZoX_*(=e7o<%Y1UE-%OVcg9PRhXF|Mx|QzY5$Ji1{0*&U67 zr=*ySVkxH&c(Yw@$p?l^aQT$X6EAk%b*=;n-O?Ie#YwikGgRk&W5Xj`O5W_P+vu3IDsKVMkClJ2d5M7)$M17mzGtLG}=+EWIMZ0cz>KV ztgIAh6h~;3K(dsG(%u{O`>jut1FyS2O~x(t6x-_azOk*A!OaoM(>6;p&at^3_h68l zm*pEM=K}9|px#xbhN^BlZ|*o)7fa$6oVBFF z$ONfyl3KmxFV2z^)H3R`tA?%h684tqTZ^3Xg)+bs38xoixN?2KJ8(SkW_ctPsOeIn zjIe-_$$#acLld68rczsjzE@pbn4VcwnEpB+8AC_;2=*>lz{QfUVf(q`qz}g(NDP+i ztJ*jxolvEnwOYy_MK=Yl5(85RD-dx#>OtUjbNDmcdn1=HzbbxZhDUH#KdhY5Q;O`*3?W<8?zw8aSfRvU7+} z^0}0rz^7!Ud$MyRzc9$ZUx}%wkM8%%DV&bM6AqQ{^8N0h*%XJ!p_@5AUU7)Sl?G=o z#K#~Gafm}4Dgs`=E&T7yr1&^oir}q(X|da}E+<;OrG)rajzb*c5QjL#Ar5hfLmaMp zD84doVoMR)4*Jk%padjR0tG}FND@Gmfbc<(TG*9?sNr9tTj;0ubWoRKt*ZQG*3ky!W7Pn^F+MJbNboBOyweBW3UH~jjumY=uwC+wZ zNQ3KbXWR1MlZ|FvfOpeEK)nT)Q(k|>Dp>`9r#J>Rmzv=Sod(y`!8Si<^1c6i4ccQ2 z57k3|d@NC+NR|?KXbOWG!lcy*OnV`EL1{Jc^BSm-MA}w7skI;39jh;-(2^vx97m|K zimIC)iV~eAm}pz#k>9Urauc*{2;r}+q-6sTSVB|*kQAED>FG-GTO`3Tu$0ob5pDBz ztFlN~XJD~RGT$8j?7ot{jnwd}c)oet>z-%!Pi_~fwT~!sEKLE8pE?MoD5CfmIR@LY{HrCQS-8Z zUnA4+X}^Xr_t%*VsZ^-!1C?N+RfdVR@=WpW=w=)7<>{fF9t-PvgQYoAlL?s;#URfc z_ph*EicYuHoVxP%Cyzq2nbsk+s5!#v%1!qI$p>`Z%0Aith&KGvDzCWipZ`M&bXZ;m zlO}{%nx+yCzir6xHq84dm@z3@z7_M_g!$d5o;XB{0=nB?nLqkm)hEOGALLj@5D0;w zScVItH&YC%=x&=nh3x&{TBtENH`HkP$GfGXQp0*Y8tj6t^L7HF?o+eNUj-fbz zXl+ty^j5p)=5KhzhMp9Y%E^(nQMu^vrESMDb6{FlK+D0?5WQ`QHC?QAeQn(46-bex z1xg@zk!M(R>)u`2=zQyIyWxW-^G37}$Ffn3#!I5Ak+jUBEL(PJ=rp*dA+}S_&$j-P z4Lt(w4t7(Y&2XaAo9FhqvH=XaQea76Kx7YZbMVws_xGuJ<%ATBCbxG5q3)0Lt9?g| z%KH)Xe3-#^PK=7dOGy6jlt4D}1;n=EC;f~!>*lh5V$jZ?nHe_P3;lg_JimR zwa0%7DQ2{uw$P2vW<6P$e!sdf?WYHkDT=&=sV@mr{ue}G3c+H)SwKL&d+*e-@@AyG zvsDArOIH+oi|x>{*~$NAheJ9v!wW1+0FLBD6zLcdpT#u|w_Vp}kmdE+^@8O@C5>_= ziDy+_RshM)$VzziNb0Xe_Qwe!gm~cqkIP5cIv%e5Q~lXkLW~kfh7=i=Vkn+uh#+h0 z^)D9|_!Bmg>20=NU)=fXtX_}>3*mJwo2FT(B4uUCk}|HFl+nMvy+gIEq2^S~szg_- z#o)ctuZl6Yjo&?Wotzrh0~f>7q$DvU!|*)L0~ZWDXB3bw?t$yl%$rTJ?Z)%zZ_j-I zD@$cWo?sM>W*89UUh~EXS<7N3K(W=ImRaS7_wjxZ2$3KtmS6;yqj-@)cjiJ%XzmtP zq1dW6S@_Ln5zm9sJSFfPL9;Byi+oT(0iSLzb^_~PCBsJdJbv9*PvHB&vIHSe9K$gz zLDMu^MI-(EVljQlu+iH-`mdTNyn9uSBqUYgMV^#`+^e>Y7!~+z|5UrV+Nk3#g6BWL zmjXd4l)#FNMim;!y1YQ~&HH@tiJ$(y*9_;swfTeScq4C)X~e#J52%k;3_BD8gG5!LGeTwalUTTEI5be@y7-_&W-f zkfX2F0^E@kLm>fKAyttlIG*O!VsCp#kJk91&GuD^kR#}Yq$W%r z2fZS0^qsaoS^Mp4zYZV8hz1k@6svFqLpk?ccWp`R(b7QLkH;Q&*)lVe`gf}eu}7T{ z2J(I%dAT`$FL^^ed0wU9B4B84tvbJtkr=^(+w@k`)dj_4I^Fi^n;A}L_lWOrC`nZ` zLSR)T$jjl+uNcum_uAI=IpPLCK?=N%B^3L>IDtQA z1kmxjvQs!t5?O&*p8rSfS#KsnpXmo{URuj5xt!lO0#stySx+NXrn_5DJ0Q=i#W6=5 zE=fA$ew%&zQYxz_R^S2%h0#b^<|$rA1umwj~^hZC{-^(q&i2u<}TeW(k?rWSJ8vR35oT zzgzrUAJSKCf%)JnpRv99>7%3H?+r2Z-QlTAKq+fAKEyivC)^s#s% z7$+A6(;-!bq$mwg98143;&;z)6X#SxeJ7T8up3j1%8xW?j@h)r)^b70scW7Ne;_J^ zNYgwaYaEXrh$lu_F@wLxHbV6rdw4xO5SN5J5Cwwg0jzILd1jC znT&aEFC*Js+@}|8|8}qb%kIV@hRaW5iGd_mrj&~gS8FS-=$CEa*cW4DqEk;eDorg3*a8k*$n%w#vlHv|at+#RXwTw^4x4<{aSl!1GVH z{-o;6N|2C#Ce1(|iKNd|#f*UYO2z1}*}NGaPJIJ33$ubrhID2Fw_j~Ew|j}%{yCNcvB6)66mKnN(AHw_>NnbMJH71_su|R0=n^cw&~Lj4%Gocgd%nAn zR_^M}?kTyl^1R@fau+c(eQ#2Cy#%;Y;elnvV7BWF5dTq3Ip}xwFyEu!Suvy=?tA#V zqK{pBJ^i1ay7hD*^4<-4T3t`;V(NiVAJZS0I*_@mi|I%GauOiN3hH7uM1tupTZUt7 z@3o)fl?f?Y5O|TN7=~jxGRUf9t}dK@6r)k}mMw|<;Q2oW6p847rk6xZJM6Kxc1}pd=kPaEyrN(QOK!t3sN7vPJrd# zjA>stWZOnc|2U8|&ww#&kL{ zgY-}~xlUK^kOD)+Jg<~BRPts3-~QZu>DU}k5|qbSCI;@a4&w{+FXRQL%k^}bH!xqc zbYgA95am=Pm_D;TI&tF)#g7!29xYvVha&R zGAl?Ls*i7t7;}7Md+4{Op3yJhxecD;L`o0@QWSZb4KiK&_V&X6)XKOjzO~J(_V81i zKMNO&QWQ8+qzI8DXfmjZ`L7jDnHS5qe`ix~Uq64<#CG@v>vH=%h5vcvvCauD)tBjH z8ChglLJ|pvMOWz1}yLL^{ZJ;SEGe{^5rSYOfur#y$?co#s z=JH0w{4*`1+n{|NkuKh!Y+YSy&Y83L#miCzD^N5`a|Fq8B04@mM=I^HRs3$d^@rrM zzQgd1I;p|*Bu&$WA%}X52X5a$c`Q(TgLIkTFt5u8s8 z#Nn!gDq?&L;t+>8T-{L2Uj1SFsPC;izFY%E9V^TsMI9{7(=4HCJm9*{aise-EXX2> zTOgn;?-4zPS4nLn{LU8PceV(>gU_MVAk7?@hrcs=+7l@Ut%t$47A#uGW=K6+Z@R~w>&XHG^L#mZzMt>xG5FxYo;J8DZq#Kl!*t2XNyGzR zU1V@@Y_=8oy|gYU%`qRVKW(lN@0@$^MFW*7m1TIEV@XYwPMLZ%!D-1y?EU$tfN3pexI`WsT$pc@m`$29Q+z}`;)pdGk(s@&EA7zNk2Xdi- zJ=&?h*m`XCUQ-9DljnwWG^P06Uy>5V0D?vxO+A(nIt{MLZfCb0Xxk0o#g_s>h!jcq zC8PpJQs|&PYAElkZ2$Du&G-HKUId;VG*(aqzt~dbS)8YbRx#GfFe)$F`i__b(`eyH z#7-8|><=D4J8Rva;V(;M3dl66sv0c>nQ$bUjV7;-{fSP_Bb`1C<6?;5X_6&5k>V+W z4_Y6C4Y5Ogd-nNZr%ltac1J8jFd8M1JV|mb_uMl1_tiCv?2cC4SGeYT={>ng|Hp={ zu7Q0{+UVc8(QrA2xO| ztZ4sOLhT<`=0t|3HAxed**ANgCJ$Ebm0FVT+9LfX~1#GE9DoUhjQPA$gGX>=F<5{qFeR-|Z#;CNo3gFMJp z*`sq5*qAEq><>+Q^@i);g7~zAAt_#&og%&;*k`HVdIPuC%W5Q3+(63y*mLV{tU0z` z@Bvo>&uEOQ(1Zj4O-g3Z2ab}bT(zj{WbNzTtEGL{q%of4)^)uZ(m8lyFW`1&NixVn zY19v}7sc1vw~O!0UVcY-qMj!hfz~8ml1K(EyGM=CWha%^7nT zkBl5aQ6xdpqCesZf$pcS6ACGq#@FL?Ryy0I4HN&p?i=XDFm(J6NtJSZPF~Suo|6Pt zlPRYwJ147p%U?P!JMSp+Y1+6H{YAo&*nj%9W%nBEn&BDkk_;XrDI&)Rv?8!nKX%^^ z%hAPlxQwRiX$MRrqP37+s1NEb`bE3*B1Lm9p1Usg&xY^1>m<=6m~jl&t)MtoWn_hv zDUzM{`QSsD$u&~pTC3Y6(kj#KgxesF8Xkmx3!*8}6uUKjXRVDVmnA_=zE>wt z_M6Mr?F3T~yUVidwW-s>>l7(GkOUyA6wLxur>Ihdb@j0KIJt{$d;-7!D1xCxk)lPO zBWWf`DZ^nnpgrw9eg6nm@mrkcX^|p0T4XtY1ec&KK3>;iJb{hj)Z2dTisXg+uMLm8 z6++Y)lA|a^2y!gY8|yEBsJ-=!P4nBN6`eex*_#3*R+-^shSy|Skfp7@n;!kM&3mcv z81IguZMz%L(D3)}<)kE8vx3USyUx7TJ}RY8`l7vcf(agPhN5`^aC$i&0GS7ZtgDEvlVFnUN#uxqFFhPaB9Tp!yfZgb$^k>AjGk^(BJpuT`0XLtdoH)_ znfz$@l}nHS$fTGq)j(X>bPgYqmBkKqOfC4vB$dH zaAxg+J38CGwmtPwMHF({x%;>!$e9dI4MeW${saeIq~T) z%=pP>l&&t{xKntzrku)&w%o)bUTK1T+&e#~<*;q=+%~+Uadyr14e+Q1&ZV3aX~`*P zd6aR9j^RsOC7ck!TiX>8s@6}ib6XmI^ZJB($oYu+G6obQv%D-x1o83~Ic-sA^W&uy zgO&A$ucD<*xcZ}gezC-iv9909Pm3AzJOT6G<)Vlk>3qlG@&KX4+&MyU3%L27VCTL# zwX5oHb%RgzVwx3*G%1iGDabNA>X12X8r;!)>?0R^|J1u38;4=dXs0W;XP#T}yCU)5dHT=BjLi9K(n}CBZIBeL z=^lH^uqocHw^t1o#gwGZn1v(Xk{eaJ*#f!VDYo#m87$vwnrLL7PrL#( zV1&@TpBJck3UG=N&l9A|5DHshIQhr2`OmzWzdIQfF3R@OwW^}O!$GV9J>WRR>GsE` zHvVtb;P8Qc2v$=mnWkw)MIC8QM^Y0eUyLti+YjIT`0!cZ!zz!3LX14i2#QFnK%l^a zzkb-7Pf_+-6@2K#n(N2S!EF21_RQUF`?d{nNCoJAu`K881k1XK#>y++s`@E9k z%SBJEMi|~L*^tMXXJ0hhK{ePI!haX4LP9Gn!}AQOs*)(39%nsB?MT91%J>ZOM&K*s z??wQ%LOYeQoKB|5WDWD|J8KTK?--BOOp*jy<^@fbG)_Q0A}`d>rKUfpMub`C*|!ZH zw7cJzVTm9?7D!4V7>VRWbUE$aQdsp94;QA_>npWWhAAK)EP8m%{>uFE!aLO9!h4@}IrnKjbG5P?Nky9}$b z0>=ZR=eTD7lfWK=OVoW)ToLCAVSaG{nQA{Jpg8Xn>?; zg$5B~um$$5H`Z)@|NMq{QGG%D^$C7M?tz+qVoh@1R6F7Akq$U{Es@lOd5$xMH~KEp z_b(Qd@Mvz|Qzw0Ww_#0!d5;o~;TewSHBOKzj>W}#?NT)`HJL)^@nZ02?4Jz(?ZcKF z#OgU!a7deBoc;V(2*B$E*)HE;ASs!CwbsUqloNA?RjTq-gz^?x)d6#0zO?XoyTH_h zV5K6;t6g|Jd)6T4+sJkmvrN!>nnw9OI=<1iziEWyQCf($Kjqa92#;6-(^&O~suBME zre#DUu?DNb{=)UQtcnb?+l3ytN)e88q+;IBb^5!lBf9;Kd%vxtZ2KB*7Y=Y*n~-2X zV2-SPUA6G#e^j%4_En_NwSpZ`#=5>oVx-YCtdze$UrzTO9pv(TKTNn zRsPU-;QcKEE1IWc3V3gaEaJPLqBhX8eAgdc@)D4 zAGksY$}cOZp3<(<1Z2M^VDM*I_d#lm-@Dr2U1a?CnWLK@c)p|M9K0?3cYm9L84+^` zBHV0x(f-!`ttxd{1A{v}5ljVUQ>@>lVB(N0k1ySm=k`4^=%X{6Zg@HcvMYNGu#UNV z;ri@0JP@yU)v?Uk`vTwJqQ~Vqb6@B@EDt5q5=<}Io9&-6EiDbww1+}pCTm%eufN}G ziYD@ap(#q>)z@FU<{aIiX(d+wbjRZISrVSRBl{iq-rgwN1>ZONZcwAx^zU$5`EDs# zKH49gF2hUqtWEW55VtfB=A;7I9Jl24dL}qYl~gqfh>}8zsLf#w13n?a^s@cdxgCC( zd=_#>ABQyQP*_D&cu5o}N!)YYmTz2lv^A|S<#<@GaujQckJCjR*Z4R^myeI{l?m_+ zc-g*de?kizAL)6+%PPh20?z|gMum9Q49}aj_8vWl**5nM@6SSGNmgM*QJ`h1i*-e> zlI9e4B*oRW)_(NavwJW~W(4O=;F@W> zGhWSYu&daO>1+1_+2OcIpSdd3m;TXf{aTmr0)0i%08`A zJ`7K>WO2Q{Z~i>K1G!Pt(vfBfOjUX__r)|^nm-vu$55tg4&%5pCCb7!M{#E-kCU*8aG zg1|5oph!-VMN&nbqe7S+G1}T<@BQMKUYjVmz(V`};s^;yD#0lN!Jd(i*L<&TzamRI z7*?`TsZr-v`}!si)UQ?(kDmmAq(zY?Xr3o2fs5WH@pk)CLi~7v8+P+pNPdn193x18 z*Em`n2qsnK|C(;u7$=G15QjKiVE|*Mzk+#gLVmYlo|_E5H+uhKowH`s4tw@PE3eTi zwJyP@m_qvY=Lu&|ms`r}=>l0Q*1NwMThp8N>&|C%`sq~)&)o%EZWP3k7?77O1Md)@ z#k~8g6o_3Lu9>8FzvM06ZU;{L4hY=V{51rP?ULICKZt)6m{W09bF54 z-H>`qIn6{xDTDmx3je$H3fa-9bQ|`IDyQ-M-4$5R-x$$Dzl%IB!_9#=3aIpdJhBkS zB!nlX(~*2jvY`5|O(# zQsM}XQD_B39^++~T{)AwB+(qkY<0Ypl?R-DAB~hIv#Kmpv=ZdIxH>Q8wy^QcCR5@6j8F zA9Y#f93&lW3Z3>wQXGs%^1aJ>@@7;}kU?~Oj9J$A?W3l5Za#k_bX-zWI9zzk=@gmNPof@3M3BWU%#_1)ilK&gUVVN-13 z^G0CwezfJG5Ut-}-1o)f|L#Gf%Z3l@D3cKu8Hp0hVB_a$L(pJ{*-^W)YR;RxkA-z}M^Y-o zb2LY4DucSYTZK&JDvME-U)vAup7pqAK{GtVSJGTcQ8bljNnTbIA@kca7fNz|y~-%T zDsI%b_Kl+^KXPnVxbUAWNlAjH$O^|uX!y_d+X|bqXmMZ~T+?y;=(V;jJ10UH4Va61 z&6VYMpec|PMFL5p1(I;r?_1feObrUpDVNe&Po&>{9O3Uq?+w8?foL7&cX(X>y?x5A zdlrow1Upu>SuiO_3tXXOcZkpHa%T*ZRMq9q@QZe9O>Xvcql335+;*kNBja#YLiZ@2 zmp9aoQ8`?M?l4dlC6(L3ESsnOME zZ4BN`h_F6fKEZ&0>~2s_q|YN&oq^EB;G9V?=oe0f16My2<8!An@_izbmX$weFzfYF zJ5L=k?k8c;{<8M#N~1I4bW_EECElJm#32rGh{II~#aehj+q2WAb!z+^)FKUGHknP5 ztTS73XJmmw<>&!mg8h?`)9b8WPz4@bm8wQo;m`KBSGey!ycDWN%?PU+rSd?NSeB+) zg1fQZTV$tS>n20P9j#vY8qwd4-j5YLS-W)aNBUj8C~5Q$6G~Su&PQN^>6Crv_{XcK z*x-PU3HI;p_oGAsR+bo*V=2(;{FE8jew}9Br+<;8FPop{6{cJDx;r)sT!auV=xL_T zce|{CY+Dz-1#`^Nk<4fK)HXFibWrzBiu{x>hrQDJ}(sl8zPKZ z)$2s)=M*A;>C!*+d+@N3qa5#8m*3PIo$Y`FxsQh5I}GlrXRIxJ?f8V?`F#j}?_acbnR%Byz zM)Kbx^Ry7j4@2@ilIM`T70IKnuKz;vCz1T;NdA2!{~nTm8_91+*P%fkb-TO>$=|D| zeRnl^`sAbYa-jdeQ+DrfyDKHKFm!5(P}_)?5z?8S36ycbn=UDt0NVLD)uTsi{Lp6m zszex-2PJ2#mT@KF7240*TW;aq2Vl!^mGJY4I2P4tPdHW~p`u0iw1@6VxA^jEleYOHZ<2gy8dZ|sa zsxPjdf*kWfbPZ&sp;w^6db$e14u{P}Xu0tG)PJ}}fBv(c=k?bfzyy!zhFJp;|Nf^aUeuHMIoNwF8m^>q|oL`~!MA7Aen$U~Bk&E9Cpx$aj?uuru0^ z%XR#5*FpRBG+AH7H<<(Z5I8TWcOuwL&t*El8oddghZp+l;G&Md&SIc@4@Xp#*87MSweJn^?IfJ z9$6q8xP5+~e<0VHA;4s?9jS&>$bx56-43v550zW^t6eN7~VwM z{N_~}tQXJjSR8%s{^%XczIu3e$L}JMVF{TfNkUTvl(jk92)7PwjzI^9HJ^5%9wJ5v zESus4zY`5r5rDY<}cF}+O2a}wZhlsk;ghUuP3{Q+vjzGTqmip62sGqK$7BTFFn7> zH8R%$D{g~?tVR6*6jz+)xV=TEG1or=w^5h&K^14SG*xJ%-7U|R<#W0Fd(^xv;5X79 zpZ;|JE&E0@dR~5E9UJ zj8ac&$U03hSpRW|LwUeEn0!w?g+5;!Ab1z9XGm37LGXP7Ql6!hL_i$k5KBC~6Iq`L z$nzqm14GAH47lky#1^iJ4?#$P_;-5=X2}%d?#6`RZKH99>wK$$lrRo)D9?3`gR00n z#r>Xw;Y$;c_t4+3*Xfy#dj4>P?XIYsqK5G$5{GhJk5EP%tur#*Ih`1via1;b(7_z~ zJct#vyhQqm8s|E$@_9$k)9MvG503cOi;qJbu6nrI?zopA9ezB<{m4bypM&WagfmgLKNrb2!|)YG?L@~xHjp=<_u~za zdSoPD-GF{@M7WM7V)&@U2>JhkQI0R{_ZWlzt^*h#Z1W9)oyuSnUUgZeq z8IJ3va&J&pxjJrs#zaZyO^lLWE-y$dXL>Z!P$P*k+E3B)KM55)-j^z*RnWL!EdKUt z{tCvb0OssNsN+Xo{3oAjm4JW1p9ZE4rRz zbBE1`>US#&rCMBcfCC?ow}+%;I%QI$X^B=8fzlc_eSiHgcOSQ){PjSL0BK+;1xt~B zx3rY=vowUfAhfc7eaGLMp4z-=Ha3bPO>rVA2%JdK49y8N8bxukZh7`GcB&gXe#jrP z_5PFff~zQDPayMv=K-y1tQ-`-9on*FVob&pOpP4ve+PAXOob-uA?yfroqmRK_VerX z1Futdxz$0E&j-Bjp^`TP_@4OW)b83PTt-zlhi5Bc{f!*owEeyT*$|q25cu$@ETPG) zKyrctIO+5_>p^NqQsc`U8<=DKJ#Muon3_1Qb+qg-A{~y{$Dy}&t|uEfC014`nxZ9z zq)%s!{qWZJ##nGpZBrs_X^MfoC#94lfYZytjYJd2_9Yvh>o~7Dez_M$uXIaZuV;dj z0UGbO5>^Fi-!_L|89TKO4p)#A%aR1oGJ-%6tcaqD zyA4l;<_=~|KW^wsh>dy|*H9b?lxIZ&h@3==JU`>vHQ%3FU9%EYSaSCWEIQX3K3`mQ zOUE*5%*yPY@YHxc0W+j}Ku}c!3?Y()z>(MW-^tI+d&R7qQkn)tsUDJm%y2@C_aQ8g zjIf-}W!&doBb3sVRSH?<632Uv7~{!E+hdNf-r~jI%5h)IC-#k63IiQ4F^AZ9il|CJ z04l9=JakJ_Z^9c3s25gS;EI349GIF+q4SyMz~-Xn<>Qb%K=KWcJgQJt-2lNFInP0t zR%L^HszE;4VCg$B1U4>jGss(u#)mWAeBe=QCX=r}{syr9VM5W_h8WM}93BsmN8 zbb%~27mY2x2bn}8Y9CK^wsU;-#<89M-T)i5J3O+b2vJ~Vz!N|tj<(!Z<73`r`Jevj z+3J-{afrj^0>wNF9w>6ShDQnt<@Cys~ZPca4I(U&4nl z@67dkFB|6Zs=a}vf*_w|ov!ut^%B&k)dkI=YP$@woc}1nEkLN8G$BxV`s&eX$ z=77|MJP)oi-LbAmO8zg-My5bHLJ0m`1RTY&sz}k2l(=Pe#)~^%ONPz~)ca=D#eK@x#Vath6p;OqZ}PA%!7W81x< z)8LwJb`0Ih^|GXd*si(ycI}YIsgeYEK~sUqG~GI4nO}$ow=2}mu5ZBg6<0abv69<& z!@~7&yAHx{SD9i&nN&nxmE}fnwR>*RPy)Ew^U zp44zm;=6d^FelKIzzBlKk%GVmE#zO;6&Cnb0&cEHI(|-HwxHu}@Z6jc^4ugCkpqgN zF@i+yJ0pYk1L+X%c}3~Y zn9Hdg+$ktKR%JC1JEeHaftfx<7rutuN_-$b1Z6i|5^nhNVk?-;M5K`p2>kHGjkmtHb zIMZwx+u*anJu@xBowgZsVyEZ>y|j5D!#1R6LO}j5{od@>?+Y8&Prq1z&v5|DDJ|d+ zGPh&g{CiLCUICrb4+=)3kl%*hw|h#8qylFK@cGf|ATLl|isSX5K(o(L3D6#z#wEQ7LnkLzDLbhNbB z7MKsNa-t)9Q|Dd#%(a7=e6Nz_aRaB9lt_*j0MARbh%)ytrr`?CcTBClrS3p2yhlKa z1-v8yAZw(CmTke#$Q=%@vZ<}}t$M*%UIE_r6oDXBfdMMJ`Myev=8imXfnwRCNZCgt zUT=3fr0K)Azq=Jr-;n~xumVAo48t)bi!Qjm`cMNiu8OG+s#D$mZJJ~KYyz5BMM97% zi34Yv4xT;cpG`%6HaEcoBjO(2<9PA=Cr{2AhiAPwLZnEM=LLU43uuoH>2-I&Rx#ai z_uz4zsP7`U?B^tcXH}J>1dcYk>__EkXx<2n-jB9C6uKa3S<5j-+V?t`v@^-yPsCzF zmSt&?C0L&2cwP*mt|#jIc`$Y=XE^d|Ja*!aTbkn~uzFwvf^zoD>Ynd*O0sL9rsV?P zg$ox3&RjLf25Wnxw$a+0<#_0mXMX#*YFcn@MNti0KBqvS>Mv6;?q-XNp6#gZ-rM;6 zbnGq@35t_cmXQ;w}| z-(1rv?K+4^tQZuJjUWRiN{Vlg9!92{l#zF$$NLShA*;;3Oh#sCd~w7SMj6PljmCYb z!GVH8OC*gHO8ecV2C;5`B8DZaXu!?44NG<{a4Z^n{@z738ir@7C0SuaS`+~3%+2yl zI5}e0=OyoB#J{Xz)8KlSI*vcocD3>dmbxTK5-2p_6`Ced>T;DOy5TUJo^|y8Flpq` z#UUgICDaELQl%JzBt?ouV|qrL)Phdn>Q09s%q(RmeDx@Alz8`RZ=~h))Z$pG~Qc~QgUoajs z5b^!)i>}<(mnhacjhTptE^}8Tr+gJtPd{DhqGlb_6$Q^-VL{LCt!=s@!DEX}Io8YE zMdRBs##f!LFxLl0ht88V!uFugadZqy$3@yd zSRRfJnov&X=G#|@3ph9qXn!tuy5hO3Bp58*m+1VJb$!C`-!@+z_==oUQZ7`XMdoBQX%eVVDKffwnF8N}go zf#MB1tp41&bzguwOd$pV@ z3JSj5wGyuA7ad=2Uhg&3y=k)w^Jx$PSeqBd7eRTf=iTd|bbUX%aLSgyG^xs=11=bHo^&HY3 zBfJaygQ_rRn*&qy{TGq2UBfKflAR$rT z8=j{N(_Q-5j&Be+AP{z$F0bzteJ%W`e-DTBDSb^&wqM-$F7U;#(Ei41{v&N4tiK9ooN?d5=MZj=Qeb+egxDrZDJZ^FD{==N=VUM>!lR2?XJU1B!9;TU_8_+rA$upVk8NfKq2kr`T1 zQ4ck!KZgC7qo!D_y~~kVx7A;1eWASoA=(9+1c6I+1VM_aBcjNdul{OG(|XF{&bJurGn3Q&1>^&0o=kPjd*>F=hysE zN}b0?`ySaI-Ju~*O9Dyon!+i7+er*7=+JPk?jU6A^q@QbJ5B`i-|4o|z4U-aub*>- z1HFftUdXGTdsEIH zuT^csil|5byexB+AP6eY%IJ~R8>x{CPB49((rnU$O~+M)qq02Wr~p!Kg*2|TAsgh+~{z@u1d z(J%5v-;Y!7x|N)rJ_0IgnnrePbUC<=s`4ZNDydSWx;4;q(I3s;PpfN+(A)yhC{~xv z(Z4S?lu@~OH^&E3#)%`U*1s8IxJDEVGb*Q03Il*BP+NLVX|s35q7>*dGgfqn!a{N> zA*NCZeTT>B!IazN3z^Jh=q@EI0-q-ZXO1f{wx8s2Dcyia0hj+K07uE1NT@QS^EXyo(Ifi#iKR1-&8EXC+~|Kgdp?P0z6k;+0Y%~!ndQ*=hEuG?P`H`u#`#`R zp{&ZqQ^#j1gIcb!R{k@poI~cyc8sKH6o<+=93~B4U%p7GJ?G%&3=<5X^Io`VStd;} zPIqqTxgLQ)R>utB3G@LuHAZhge#icda*Om2Ro#@eIntpNj9LQCk7^kDza_@@Be8_k zAF;gmCKfR*ZFNUt41APU-crW>x>D3z23$Ma>4IXz7bz>h{QlFlqtFsRD||64Xo)Wg zq{K)JuMhwX?vgXCQ&GmTWN5d_y(pO5i)V9=bkPBAHC`M33 zp5au0Zc(dY>E=v|yZigI=e-+&Zt{y3WrC$dQDrz;V7))jDd;WPQp8zfHl0mr@TO_m z%X8tpnVuMx#nH5?ii#`(juCw=o=hQx6_$DWr!BA3)uX~SrVOtV=X(z)|F`W7m<+fc zAz!t7F6BT)^5B`NweYIlT;(dm3k1Px8Y{^k_P^CpP@B83klNgHDWgxO|0*7@6~6f{ zP$R8Mf<|$o#B#i)e2Y!B{=S$5E0T5BkMLI0xW+iv{9TH5cH2(YBog1dOp%`s&612vzV^*++X{*hicX+Stc4|xpHmuB znMBtq_3?AU2NY+HOAU;5(xfN~sv=7w!7u)Fz>%@`Drs$)3-w2cS(zH7={EykfrD#=gl3du0 zEi0A?ELQ8NFXTfnHVhw`i1*}y1ZYK7MU7F>U<429KgwY%OR&B;_~+eEHLDYjy!@>l z5J;Y935H-eaQV!)%BhBQ?j%b_mK)r%gZn?f9X9I>`?#lD649=y35>uABEbkE!-Pez zTbUROuMTV4cYBBBT&j+%LL_L0r+A9vXp#)8AiGU4U17sjP|^C?@!xtiERT<8!V8or zFbvIbG(|FDA%XtT=hPu=A1YhB?$~nt!p`VNqXfayf*_HiB&mTr=Oy}k_)^OwtL;mA z`jKus{FP3a-Slg+lPZVuYD!nUFnUb#MLIej*D0OWpTDkukdqW^$g5iaShn${qbDJ{ zjznU1JWmr82{=-csChR(w`*hT@hNZsq1*oMNEJPi|7#(6^nW|@za3L9`n==>zG~KY zE?U}rG#|^11(6j9j^HVVrwAhKj%F7f*&UN2*HFz`k!YJU@d7@u4+}J!6G>jxz)gFH z+*wep>Z+#$^|W2gWvsUKP5#|gu`F@!EO7WQE4)L}gLuEv z@r_d(#8Skn@FYoJBLliwTsfVi{FEGD{gq;rU)1vUVC);5dO0=Y#9z%K;(c)`ZLw2} z_+6DEwx>c7>!S;aCzRe2W5^t34b@9|ywLs%MXaxVjOStz%T0`Pd{Rq^ev1lqOQ^qB zDpWFEop4{X_}&}#nZ6m1u^`pd_lEV;LK)ptZ|J*Oy}k$Z1-hOtXF%%$xPEoi^&x$9 zQJNK^(+A{Va9s&hjQsw&8$;!=tnQKDga404)+;m~`~AA$0^?l`(c5t_PQDDwtB_bS z^C2+?eqcLCC@{_t{%twriTLwN2xcXsw0bh>}xbw1@6U+&r zJ5&*2Fg%jK2ieeWLh4z9=S?%KgKf!_f^r zX&z`mWEoPU&?9Rzw&2~vCstekW(2DRzplP-olP+WLkKJb1V+7QK=P`BXZ^88K3`+s z*g9kO*S{=j1^LR6@L8E_QpyS(pcsuMB;|&i9vsr=L{jR|GRAtNtAjYM7zwWi25*K0 zikk~ftqjUR4;o-muZ>hIf7A`p6Tyo96sDFsV-(W?8f}x6S@_}91r;P zbY9}sipyr!W7YztYz;Do*?uV8HbV z1&xBNA-kzI)_3OaNLX_mN(+yhBJ6+#$_X-Efo|D;p!Sxs27T(#K>ndN!{hYb(wD!$8v#&9&r0gBIc8Ami4 z>b`4n(~%CS;rmkm3N|nmsZZ9^YWk7j1>LtK)O-D{AI(0k?;;ZQt2cg(X(%iwFD-)s zDYyq(4uHxKGq7{vCwEq5M_<#1=Xnnf57LRQ&8jemu?EpKZE4m$R1`KlAni^=RxuS$=`0->tOHJX5}uOrw=H>|6DfulSwbL1mOw9@wRtU= z-e-wCOF8R-lr=3Xc7}JSdsGbH%QA!*bg`H2@n-=gFhp`^gW;baI+S$j>vD|^pJxV? zA>^rL^pr=Q*T!h0%$RZt3-OYB%VPGgq~0rOyqn|1M``1?`G1VxBn+Kd%6NRt*Et_3 zE&4KDWAnGL`1!BfmwJuRspUGKr`H;|;vCNg#pE|=`Q|Mf$8x2d zhIt>Ix3MnZ#wzDZhgkhfy3W^G0}L=A-u zuM$?app@DPDX>1W;-cEigi`Kb{}LL{#xb7PNHe%kaUlCDvnaKfaed#X#>a1GT&2Dq z*rbx>zp>!KGQv~kINwVvA|FOSd&(C5{ktn%S(5gBMQjM<>#^1^OANWBq{vOa#`v+n zgw|Kfb%KYof_UW__+DW(n;#$_v@hfYiB(@?L|(%2$98Ga*S|mG>{vGD|9A8Sd=7?x z)GcLtvXl~i`7!09=yDWuexi6yE@XLQ43W1~0M6IjF&Y4iX@K>S>D&7{e#2UM&@91I zBu5GqK?y9upjIB|6GKz9iUHR5THgQW!&E%MnJ7x-DV`Q-MFD7Vri3>mZw_()qt9c# zS5$a!ytz!u#XII5VU^s?zE)G9)!JB77>08_)Y2rcu@WJ09QS8RtGPLYd1#Jia_c7s zYi#kgttrwL_`BAc#sd-@Ec5NUJKZEdF$W(0;Ye zVnLKxCae-`jCESCvz@+}8a3=YFAF3ikR(m0G#d6j6~h%%u~n+poblz`zc&VN8^s6= z#qb;bj zvtMj}JI`H016$-2!Z2hAEeNYFdKk!3SVeKB}^P5h8&(oUCuU_c6h+$%|FKmz{8 z&=kc@hSfgj$;TSL)&Nl=U3Nr$IcS8gxD<2!P-pn0%#I$(J<0F#qwynP1Pn9{AC0lr zMLRZ6f1v!0crKPna?E8E5%d9y>ZZ?<)N&Qz$pS1w-0Aq77q9h@~X_ z$E#667M2*o!ra!U+b!RD^J{RC5c1I*q;-g4Ck*y8a8jo1b>G?I_ggo#U7iMsyN2V& zi0RTcL2BHa8C(z@=Rz;!Ew+Y!58()M@zIhC1^Dkqghgy~gyv&C&qkilLY@ytW~GV{ z=vE|u*c4y=7L&)CaMQcDwwM`x(V#Sm6nT(SHK8TJu?A zwvM{t*Uy_H*5DhfhUOnb@^z7XOJpToi0HiLA^BcN{sts(NAg_|H>M{L<;o*SzR4Am zU$OSpG3OKN-M3HQwA_$=#Bs)4KXE3++iOji4tf5jW-!0uQkB3RExP38*Js@|e{Ou4Eb^RQ06)1}`Jr$!GW zM{uGnGXhJhvWy0>iY0L8Pw?BV=iVH-FrgM!PGbmCU>Sno2$~^9o=4@hen*0pJ%|aK za=SJ2%lk&(aU(ux3JoNQ1r$rDG8#1HL7hW&M|RD4QP*$1SZ;WuISt_jjR;|^3Wj~s zNM6$vQI!;lT{B_5tzbB&$#JS^@Qn#rd$sTI#s6kN3AIPKgi7!%K@+qnh^mN6s1F=T ziI~RT@MK%7{c!Ni6JOng$WJe86%Gt}^Ws=9KPV#ii_9z{5mn`7x z0o+-@H~8KsYW?xmvc#!n+z~ivHH-OwaqCCT*J>5AoGQp+K?O`ZYZtQI6y&?wm~yHh zgb^#ALAg>M*Z<0x<9Y{6A?d83z#Z-2aiaC(K1a?g3)|usWEj{~r)-xy;PMO%_<)p&uGooj@;tL?igj}r-=;>7z!k1TT7-?ZnCnv2 z;}1xlKo{2pm*4Aa?V_pO=G(Wo&a9O1bO|fu%KC7$`c*=Y^~$(D^A(PN3nsl$EkJlQ#8xRUfMth8YUIVZRv#{K6&Q~-<;(z9^SgMz zhu?L`gFniVih}%^y=KDC`SqY~5tz$l#GWe7f7e)w?lL$g!0Y&35 zayBV6MgK1;XlE|o`#6l;Z~cA$V^bz(!AoBe^ol;9ICETTV62muWKQ8(nUFMU@uvfh zjI~!ugOq2Flj`rc-rZ09?94X}@dHqhAt;1I@e(f)giJMR-u>8j(~gykr5l)amBsTh zn-*9*{`z9}%&+mLY9cF86vNO2Pm2r}W~%0VJ(#{*5m&`RYd_b!OOKCs36>yTGaAcj(D)*cNo^Rj`3321_koDjLO~{2GwF#e`8UBL8B+D?)Ub)BFz!#*N z-XJj6tM&$&GVuA6X7_F$Y3;^yg-5M-+czxhHY5T8xK*E%7jmzB1D{KByCwtYSjmqP zK$aONc|2)df8$$iKb!%J?iY!+R8EsAR#E_v7-?JA_g9bpusnK5R3!#6;|B0aYn79= zhAwGcBYfTQ=?0=cf)!{&)M!#5WYj9O4<-%AA~cPOI*kO4&}dpw37#fUEI$81BTUiH zS>M}q?m&Y<^}~zKH)q|bZQT>i)$%l~Q$>PakU8>TNdhWrekmrN7 z_C8e(*C%G++2!iN6znplb4{oDN0DX#Bf{;ym^ME8#{mng%q)@_R0tC8J{pV|JWdbM z`5W&i55gt$^@%wjVR+UOX;yL&k`jTFFv=zD+9P16MlSm+mG4>w-=30Ca0QAl7$Ne(*y&RU5d>nC$dMmV?>Vn_wA`-`DcvNwk775$NQmjM3&0p&#gbR&cwLW}1gc^=u(zg_} z$^5i8KUddVKhw}f%sa|d|Iz`qJP^7q4TeP5b%>J4n;FWRAf{Zp3%3I9db{8|Rk^!M z9aV9}vBq0{$V5!n$zQx-HXKs8#@}iHt?oj`{%}NM4>1?MeV)p`%#UzO7_w#Z^DuK{aD*R1o+`MHSA{l0paT^_od-;${| zRFm*nDH_(9HJ!PifMxA1Z~+-`hosT8$enI#mOV!i{8if@*kQtCJC?$bauv((bNg%F z_`K|l{MEgKwsW=InvNL_Oc53@Vckv3WYL&~uyMT2Umu`|7K0tdDm|P<`SK3Z34U@4 z^kx#;NLA z)jjC9$!n9({aG-w!+U3-(odxfv7g#=Rw~Xn4Yb#|bToK=HqZ~T@)PhAZtGlMWo*f} ziX;g{b+_|~h$g1#)Fee{@n zcUD=Syj3h(N^IL;iH(O||06CJ!`$?V`{fH1I_P0h-}fON_>0yHi)S+zIw;hK_WH@R z$Od`$Syh!Z;id5fz`y%~_Xc2{&&-_oG|0c+k31g3KOR$07Tu!&x9Mz7-cH{AVEU?^*Kq?{3d46|{1eaNGtXQBgHI^qK=V)7k>Hg`r>yt9{HShiL%O zDa(H=9n#uXLaREVZFj5r#NhfT-nP1X<9U+lXU&rR8z1T>H>i zFx&_j|J{XQ;VNoJwE8s83jlYSvz2m3VZdgRzy?*>Bj_ZqJb7p)ruucf^BUu`eM^xw z1v>Q!Xrr3Sl(NfU=&A(B8>haOc};T4156YdyPIfN$dbk$KP4kE;GLQ2rndQidGS;X zOGkYNJ*Na#zww<P345ETuV9~mHN9G| z-%HyC6ev-n3P5J-j=rabmt5CcXg=8KU;29{0xjAN8-js)G~aDI*?Tm~#-A51H*!5!wCrJyx#~72y^+nn4TqWg1kCM94 zQOST>+623sI$k|uX3ayx*@HlaN(6FiR!nzV@}=LMJ!WiB^bF)as`X? zvpgGL>&urGpWl`<+A}O(R4yztAJ9?P#cRO+4{Zv{N$fnX;erd{ zgB$htI?s}5{{*gi=T-g##++{1VuNusihLAE1tTqaVP-Qs3sKb+T4%wpJ+*AR(~R0r zjc%0F1@a&C*`>LHJHSqAkS~&J5Z#v;lbMI99EgHHG`jZr<~~a2DZLgjrh3{*7XFD& zQQ0KU1eKKtMZbF=$B$MFwvv>z?vrrG!hG`oDb;Gc*4FY$T3UWuqM^xF<5flmj$J?f9 z{_Ze?{HoP-0!ATkXNMXq_z_iBSP(@ix|r-%;Yc7=-J04|>#w8qG_H3fI}CWewI?&0 zY-EsfMJf#WQf*Nqo-&n>PYdK4n_MtH?@@y6Vipi&Qbwz21kCl>zugu~b~8O&Z^GzG z@Bs2@tJ^&RYBuT=P-qZ6B^^1#b zD}K}2jc|`*kP?@u2y{>W+e8j zDIaKyN(QZ*((j#M+4^m2xA!{ZT5ulqp&mvBmKjS+tGoHZ5!#b?^-cGlBx{OiTWG0z zG(bj1CMEdlVY_ja+7aW2)E1WS7Nwu!dgiu9Xp%|p=n~D43OgH9r=n0}>a1iN?LEEL zj+N73;6z}nMq^UPD=N^8i+h&>Sqi{7#O`;y%5_?GJsh-})Z<%y(4?e2sMZbaGs;#kID0Y`?tm*{~;2pvl96NzZk}!){BRr3(aDFOG5&ZFbfHyXLOzX<5Zoi$__=6yba{IRu|~C;#4M=53B- zeqm>@M%bGAv~}#O?OmN@lu?9P1d^p-pt>`$JYJeks@d6}14Z_+o749lMpfD6qY$Lo zf@x4nBlAJs*d){E@2V}Wa-5Hm+u(RhC;q5d z3GS{apY#1;XXZjWMEubQ0v-*?D0>nBUMD9WLOI-y>y%a(>%@AL2Kp6NTwhZ-EMtOz z@MFh1DD>M5`n@Ff1)gKB>mObMbdYL_2=oY(B6QEK@DKbR1fDp^GaIwLRo6F6_!dVK4tiD8=EZ}*(Pzk?l&Zj@9$QD$njd|IV&HXb-eYt}_tk0JG59Y6mvFx$7))ev-di9}MhZ)mGLt;*7P=nk_FbU1 z^zsrlM9&?hYmcF}@QrSeO?togy+$FrPhZaGkpF0w@9j{JXGZ}+z8a5)r7y7B{uA%} zw-qLwbLZW@=+YJ0fQ9i$q-q0;XU0>fXpVvf8LxM^3B#Hd?p|3u$Kx#Z{aj7Wac|Z6 z%TASPk7|S6XRZUJ|K1+R|Kk!^4Xy;RL5cPjyxn}C_3Z%@ydVqXw6df3n0dCFNt~V0 z3awb|AEl)@Cive9je9SbkaBJg6s2LL+*DehtsbJQ<;8!48E__Qz`C3L!-utwB9U3+ zPtTs>B6=YeM7#SZp3iJ6b_8lwk_Q+6O*(_$+-Ve`l<=Du<;(e>R`Tl5uC^M-MMmD1 zMFxRH?#IK6|EXi_>|z$O%{2JY|5(SX;G}~UOEk0VJX&vUz{lRl%Fb0Wmg(9GKs(sf zGlbuE4~HXbQk|#12PtgZCY*hF-B#>hqzBklpx2$@8%o8cV!3hKHww7EPK0|f#u9-T zP{Iab(uk&bu}7}LaMLshQxFTnkc^B;#jVsK^A!P zC9+VuW@@!CSLFA$wGa~^Cgs&>9XWjDcG>SU~rX0A_fN$O$3uf=CiH2YJhHSCn4YV6Sm+3&KMhp z;w^tadWZ7p?Cs_re3>koV;-UDoBN6w@q?SlG8M&<2j@--#?C_$#srASEJW7s`9J54 z3p>LZxgdxet-U{Kq9+-NLRcNEZ95>i6BrZ)M*Lj)_%rh63rZWh_Z|*8PE;~D;>o|I zvPrCxcC5%4V)r)cA*fA}(lg?kc!>3RlwJtxcCWL@BdstlAA%4vgm!YilcON@k~cVT zpYyfSJqLf+$+1=j=onh8BQh4fygKOwe_-uDzeXgt*!jX-LX_a z81B6|xtC7qeCf_2{7B)Kv45B-^9xvGU;Id+G}^@bfBZA+9JRiJg22HMXc$)Ec^p&_ zQ*r-W7yf5)_}>@dEYNKuj~crThXkGVng|Jd$cvJDy9As@_fWOA5Ae!M@_Q*9jM zO^CH_RsH9fRbkkd8h#4NCV5%kq+T$Ton;5@sx$S*5j;E*xkGjR*ZnX5j+YD=DfLqI zq$67Kojkf5Xy0w=KhE1aBDK0HAaNP?V&4f~La6;j)^M7{V=EGOX^H&SNfE_Ai2r!3 z`Uli4w*mizg>M;CMdnJur;_}D5Q`}uEy0R3d_r0;^We?KL(TG7MWR;lGJBbq@D>~( z43-Q3olAsrQc?9igCQqvt`#=x-o3G1U3S+2_>4q+*xprs5EkP9&H^CI%fj8k{wkB# zc@{6$V4I0%WvRi2bP)YcUkReBCZmQwcmG&0)q1IolHHkYT;VF=j_B}Z{S&*V&_=iC zk@wirpixDRP#UUYn!Y#+K7i74SI0JG0NB(6vb-Ww7HZmik*?DT80z!UY z3r$HF8>gjj82)gvWBz?B zAXvFU<9mywpU$5=b3EmAS5rz8TpUFJj}^cxYzbIT_s}i(m&@I9zy}`iu=x*YYu_xD6B4(B@W;E3s2x5=Y)UZ zC$W5qK{@6B0X)R&7Yr61Y&n;|R##cri51M@ZS90AiL~x`OAEYrxilyWX0m9_cp&qQ zAo^m0rRFWd>xsBrMyAys^6N@MJrhbYXPiup8Aa);5$XyMVuc=fZiuT&!v*r4I{PHIA=;Q>Ovt50_|ja*wb(}zy6xfK_qg7 z8L_LBnt!+hb1f{W5&*;LI{V`jONjDybGLd1M$=^{J|r$d(t2#e%T%NiMCEyXyDTi% zJ;%270ej?Kby1!Ca|{y=r4o(Vqs0>o=;vbpb3zF4(lL8`1yY`Ng+lGJVlP@K1@An7 z4xOvMnRab`_-+S!pxi(!MV%jIqOI_5kVAWWD+UCR*U--&%8NR_{+9azV%Z1H5BHpC z`0&;X2jnpVcX922d5RQE)nObD`}``+uUnyaIX|^nxfnL@u>-hG7gweKpKY8sE|qi=pIF9A91$CZ2WtVZTEry&s)Zm zay>j8glMx)KuoQwgRQK$gK;18t}!0jxR$S;5#TpC3^ z-vkqF>)+-Ay5_@qtAQABi|@>=1}v9UD)?}0(`%-=G%&v2&!r(gepUZfV?(=n2Uuf^ zDaav~*WQ4zf05?Hi1(cP8F=PCwzXxT?d);?u=JEr(b2gRcJU?@lhI|@Jf)wn@$!Pe z-HDlK=i|8YIUN$gbqP4#2-R}-rTnlNCwK{9XQbO1KaG4Tpeype_c*^^W$>ua={yIT zd%9^-4Z`-vT=+GyneE`?BnB&tH3c&NsO)V0cq-Qrv=4qwX4b%}vvZ|o^t@hG2v9Rw zYxQGUL?KhOL12`M9WH6E2$Wl-zuPG%;+<#C;gNMPhY>^B4 zhlh~KcQk+neXxu8290`ufPL$3+je7F zM6PwaPi1Dnj77AL3Pe@j#$g6Wc*l42o68Xy`Ct3Z7yFIvY>)`{e+r-_PN9XmdH=GH zkq_A}(G9cNt{{()UW6Yfe?h=wUyK7mHNg*bPkS8j6F-CpdvTw2CtIZ85ytKz##H#D z@SL`)@!_`9Fgn74c5S}@KG#y?Jn4ng4)U^OVbvQcS*P+?l46I^|YrdH{Co7}p~=d#iu*gc%qB;?ZdVuoy2HzR=HR4EJ(;YsN*INeTB^cIoy`5A zw|T55oIiIFM`ix@y-9sY+9_A{sVar2&f9Zw&gVR%-YIN)Z66oGmFsJ-vhjDv`|dF(Ud(%y|vQOYFrcLXqB(%5XO z>a=?YIFq77efA}kKDD`*Uyy$?vL~2Fu5Y`>GP$mu70ebzI?FXj4%4dOSJKH_5k9d| z(ynK^33qxSPBz6L4mM_UbWCdv`F*eDy%7J~;Wr>^m0$8x0i&bAYvKxVy8bV_9wyZr z;NKM<-xCgkoHqGgtr&m{om)FR(Nkw92qcqHpuAz6WIcd-*}2efB#CJ>pKN5@@SAfIk8W%NvB6fV#C_D#fMYJ$m$F9uFejCHY_TBHTXq4eRsb0-k z&Zhwsm1sX=7n2ZmhkmRNtEfdXjWo|>3i0@3$R+;H!B0256MY z{*~q{wAiN_Q742^7UOBQVkT*ttFPJT0)}?Nqb&G2ga5o+-Jg#4ZP7(ks>>rBy&FZ;Qug_3ku%Y%I+FWwQ%^ zx$digH$v%+{WVz1_Xo~o_`|l$er08%NL*x((9YM4HV4jW1-sIj1hM)%ULoLsfTc7A z`dMv_>&)m!)m4|?6Hz$OmO`|NDB`E7c+uF>?S3Zw$e09L>a2Qm8oy#Y1aRq7r~E0D zVB`78T5BAzF}HK5eRsC*`8D6lqXyyu)w}T&oVo!hm%V^yFblwP_^9}phJ$rvM#n>%G7!gcL zNyurdds0X1klKv@{s+fV0<5sQJ7|GPkwK1Su+x*X6=wRJH~3#V*^|`+*z7&r zX18QvlTu89>+oJ|HXQeQE+E<{DqJK*2s@)lP#ud)-4ayV=Ka(2rA-O~Pk{r`Y1ZMa~e{@Tf+fWh9p2CaFVeq*#GI%03rPCme}91&&2% z%CFH2@sypf)p}OrM<3F}?|3Gm!-yd42a0h*zCGf#!YvD)?e&MmrbhKgjfFXKN_Y#Nk$>68_-bAYhiW^d3w(v-kJv07n%U|#PQ+2 z4>q|2AQGkQ+K}xeKbxnDD#tK*{$Uh(wd6^A&DggZHr3X8`Ae!{7pBrg!i}+!mCRSb zE3NeO8`Lf3Rok6mW+PKa>t1{Cw_9B&^1(Ct&88T<;J_`v zC)qm)rFmaRB)sO3wZj3|q#CFF>lk+2y@cCr1L~&1Ogq6~yL=h*-^{WwUwv)ED^V4C zhgr6}bP6jj6&ZdH)%Wy_dPgBiXXs8yS!x&=J?X^M|Fq<<7bBh)^o zQl^?=8z9g23)-y3m&By2Aye<_!7cxmUgjB{0p7fV)1YFf2V~0UT`=d{YulM%uR@Z_ zdAfS(fl|}nRUi+F(9WHY7trWmyj^*45WkUZv!{F~K9<79lRyMxSh*5*_CYVC;q*7e zY;`{-NTdMbI-@{>X>5|^RI`VRaw1L4^)`Gqxcrt)_xmkwZ7OJz5u~FC6q~^q!?V)z zS4NJ^5xl)OY+3~r?WZ?}OvO`?X|gpjv==-z`WtU_q^t)HAVb4STB7lgejoMlt>!-~ z9)(f$knfNYPqoy1Ck}I@FPSdpbl{{AM6C08AGe|krN;y!?vy?klCN7dWdoAcN(wNk z8Lzs8+L6}$5vGKiNoXmMjaSD1CbDbaCNyTSU)->LFi?+$I z&H9%o!olyl?%e_WNx2!H6X$d=|1PDQgq2?`ieW5=~8%m@SF1t#0*Yd=dP3Z9-F zsh-pjxbONBl5sTx2C!jOf|ef#^XsM?#p$xOI|f^tVv8u-#Ty}bGeo|T%dj9he2U>F zVg<)BP=*L589O}Mu-!W{aOZ}phQi7x#8@mkSXI1k2i7{#I=VFTC+90BT1F)YQbZNM zPTCM(&pc)Y;+=LSVT6TC>D-NnM?H0cMTn&Ndughb;3v(qGLG|8g9j@i+hgS$`0dyd zeInA}<1A10iTM5%80)<}Z+k_6m=hSYWF1Mw6R&J3E%rWzuw6^JRE*7jbh=D1T1${* z$Wi;LfQqY<4r#WGo-UJ+zl%19k29-*j(74nj%?_7Au>Vx@uTWgSsl3(vwd zm%#nrRogA}cM`yN^kh_VpE5t;5nc5yqsXnzDHqnH);~MNZTI5?kA%TSe-NSNz~7W0 zQrW-M@AOqO62O)|Sxc;+B)ifta7J_uq*w5n`^2_Fj^=5xFR}AA%r`bBYcl5iz;gXo zM>E;`xm{-dhG3=J*OMK}lzSZOIEZj+GTCde%xphh6iaf}CLC+UgEN%RvVvNMOC%f` z{8Tvl6S}v1J*3hJ)THh(?HweZm}+a&EpYYxnHIkL%kGnhtE8%1DhY3JmV*D-z3N2T zgDyGyhZljewjBJ$A^r4+*`g!O{8yGxH)ug=V0ZgkcXPtQH)O1rAN!vHr9(eRT4!hf zIRDKq1XlwX!pQEoh4=DRT!vDXbKjLT(Z%?Jaz`X{_=Ix~Y2Af;{&xQO0_Kyl^q8;r z%#IONMlFsOg^^C_saysbbu?UK>omAdH}ktjBJzfP0J8AvAh$QjTfGhYxmNl|7?fRQ zPY7*2w3ko=W2VjnY3ibpj$wTT@Iz!=n0lJjbadt<#W^%TK*z$cU<+IBG)+XefLRFc zNHbFmIBoK9T_lhyn{jC{R~zli8}uE3naakWVDws?-zu&_Xo>q(k>47$sm2WR3~=!ru%bxW)ZtTBz*@uF25E>4dw+cXlo6ljWYIeQEP9{g`+TsU9s%N=k&=^mo+5c0UO##{fRAO9z!8{%sO zcJLw58II2OptoCG4s-zjt?Q2f*h-Hke~g=YLD~Xg-b5Ek%P!)Dl@Eniilv7U52Cq8 zl~&BQ~J0v-mo%~n1tyCd5a< zmhS2S4^*sFu;t&u3x~uwN?S}FT^sbWSH$PCZvk{h;7O2(nE}a zMenUf+FUNc&Mn+{7c_qHc{{%XfMQK2s7VYEVd-jFDB96GOx;f2XW-;t(!vhgWmugv zGwc7#m-DRNSP0VB)X5NHYUU{>z{4qizJ|y%A_KErSA7i;;A+%Hjh{IcYCTnD=fZI* z3&zr9T0ee6`Y{Z>34$j5+S$c06us=4AL)<}W|J<0=Z0{W+Fo7kE5P!YzU!FaV zfzxJ2G8ldrdXd4k1ph>@(3R>%Vb=c-ihco6Z*0TY%>Ab4Iztq&z@d`@*`JHPnJW)r z-1}aoSqvVyFu?8*Tzhw>{LET}JD;xIj+7sSKK#0fq9l^8Ev`D$0^j_KfCXU8~T&@l#Yn z=OXj$Y6G2Fz6FE~SodFR6ygfy$EE^D^_pg(#lE-GDx~MKxfB>UfEGZU298fv#{i zJ)Bnbi(mI)hI@x6Il$wR-UFzYvtOA&yeyY}kjE5o?DT1rc~Phh#hVJCTrWW~3qJLG ztBrqf7_Xo1;zlW=3#W3;y-?=}5KD-^}Ur`%{E&o>P$fJKH7$6{;NR<>bjEYt{*{t~Ad;b@LtdMm>@{%g^X}v3DMo?FIplb|+Cdf!Udk;cn zhZbvV?ICuz(x-l_)&0{&U|kcgp5(70?N}k#Rrx9Ft_WM*L0lQ~l}~!5DcmKl3hN;2 zR}PnDgNuL8-Ryvyo9IDu{nj6^f-N38tGrj5zkM;NtHiW78C&>R$l=Ea08VTN&jHm)teFI#`mWFdQ9YpVs6D$RvhFB&&h`NXyf!BCG<1c31J!h1|_^6nT8aaHU)zI)am zZ>3b9hxYRr__a_z!9al}<^pkka$hX!&H@0`W%N8YwF0<_4B*CBW91-U0YfKL>(U-6(c zB)xyo5FmtIqdT~AnohBggd!fuO>HW2z&0PC&Y0r{BpgsPQPMRPU$V61mL#4x;Jf>P ze{~_gBaD^5Nz4DtMTs8izC-^37Dao<{TamnFZ!m0YSeR$GZp3UrY2%9wYCj&PO_e$iO$u8r}oLnQ%>MVB?9NPzVY~ z3Gb~-%XcpLrd(?xERdjC>(@<(Z%FaXrT*3PDZNHmZhB?y zkT9kqWv^4m2GX?2evHn00eckNDCh5$E>ytppx(!vIBa|kETa$puTHvY_;2kf^T74@ zyt!lN!3!#EXOR)EU7N1!{;{9}O^GFB)o)`d9V|u^rqUEi45FjY@TXEi2{trw5H<72 z;IYybsCu2OZMlpJ+RSV=m5p`UeVju{&S!cw-CJ@D&d6x|OP$#7j!`;iu=?!Af zq1MiqU|qNyT2+cmcFF^Z5qy~Ujv%bJmu_4^+J!7oFmREw1K9RdCMMc%*DSPNH1U!a zaXg#tw%1!pyod5Zg}X30PdUD)(pShMKr4VpU*)l60_M%XB(f-k_%cA~ zv##WL#VPZ4Ecz!MdwpeeAjdvEBcYKBX#w^h!XupV$UQoan@=MDHV>j^lkI7Q_KF@H zCuMM9Gaj@=3dRXp4+i~~k92Y@sWr^UIpSt=tvg47%dQ^N`Ld{?LyLg5uM4w353a0{ zRZN>hM~8hWkVmPX&3&m=z6-$VU{qS<9QciFz~$QqGEk78sS~FNRN34tK3!T}T+w&o z`#c{X<9<7)OVlpF_{u^+o`k`C%g?B^{y}}+#b@!O0EX7nowPBod4iMlFiZYxb%Ob_ zTj)L=(<@aRj7lZJUr$m@?t||zLw4&&y3L3)4jd^lA?dhS3om{^Dkig* zvo=*lgKQa2UWVHAY~8*=zvXYVBPbMsm3iVuAAx)2CrHopfY2BetWRA|onEBB`|s4N z0lwD2&+*FpI8ZpIGfWaW4aQ~MMgw}i%g%3JBLl-HZGV71Q}G0q${^577t6r_Yq@Wi z#oo51D>2R1IK)}nn5;83H$5oLYSe`67k~2%=l;v%eD$KS59fpGvh5vK8R#jGH$9YK zjG#tn1oOlW;9xvH`LQgp<<{U7s*RhQ+Bt@E9NN{aC%Z$Qwkg@pmF^`aLL5EYyPZlff2br;L~-?Ju-GfF$j!h4(n|yD>=R-6gLI4 z?Iwxd$s`*dwN?2rOOGCCxg=aWRiNP7w;?Bw)eYN49BWbIVyvEF$X3;FBp+1FU!bE6 z)nEc+)5wQu*YZ0=@==YAzwl%XVp0`7_|xJ)7j6{e1XJx-@#t!r^r$ZC4;Q zik?ar$j~wk+T6 z06|kfQ5073{mn=i_jO|UHW2Y=VCW=R`G*w|h&Yrm#vv3l!0+x+gmK;o_tFh@EM6=7 zvN5!*5yWF=>X-H9NsZn`(}I4=*O@_DP&BhCj>yuv17nE&eLqAD85~^7rllsgGBx<1~3Jl5?aiU~}vd_U%HKk;^bN6WAM{Q@maza9VTM-;9A_(Y7y zNu<5)7MA#}^8#*V>g$U4o>ZwtDs+Te5QS7T^^GQdI^fiAB543=Y9p#~+${gC{l~#c zu6>4!*gvCnxq#2vCZosj_}d!9ix{DZlwou%a&}$N3onjF zZg@J%v|36z=ptNis>Ey+me@RoKGM6d)6Gk%EG;RmFY?j#{EV!TcZ@8TD;^#X^1p=~>?hxrVtV4Fr{bH; z?did?J->)vXg~l$QiSnKgsgt|A;E|Lt+evxo|mQ` zToq2$QFM|r|Fz`Drn4{~WS~D#l<4g`u2ENPV$jFu^rC^!rnYLiRXMemoE&C4w$NCv z&V#l<7BVkzBge(uZ$uvwm{_6;C(EKw_|L89(dbX@pPN61@iEim*~1{BDgDm(b}4Yn zxb?lffN7$5MZg8kr9bJwZSoAbvvUtxG~VEZ@VwMs8x;6ad`AQ2349FII{E%{qW>J_l~3^N`~+}l7zWnwwE3fvV*i5N zi`PfJ5D6stY$dM&(bWh1pX~+PO9~KuzKWEN5_z6M-2z;UKH}RK>6J}eN1a_vH#J!n z6i*&-je5MT%;u{-CWN2?k!?aDjE1H-<8OU=8eA>gDTq9NUh`>=bFqMyEg7}(3vZA6 zwP(HE%E9y9j{=rU6yy9N{Sw-mxt?pHQp%nSBrU?X=WSueW+Sc-_ggXm0ymh10j4Q> zxUWEuXJGsRPyuaoM0S!ordnL(*^@9GdZbrqJ99~9-3y2YF4;pOsh*YB+$bW*ukMVtU)TKf}tn}LxJ%|!DNMK_{@N!-~H|}XR7hyb^ z{ItqM(~3r+4PRlutP}al+qJj&^E-rnZ)+86`*R)@%I`B zj>rNobX=yC4mA{Heo@hUi3SDVC68d;hdlvl#xdeaDal|u20WQ@LjmrUhFS+Vc2;_&;1I;W?blM{&!EDfb+9DguY*~7PLgMEx5v@? zbJE$%M?bg3!my8)u2`;`KpYO@&;Lbsnt%0`Hf0MijSPGF`GZXqg$oqxM+@`m6SYm0 z4sm$*sln)qJ{{k#7W;(njoiR+@5|R zjG0siYM_`@ZTv^{ezRv=zA98SCrJeQe{R{xcbabNFy5hLb3hRvG!DNi1c zEgZ>M?%BuOP1i3zx6pj&yBXJdN7XXgpw!GL2x7CjqR|kY()L+-7|G!?H)HYmx-4C>5A?Y04Z8=Aj1SzbXvC$rg6L*7&ZtsiW8q z-ZE=San1uJrcNCP|RkXAmi?V~`_`v>a8+rDZ z-f+mzpvqKow$zhbv^==*$^MA%1-ci*3I`Fhn9^&B8f(d;QDTk&WGi>&H(W@RwMor`8e(g(2AhK?Kt|Fb9F@{)c@Aw>En1NwUoUL$7`g1UnsX z53JvkfVqNy=JuiZQ2$HMH&-4?0lxE`R!MIIZ0ZkD_voE3CP^1u8NtJQ+ zt6{1UNc1X`CIFCp%8`(5>9gdHQH_OQn^Z4ZcB=O~GmKEZCIcnI(~>^=!F_#qboc`P zNiw}Td!uHw*!Ktm6W*Z5e3#MkwG0wr`V?h%z8*@zDL< z{q!fc*J%J!p-?N|RC{yOw)Ib)>b|7qtAJ&}>uScI*MC?c%?cXoFd7ux(dpW$&C^ZJ z!x2L-0n!^uCxIocILcerUVr`lO>+c#0&4N&<%X7&E#*@jqtUhXf*Vh4@U2aJdZueg4QOZ6-ALo^->6m%S!>>ed$2@?&XZl^Ia`EN)6iV{I1+LyEs!jAU-Rt{NRz#c z3xBHoGBM_#H3;7_VUJFg&f3>XKkYeIzrJYJgF-nt`UB_QAXaPi+TYXY4#e$>Al`aG zeJ~gPj(m4S=>GvWy~hph0JVDbEz^y*xXfelc_-9*@E=5@7;<8lOyVa+G3MG7IA8yE zhBIFB_-r>=Z;de2##IHI(d3g^1@Cy}lW4bx+rP$Z1PTL>FCdu(YGj)6BW6(gU_YD# z_w>bjs;d4K4R$U(x?&L59|r+96_f;hQ8d?)j{{=DIJ)`GJhx;RO7^HZrr(0^%Jh;I4@Z^4IT3R>a9G7_U zyjR=+RxapM=+M&)qPhpA+vUcxhgtZC2D%iuOxyl7F_#9snr>7FDtWe7*oq^O)XI*l zQe9^{S{hyw$`U9{{0=jg!cXRKcOD|Lm>=)lg;2Q7%#Nko2b)35r9v#m(sZxUPenlI zv-_SLiaA)Qb(D2i))fVv8^lG*NAbK^Lnip^qTv(=7J#5lCvHz&S3HM2x{S&2a&bEx zP$SwBsobl8?Hq3kGVX8fVh^D%SNWQa{Q;CDI>g8Vffx&hIGWrl6fjFrT`Z{#{d(hz zf2dvt4;yy|Hy=$qHIpuW*k8u{PjsQt=3M#@%I5b{W}ad}-|%`I#KSv(sI^ z_@`z(-YmndAJ5}LziL7~;{!9|Q&bxXbfzIq5pAN(yP8agaAV-pQ|XpPM!gDL<+0R} zDxR1NLR}$p3{AG~wDk(0e%OHBL5@^~WNn|R=yDHQoluJA6H;vRzR-3{Hl zm3%{8|0rGS+8gC{z|MX+Dh+P9xmPZ=C0St;wgkhHAGy{L9))(#ioMTGQjQA|TSc*D z-acMEN~#T?6)hzcNSqBYTqNfIlma>U@mhSa6Tm3U0Z6%hK zGc6$D&wCRSHlxJ*=4T&b#6KxOD@*941o?Lz@}MKbOaQ{TrFw)p=C%Rn^0Fyc&( za_%N4c*HI>$7-{DVc&LeP3POEMx+N=kymL+;Y3~#;bSfZf%ae6XV(4s%{8MV6B?w< zFeFZs920haqCH?Aw)1N}|Mq$Wyz#41(F{c?GOmiGqA|Ze^2x~Uv-cIcu=~4Sd}LKc ziRf0OX>rtkaMik1|7-<4HV9J{7$cS-MPA|sg_5>Cux5e#?4l&1g7kGT06Ycj0nS=f_d|x=$uyZ|`r7mUaQxgS@MYCr=*VrNHe`RIkgc<|?We zmVG^5-x?zcb;5r9;ir>lxPc3n#zvu5Sl;FKW%<4C+_C-K3?-k|1O&H4(JQ28UViks(d*aJAn+h)#TDP?triF4?INCI{E&8B_-DZjH zf2$=OyWz-~hep;$$LSJqnqx?c7D$1mm@x6Eye%ap4IdjjwsP5{&pbD?5t@AfdLNWX z2#iP*v?AfhZ&@{N*83k6#ux<9Jwbd|v~uFDr-S}JRm%2J)>u#R8rW+;4te?LZ?RZ_ zk&-5pw8Uy0L&CD*K~M@$VGXR$8y|nxPl|lY#7QEKtDGw0Y?!`CTPsQgR15n%W5uzy z|BYaTo_3bI1Hvs``=-U>pG*X*vnej zBLk8a-&kmweZX+ac!HpDM%4(C7N7q0Z~MLD8ro_`xp*nBQbx4cy)HJWS?!U(|Ii{L z_YqtsD#d6VD@z(hXfAh7zGwcSA(isa)hk>pYg6Op1L|YPCyu!@4{L{>3L`fQYL44~ z(a%EHY*ilrRhL1SvU!`eW*Lv7qVxE--B7acL6{GPj!iCTwKT%szHU*QuVxh*mxg=) z%OMHM<@OEDcKe2?THWzSvUj%IY1Tn!S9>cYY6l~h~yB(iK06YZp*E?Rd1!V;Y;1cbX)D6L(}L@X@PZA-()V- zXlTGM6{*v5v_g@LLescsOnvw?T7 z#a{XIKYQl}*}K8i29)x2ZK@`EMg1SJg$NXYkbED+93R*DaG(ylT=&5Eg&x$E zS{xkxuRadupU}$(GdZSND9`NHN$j_D34&K2!$ldPwA%B~*n@~P*F&VV9>~K2TMlOs3uuOyAJ+tMv3$1YLWj%>yj+*rPxAo(iV@^eAT!uM6fSo!3D+%yF$j{PM=@#mNgU`3M zgggiNp1OiJP>x=aVZ1kIrH0}?{2uroln2@gf0wx9sspw!Z|b_94;pCK7w)lQzH7WX z+2>+OP3Cb%PzZ`yS>3ZbM8myCXbNDK{=z<_feC@Ft6pW+?eDaX&;o+uf_Du4gIcfI z?)-}qg9)O?*Y!vK#oc{(t|#WD0Z+fT zLp9u9N`$)IQZ~>%LEoVT7dK)VuS|Vrj)0cU;w(;5G)n~3S1C5^n(mXggK4WY6kc_~ z==29?_Pal#{u?1vtfFwT%xMBF*=vkM*=`tJN$!#LZ6rl|R%1m~(r`h=8CcQ&Nl$OX zG`yysm?tmy{+7ux7#518#F3oLDYDF?3=7lB2w~FV{6DRf^`wK(7~ z2MGSbw?{A-3{AWG=h65GBhmBFta6m8TbDiRD(7JrC zqh*Zoi3GzD=tG$N1LJa}g6LC&b#P)(V(WH%7f370d|Ymbbsk|z zwiwTEya3kElTr3JfUm#B8HS)*IUV;S2vxPK4Q=*N1K zg4?$!vt-|0dznqwc9e1X4Lcd<709f2#Ej5|adUzZxM>;pv$W^`W}S~aQzy0*hG-UE zBcf;n&qC}=w_YLoBI%`^AJn*CvU4vrw&%SQd>(g<@x7g?vEkM35_$qv>PA_AdcT(< zF8Y9+BwpxD8$X|yS~~g*#Xm)R9u45}eK^&4f12JmG#~3RbW*h%ejo2VMw`#l&Nm8X z>I$hJ13I`?V%H05{9$-cPbI{@3(v%L!7z8ZcWZ~xOvc`j^> zai5GPPtUIt+?htGp&j}#LGqSq)=ML=aU$*meKC%2S+D2R1mpRD?8yJ0`o;M9xKh&1 z&%F{_!0Wx@E3mBOs+TKJ27Eu!BgJ^$0W^X7-@0+x?1vNe0&2yG|31XLe0-S&{~Hkd zuMkD<_E4|HA{vahAdh-(!j%D!R**<)Cf=FzW8#SkCYWG?2__^Ky2a>_6PCB!SnRI5 zQKS#`m)==$Y|vMCZ)llZ__#mTr2VXaYgdctG+V?aiQy?(W>gV&j1tYwnbk4?`}OpM z-=_Bh&WnmUcPO&U5fnutSVH1>xiBu1OSyMJ>y7D+eyEUE!AG{7lW*SuVx-&-L2Ri$gdP)qf>Wbi1@JGkGsJ8thV(Fvluox3YE zMUjE1G|}W@QKxUh>Kxee#NQi%4cLuV16(61LZn5FBs5h(#aj_=p{d1UoDVqZEsjyJ z;M59r@jUcDd=n1`&dZ2`f$&9rGy|Dv7#eTB$YIGaENRo+rSn=CgkLK%1muZ}PFn^P z1ndC&i}_ieJU?tM)esSWZ5a0T(Hi%6cLSN0LDbDc<2g~%a7M!w2`8vG@AiKBz=7MX zC>d-{jMA{JQLuGja(v}QD+dOYP0=u{!JdKd|Ko@l_fDl4o)kz~;3*mv_ih<$9Hzx) z8IJX}H>avCi0r46)JR2R36fDbo_XlSr%tXITnXS`gHkSWlm*~$Y|oPSswL+E12trL zDS>kh5dH26>QEmy$eSmtJ|7%nYPEitt%eS@Qz?-@aXvTU%7)_WuJ~Cp0-GQ&{cgk7TH%|HNAbEy zRtN+GRlRP}7i5vvW&9 ztB3_NBtvn6MoThIgjw+-4TNvSa<8w}^xN4Hi>7%_!#Pz}SS75@HY7bGVxDE)+gy8V z7EOWTw1DFz&fz4_(+o_3R?!)bq=u-u%*NCX zxBO^*9B?KJ(Kg2#FERovX#v-kyY{`Cwcy{?j&@N<7fYP~#|oFqU9jyEZRzp019jA9 zV|7ozvg(h%;E)?+NCeBm2$I0uzdy5bz_$aqrqsiETE5WLJbr*`UJ#YOcVk(V zS3bYy0BA$LVH-pa7e$HSXqqFO)20miaczx4R?VxY0J~J^WE)k&ldh^fCI_&f{QaEhwk;ol|}kGcAK zo%s4^vv{#Dk5$WG&>K{~&=#?B5|>q3Pz6z8sb}~U?aRBiCl5?83a?L4xPolce)6gU zfYe@WQ@6XTbmk)Z;h+RXWHpVJ6(07(F&mM>^(0w**w_2FY4@X-HGhr4g|wj35=RjXjuX=Bb-GVIN3~0aEm4KKi5==_<;7cHPitrz zFZ7Jjg|rX*sX=o4stqCrwH8E?=S7mm6`F>FTGxjpH%^2kr)YOCzDfL81kPp*cd9X+nYr=@GoX zVH&<6%)%Nxb#B4>&p;RQD;snHR|J*Q6owN-=GfZg7lDCkf^AP(cH#nd4WU;Bw7dSwpR>tlmonN35lL@A>M%wC}9g-b0ORX~(&S zEwS$<+H$B{8O>isX{QMlLn30e+>71Y_22i_?gjqp4eosP$RoUKsE^B1eLi+8vK>jGJX&pt zuYwuz(&ilr&SO`2-W>nSlYa<%yo*x3#^$=={ABUi^(0GC62;>xLBj#KQ)Aw)9>vD} z(0=7VZJICM#d8Wl!2;>MF_QX4RX&Mj zdnK)U6Ci1vksLOb#c%v1oWliz5><^*p6$N=*_I0jJMY&&^-sUD=%pX^v!SQ1s0s8r ze@0l)pGB<3xotz2%;P$~uF>Z6-=%dj&SSwr}(G zR#5hbh8oGdC{ctWYXqSZH}NNLTQG#Ld{ctPp-PPJeH~-u@p|yM+R6sc`XEk@v2oT1 zOEMFs^VCI8f7i#2(%ZF8-%HdO%XvM?azz1{#}(MRm#+P7z_*RjOlM$g^Qo%hQaD0V z0y;gUrXGCxxpjenjbK<{Ylst7Q5)(??9I3C?!P&sF+_A38K!6qLvb|COR}oHJ#c^X zMY}hrp)_9>>2?QHO633J1=M$sBcXcQM{(jc=P# z505;b1PIaT)k0GkP|u6d=|ybz`DM4(xGS;(fvO2C!-}*(F(ORC^oq)NzkxMxv3d2P zreNR=RE%;6QgC_YUaBWw^#{bf;k!2cGwyU` z4S#_nRZ?UrK?^&mf9=!;QS@8b7e@wOv*0k0TWz!&%BTr}a1C>N{RN_~=H(lky{Q{T&%#1e01F`tMOf#jVJY+V6P&MG%6#n#lDAscVCq-j{8{;y%Hg}8BD(~Z{Ad*R z%f@xH9Vj>cm9dgCMQ>yI$~O-zoYydt5J@3uT2@G26xCpC;S;yqdnj}D6B&TSrLB?E z@Osx{|Jkx6fA%{Og;$I~GlZhik|;=}7hY||UU+fw!qiC-*DlN9B*VxA$FnTFcKPt5 zFUulIa}uR1?~L)iL_@VQDhDmt-ikG8y#9v4??z#~NQEURoELbFhH)Z04Rx}9h@F1X zaq6X8+oMnK@c4dmp2wrg{)?vfS=nN~3ZLlrq(?^c53z%P-m-tyCk>2*99|beF~!aC zTykDcj@u8razJsyMcB9<)B4z6{>v6wDuGkHrfG^KX<9&wem` z($wQ?EvO^sV~F2v5#y?TKgNyRX4!|0c){K2mhV8xRl>Kz92)0bKGok@QHH3#0*}9& z+b4@&MOEJVa>T64>LYfTddabdrm;d!1vQ8U!?_Q;_szHF^!w33HKK4ZDV%mm1#XWL z0x)QG%V+w{p1Nz<>~zQ7iPupuqv{r%W8=QcmGjEcctOOpti=R{JIaRp_sH8H^*npn#JVVD4= zmJ)s+O^UN0TVB1&oRR-TyonoXjX3#sTcM`rg0x&Ff_$7s)TaD^5ilrqKd=V!t3wpx}rH+6@$QemgEG2B}km0X_!3P-8A&W@G8E- zu6?BI?K$~X!v{`BM|ZBy?M_{@Oz2cgKd_R*vosCIKKO?4z$aM$VqsKk{MMNc!A))XXo4;@RS1B zi5v7ZGcCTVPhvF(?^v+z)jHuPt4}9I+86K&$E&QM$t1_#JtTEy@jH>nVm@bn!;-OH zFD~MdYLB8y(i%aDvP@|$cMY5V`|6s7YLDN4mL4L`d`7qHOhQ)oC$@tgb*}9%$fRTB&(#^2davG~IKKWJ4V?Y}Q zHxfp0Y=DFpClAW=x&7|E+-|BS7I^$GwdlU){@H(FP0PuC6DzNIsd><%_4E(6u;Isf zGdi>=CQ-8(U6N6*f%$=v7CoH<13yF+l`w0Lc4@{bfZ>ohzy|f(9_2Vi{r?X(dDmmp zUzrUEvz1mO!b~C=o={1Vmq}dtY4$rC+kBSZLI2{TbHPt`9S)`o4h7SCosGB9yR^`z zgZDu>sPAByT{ZarAZO_N`w5F-9`(GULvfxnIb>m+!Cv|Gk8`_bfVmxRNOUVK!E&t1 zsT_fym|pAs7Sq~c&p~zE4{d>?^t8Gz>Dqp4FeO`tV%vFLv-hbB`pK~>*wO;MCFAtl z#hQ8t8r)zrT--|_=`y3sia}^kQrGM~IVYG7d?~o8N}Gdu5MteSh4<^7TA>0zr*0q6 zR6k>y=`ZHkL%&-L3m)}yuZM_dU~W(s5;Ot_!~p+t-NE#~dcF%nUQ*H83-79{^;_~C zI(~dNH`suOU(#hKx~pf9jl73fG0P_PA z3umwc$9Jw;;cOZ%j4LiymGg3C(eGk7R^T`mCs+}OrEz(76ghGp!{(fS@V?g7!liM= zMP)RD!&j`Ul8j|!pzQP5_969}zu5?=gp4UFdE7)7MUaQ(hHD&a7bSxa+F3(W6u|$t zL#k+49L?@L_Qaz7O(6EHIAzN+K|{Kslk?TC*=~>Gk|pZw)RpO3_ijv?0-caxM_3kx zot>?JP!0NiA=FeH(bc*MVk-dv6Zj!zBvx8#xwP|Trp3yKCp+xqM~5EA8iz|-ifen1S10>i z9EtM+M@uwEkSnWuR<9{0aOvd;O#uvnCONzXDUKJ~H#^$=j>sOXRfS|#S(a%f%u?q@ z8>;9^jxofd1C8&9Y(GsXDy4CPqS0*F6-l?ou?}Qq$7yT59rl4}T{eOtd4Urcj;AP2 z2)kkWNDoqe%XCx~>5kKFKAgM!3DBKa(hYAWIF6!tg6CC5eVu#$;TvxJ&OSgNs}8!j zC10P8YS#w;sHCsC|LLN>IZ1@7POsyPle)VKV3G4A&zt~ z9=1$lJOlmj(JZuH0DCHj)(h%;cwI0rb+?A}|4u7{w8*?Pll;+)-9NSRJkyI zz|cMli90q5zQzl0MbY^k7Ne-Sd6fUb{s#6jurGmqs9`Bdr0NLKFK9m4H^Dju>w2QT z0RKVcQSc0uUy%`d4uJ1RuMeg-Uu&F4wGRbANVff<%#iY){5q(DvDan|Nde5zf=B%M ze!&Xf&;|HOx(NYDH}?#t*8$;xz5ylu6#NcY``-hEAbLt9hvvcfEPPiN8tUbp)BVuy z)z7!{ZA0G!d0hrsST5)%n75vg6BjUV;Q2=u<8$~U#Jqv?FDo_le1HL*C97Ty;YCd=P+Y`QYz-`ro zwn(6|tw3g|3c@TEbBJ%}O&oW&37Rn~;>8FGmq=107@S~I_l$gF>)A{Ua7H6ad~4@) z^q=wi^z~0Rfn=nHE}0l!<~fSwC6XiVdE)E$9*H)puS5>5PGFW;G~PM zB&TEkert_dDu_{H5Z*(-@T6Rwef|M?iW+dZ=bA)`u8G-dZV%DpQ!AwnUs_yE?=ju5 z{jomQ7+Jz9jt$56w{0;zqAH~(Q?w{cl)&*MtV;RKvY?FX>j#5X?yUY)NItyEnvQ?5 zwG#*U!NHtu6u(g>1AenPNx{XSUE0QeU$y?J(X>K4xrds?mRc;v`Pz`lB&=unAhhQ0 zs~TAn36lx%E(a@@rz55Wg-6jORY{2LHr9O6Sk!Tx`jcB>zrQXdi-lNtl2-^prbSth z-@Y*7xpZo1A*odfAq~4!(VZ?AppFo@e^yHaN42FF-m9@7vUQrQ(Ht((lqe8kR#rq8 z)RJ!O7;?gX!$T{PR#qHC@i-|6I6*QviNiu3=YMoYAQiRgH*wrPX~5tA=A-3E6eI8y zMdCQlGK>)BH11_xF5VvD8>flm@y+M&{$OZip_s-B5<{?>#uMzmgV)t8E)?q!;|XT9 zG{F-xJHnaq{6P}FW&kbi0v=l@1q^3R!a*<_5W;oNlid`OS?Nqff(!`q6qouKD5W) zc*Y)38U;sDjxF>EhVzBAZG*_^rGlbCY3G^y>8s9RT;bB`7swT`FBvUJXL5tiYA}KjHW|IivYA)uY7S zBPcxJ42+aSEzAKh5}`XfZhVo?A3hD`e1TzOaK5lOIW!Hgrn6(qqc2zS9?A?CEqQ}@ z=M_nbQ)xxyDH(5h^v<=Xs`7;@yKSPRMh8$*2o=@E@#DbTu9>qrvSTWWku+6R7(y0l zxRcjL-5k0)sOxXs-m`H;CtSQD;RHpBBF^B3PPj%wDGaFpi`IB|N9}=IPu|}ch+tm1 z->9#Q#>%8dsERBJE;(D3$M5N~SI2o-(V?v713m|n%gOpQ!2}b`^BusuCcSQgdEQ(yNRDxwO*+v8 z6QTore%Z`@%;e+B@Z7sxdV#5N;(JBqn=AS`UeW0+YXxoh)rvQ_WR&e|>!G!VvV1G05xn%MI6XL_O877^1Rf8Fy!5B>YXA`a@ zFzJc18Bf%>#(J)rbc+e)8Na8Q=e!B#e8&tGb-h^Q+!taU=R}W3l8M6Q5{8;|=4AyZ z;`#V;zOHJ;*-jIJkaCsLOD3HbgqzJFxFpcBl;3AvA>)u`qJkmRD<|U$&iw)9bes|; z-|AaV`qMo@=T)~D&v!UYhSHQM-*-t6{g4vp_a&uIh|5ebn{a8@wFxGe5Ph9Rvzv@p z?ow8A8n~Uqi^&$dCh{D1+@}rd#0~`VgVuWLlU*Lg>mqQ*l_UC9uUquEr>GY$TsZk@ zdjE9LlD3Y}(ty`1tEGqI`Hqff?wZg{f3mx%TuF1eCXtLw^m<)dp0~RwXS--zQY2B* zBw6L#_f>zJanS0hfoQTl%5!@OhoVv`->+S4ii(SYt(KmSlbzS4eY_qF?u|)dgW=Bg z=LLV|P&DoGe;`Tow{vEMwF`N51qEo_Rpj<0Aig}#Y}qV1@cW86nCrfptT zIeeQ_WEaVD@XodtxwG|ltf_l`*K;2<)}I_O-r0P4IjW1M7*)k3is5-}`nZm>n1jR9 zYeJ1|0;Lv2zO+!Il<%Xrw=BoAYk$vjo~)}s$qye9E!V~HqDbQb5Q(N=kJvT8*SSjR z`kPCzQ0}`W#ErKuB?8ziH#y#0aofvT8(Zm5#)P*qhu|rf=FU}ysG>5UAV*TY{l)y7 zy>7qi+x5_$zpWa*Goz4QoEuvMT0?^de)rwlWbui4^Yc6E`I{VrS5C}$_k9XIguym4 zIOiH5dcAoQ-MM44{G#Ocxc!q|Zr>oUJIC#JPf&daraX7%`03PiM3Zt8zPN@vJ}$Vk z?e8~%vwo1l*oyfsHP`QTt3DTn6Dld=oFsDe{2BeL-qQPs6_95$8iuAY)YMPTy&1-h zNwSP^+_hxd((d)@=uhyX#`yeR^)?qxk(@viG|v*8K+vQu4LZj7UZS8%IpdW8x8J{nf|bk7dx7>dF+RWC(6NNyB_|l~ zVPF!3|K)`TZ+@BTZFQOFx4hPuXsBOK`dv{S^(2+kc9r%Vj`h82m-yT_!Gwymj$&OW z)yk-yiQZQ$>OO5x&^(!NbwKq5zn3UJv%HQ|RTE{(HDFixI<0<{&F3u$eymXm_m%2V z#emRw^Hk^@k5{|qMkqz~D>(|C2g`S&Ddr}}$S>EAlfO=i@qg60QBrwcQiAxz zlJbT2DE(o|xF6B_7b`!U6s3PA9cqFJiR0@`xU%_>tF3?K9q%!7shD7bxv!OTFz=_z z7Al!TV8T_*j})nEf3*|;re7W~PB2hT*K%DdsYx11XgDXUooc>*;G;VItSHmmttp{t zv=wNzjB>2HdF1V%e4VL3nQJX>^p2pm0)PIE`BG z?=$c3YqmLnL_4E|J;^BL_pz2=*5wvA6e~UpR*T1x%Juta+Y~Uu2&3SOBdk%hqA8-l zk_`8UXWY&a+r}iJOxV`cjIE__Am01T8eci6?=2BVBGhEg>v(AV#@*{b0#-^l7LA10 zt9n$?r@FivAt@>=38JiVyXQY8j2r1o1J@HZpwxoM7ooS$@lxw1$E+U#Te=zHgRgHV zH0|maFkVudkEAXlV#2odzA3t4Q%`?;rcC?=QP1O{AjwhsqJ_j(Nlm<^lJt%Or zeg4DVqn>X8ZDV0QAUW4iA2-yUFu?~igJ@3%k8Z$d{_(MKry@FqWK{; zR?|d{6F44+?TGK%kc=pLilg%DIrG2Du8l%3lEx{57buDoD1j1S?3$pHhs%+TpX%ta z>&O1zKZHhLmSlOBV+ocPNP!PCh5YRGU^nKap%8eQqvxStR$jxm4WEwCB}q`O&OZNu zJVkYlRsF6>l<1n6t>zB(7O0idhA(v&+uDB4w}zzAW@(yZ-?TsTm6yQz+t~Qr4w#@a zgd!7+pb{!hyk4jK)N@q3WZ2R)J;BFgdMW>hA=ZoSXx~Raq~Y29$bskl!NQNksj5(PmZV9M#yBotlqkYme_pPs%R2Sf*tBsxge)jQWuky)NR zzt81WwE~|y$eo+3Dk;D2_#!}bgsCIdjEtE!%VUoF?|Zx&uSFJd(3;AsvcxDPEteM& z$77E19e&I>akvSZfAXrbYoc56XS*1J(w=(_+BO zkFSd0g@X_nf@5$TXGvV(VbZe&v_rvFJnrcC(v19P$Fpes${{J-rKp-aSMBGPdgNxg z$L5Ou0I(5&; z<^$}r3+b+Qm5m1veqL1PQ;r`-9sKj~3?Me?6+tk?laxk@5+%wqH>LioHNI#)+yeZv zl+;(JvxMI#$~S0P`51M^EVx?c7;@W^hSoK$(1NQRj&rFKL{EX}SBDhj`rSF|Ag`*p zWznxrKeBB1+z!rk;AC%v?!k}Lm5EQ2`T}hl`W&Pa^j1k#A`5#;3!r>Z?gT4peLv&a zeb;^dCl({uH%U|6$Bf2zy1a*)pvSl9EWjCBP-Io2 z7)pezupz5>+41#D_wU#`9#nC}SlX*jP`#5~qQKH3qbf8ZN%mQv=hdIP#~LlB<`jEu z&!^Z+R!*z{zK6>}Q%A2j>Ni>G)cBi?w9%$< zw!CR6zbonc=#hENvCCKO(C)&s&d8S-jSH>qkN3uCgIlx({?E~K^YCs<#y5;~&BKcvF7PZ*GaB#mdGaQn zxaHnMnX8{D#N~~26M*H(AB!d z2Qf(CYW|c?JPjMI)1)mz+ptM@%>Vppotx9W>bDo8g4kg^sXqnue}x zMYq@Z&+K6OUsu=0~CY^E#K%Pye9*lwb!NuhRR^Sz}sBTXY)FuUGJz{`~m8!E{~Y(DL_n z4^lb5zY+Sp>B?XMygniQ?+d1L^nuvWu4q99U^3!1a86RZYwJD7W8+5d>*Q{RK0|XT zLU6f#-BeK->XUn^o_y8YO_e<}7WaGq`BpDEYF{ETNj1l%si$ag-gg{oKCo44W(1d< z1-b6LT$fk%i(_4^ObI+kNgOANwKv!8_RQ){K-h2DxU79hC2zz%Q(GPX>`t~Fn%x{N zcL+8n_o8D{m!L^JLu(qN%3*Q=Th!VCB5Q)EF#DhD-uM0S3)Akm#&)=Y*H9*EfGS*S30>3u z5bRt%IiD~iL*aoS8An~Q=sQ(#=E^;A@gg_ z>yv4uV_Ks*Qe z0*D!X5>a2Y_oKBJi&2(_VhH-!B9|X)c_;xq1C^T;<^5>=h}J$-KjSsI#iEpVso2DA zj_Y3@{O<|sf2*zT9Reng%>xxnSPBWRJu}9C~JL+%2A$I zj0V|n(2|19l@iq=JOk)m~NYlz~RcQ}^*^bx)6T_C+0Zp1T762Zwl zFOaxMyA;*u_vTHm)@1OB&t7bimI8e^9MRF?5jq+we&6Z%(YfHBr}oF7)kz2pEs?aq zs~U&W>YSh88Jl0q|6ynbrIr$YAAMjyade-#6?>pLQtz1JSdyi1j^QYdWWsK4#a+Sj zQjignK6PB^{+@htEBbPn<5`}^DF)|BhUddno98*B33)zq?EdZkmACn88ZQK%kF*ZX zA~h}cI?k@?*nY-6K(lC4xTH$gETTDAmS2>|_f;qRT<%=AKUkG_p!UqeBFzXqr*IrCYD~0SMVm8pe|r2_GG;dfvg4v#XpV6YW(-c-qs%aHw~c_1r5t6 zjTg+wvX>UXJk^WwEI;V*Z(5Q&;8tK;QA|#)soUs(j)0gg+TMwG=xJti_0Tn=N$gDX4jg%G)eQ9d$pr`x&J>xLo)!Jc4N8a)DN9 z3CBrQ!9~l#Z-<}yaf{7YN+AafsQ4f7@pWB-mR08pviz+HDyJ4C)F^8JLaFsoKbk@F zodNwn6>2XEKWZgtoE(-b8;Tzj#gFczj@7UE^N%$`?&1V5F$^b43`yefJ{s$w7abbk zIBp+S?dZNkjlya7B1*~S_Vv!m&-41T@>SWbxmEAj>;*4$Ek?oL>$@f-4X@~!wkyNWYXU+^lGvQk%rgwi-TJ`fK!uD!Fw5) z!AnCH^{b;zz4x~N*|G2(O*T45Xd!+beC7J!XaY0iS7(PZf6qcGfEjVN^M1s4uy7Twr!JI_*PQlo z#BS1TGMJcKmX>~44(t|v4Y_7J6E|a z>5awqeS7h3Uw=5>-Dvf-9WT@Z2luZ(e?Ip7&i2>bRCreW<{0|Wb&Ioi=$zd}mkq|k zE!LH#=7_m|w>-$3H%^uPF5l!FDbLf1{<-hBwZFtr;_}toOD7-)Q~aAF%l@3F>joe| zTw2u9Vw*Irskk633c)JN#vOj=i4Omy!}kdAO;@{1JZVJ-&mRt{Quevm_C!1_1)OCx zMpk7>gip(Tb21In@S09L#MU*>Jo#c|3pGY#IYwYvjZpDMJ!Y*d?j|EO?%ths*c;%} z^KB6gRVYs37y)NRkrLn&yDqZmbB^V`98U@#>&KR*S26-?Nk(w>^K{Q~`vbRCN?w8F zQM=~lyH(%b3omAV@PN~Kw_b$4BU-9w6p>UHk4C*{l*5_<`eaOkRBT}z#UaH+1Qr7y zEdWMGVFB-tG9P2~a^Hs9LqMyry)8DD9RP6P^Mam&Sv_wLJz`;2?>mUwutAe)fyM>J zokZJ2tWnxuGscs-yPj^-3v8h`ESe7-pmq;*zu(stE4J(33;HJM{Ou>Jvn#g!X^1u4CS? zbtpc7cJ{tQC(-nB&+5afHk;YP5@`UQj+-%-ddw@1jBT zQofJAOtm|0U*9yLK{aRi$SAUl;24*br%e8+;3w}FU-~m(dxMh+LeO~$M))*qY~>eW zh{Jj0&9|!$%|}Y91XAQ=LFEOUmrzow*xhnfb1v(2bBzoAtA$IcbbJh4$+`%df(27U z(~RpXD!Z05A3vh4&jN&WHd^rjXBmRvd0dnzwbDDQ$G)`T&2(TQ9_zJTqTXwmU>xWh zg*dU4aeZhxS8eC)`McJ9V5@_i{J>R+%us|Tv5c&7Fb+(#M#HZ2W~Y7c<)d_gIX^(h4q0ETeJoJ=TX&Qlqg= z3K`q_&Wx9?Y4^&`2zLE~!1DxylO(}XG!3h6?A7nAQjmRY=y`% zx)fE-?|xfQb>Q|vV>@m6Cav<5aT4CK{(rfkr0q&B9OfbO&6RutuR7|dXV*<*vS_tRCq^YxK zYypy812?7aTz?)=T;d6rn&b9o`MvJkvHivTo~l1f7CoXDmMKq#YATZVQqG=>=lt>H zZxKcR3@#{|s46@|8@rFHWs97`_1Y|)v+w#Xsi&HugS7|*EigQb(+nX{Y?xib&a8&`qjJx!Zn}(0?YGwT_dEvu3#cn zHLeHTL89*P-hvePe%CcBD(SeiP=5}q4#4+j_9WwezYo2a8{<5ID_3Ct1r%Ujgx=GA zJr=fep}F;56;QNW^3LpOlN+{vzghS$9jX<`2EtC=?DncX+#c1Xh<1jyK~HfP zXZ@@n(>ryo2d2`+Vs!`NiqL#8u84Hvp3YN`uZ`PMtud4~K|BxV zX}dbBCMCCT^&wbD#X?poIWB^yI{$oh_r}}FaE$C zL!B!e9$(KLku^&-o>zE=P-vCI;TU4{ek)()rWfd*?4N^d)R?VCG5VT-l;; z;KW>)L^CSQh@?zO6nt>@PK_oR9OZm$PlvC3U(^lfvx?Jjq=rirDGEG`zqUVZ={L2% z!~jk8u-=K!(t|fCb-M)5-X(e-^y10%_(B7MHjZ+xKl9ey{WXx6c?3c89MAIrra)Q0 zJT2gyz*F#L9{)zLg==gm>S44~yRbjy`vah8~p4K_nH&OLI zA?!Hv(;c5XVM38g3Qd7WrfAmGRTH#Giu!h|b5He$TE5W{1jQO-*f%=J33i{9?;W73 zimz*4u3w$x-MPS_bK9>_kW@k1lbYc6U+#W`wI(^M^{Qu6?~WXUGvL#Vml&BK zIX30j9WRJ4P77nd9-16EkETWxXU1JiUOuxdawKAc;t7Q(0{&>&SVQF6EUL4rZ|6T% zo{HoZM!_joW+{T=1s?Va11`wzPzu+iI_q}aQT>L^=<|qTX@TGviokh}4mayOzA$*m z?jGwv?Ae<0L3Y^$zEcMvhDbKvvvb7!E?zNLQI$bncaGcdo}l*5^{ZZ=D*N4exg%CT zI`(w`*_LTn$qtiR!P*zCy@R2&tlD!ul&=*fjaFXY5=y!ObT<#nQsEVP|IKzTbbi0% zk7LNai>67Oq8WyxXqKla65hLx>IUT3B%$cdT;~|k!6*L(Zd42|9RdtzFc^9uf+J5# zv_R3^xfVP5!z0@idW_oAEwqZd`i;vc*3dM(3Hi?BZ-2gL@z)V$oFt{GJVQvLEaI?? za}RXNoQq0j`iJ;}< zFLmB}+Wu8x>u1I(p)GJ;_;%UMM~_AvE{UZXUKL54=fj3;In?J4 ztNOMRo}+7xcI@3i~CX`-I|MsW^2CDizlLMEyKZ_37XmE87t7a7q#x zk(OnNxp{4cY-g{6X_F@m0qgaZ>fv-Og%d8ncXE#cpV~WDQ6~*i zMa3mc`1s#%I^J~bKrG)h^u4Ot{3LVy%5}Sv%=x&g@W7lW6U_NG!JKy!Ofc6&N!yil zncI<6#{5cB3EP)+mE(=c&zN9B`2}=f;0K-gqw0RXq&_e~Gf*%9t|eSuvsHP#>g^%Q zqN2KTMDO_iVy-w=RfcaE*m6Vf+!Xju%k@L(W%=zn!wx#1~`VG37hwLIj!d&CIom9Zd{NwG_|ixO9w43W6Xc{w?5KkQe# zC6tC&^RRPU-k3)wThSqDX^9nRf+SRmK)Hzrus5n3?|Dvg-^rQ}GzJ4Xqo}$f5w(PK z9IFyCL5R|q13z#0YVFDy7*xexD8fo<0quQ$zBBFJ8(L>?2jcXNRSlX<<20veEGtk7 z;gSPrdC@W;pb6J;-S^x7S3NBq*PB{rcQ6G4@fCW_%k+GeVoRhrwA9~&&8BYFgMg&h zyR=IfAi>j!|IcoV{5{Yry+I!raP0-h61gqKxZhLt`=%RTFP~dnT9kaW=Vd)VCS00% zFK=Mtg^4F7T)FT{g3GJ%gf$`MHrDB$=VPP~;P+s;9QW6cQf*(^((?O%HH z%j{mjFG!_uJ>wjjC0)4%9*^6X9WbVmvsIr#fCC;5s|NXe!0BD%jMyqN|Bu_E|snLWc{jh&vn_PKjG91r&Xo3&vXywIQ zA1tMe_dBEE3{@+B+W7$UsclAmfRn}4ij8~+YnqDFyejjgMD9CyUCr97pU41rR2^at z^10mkXpLJ>FFFRafQN@y8W7*|f=rMk!HA4FEcb=;9qMdK0ps_HPUVdPcw8mE0V-riAD~9OFuW}}1ckdMy8YQ%p1g^JM1OW*Mc(n;cqd`{Oj1U)Xfi} zt&d2GB`As&ND60ohD8O$Y=RW|zvz72+bs93k7BYuVrW93X;Kn3N`|eEPTUf@kvpyn zF1mrOl0(w)CaiNVxM$G%SC4Vwy#yCdy5>8@yLzg*s#o;qdAm){6?5EjH@7c8;MdNj zs6M|pZ?a2)5oBM~o0<*IwwwK5oag|&8Du-c*B6Nc@9v`HQHQ9q>Yku_cX_5f^6$%M zEzg&ctB$fV&;)%4CcBwY7}WCO1T&+Q--BuTPX7VUg+6hXGa6dL5;UF}UxmKkMECu% zvvOz(#+?R-FJeB5@2D5w*O6O6)ZaM9ek5B=Fu?>9OfbO&6HG9n{6G{r-_3~kffOg}GWY94=hj6F8o&K(L{U3I z^E5+g3Lyz3ENZ8_)kGXS+ntj>v46YrQ{?eaQY1;y0z(rFP4i@!Pg@{dMuj-r?mW9? z1;PG8>Uo1prr^28(6p;x-bB@_dR?ACn&nqKZmGZO&(2f2iXP9N5p`BIv=2@Kvc>v) zVx@6*I;U)#_5E{$KqogEjUOdQxXQ7L$njEJ-$(VAFJ5}duSiiIqHS>^jMeg~^8-)% z8>jc7eM2%7&9OYmF%(6!0te6Qk~f3M^B}T{&zxPRZe8pfU{FQTsIm+p(mWTq15#ki zl4?xeGOB~euf zieO|ymX@sF{@T;hVh2h-{8(D#=Z}>$jj{F#`VMvZe8{=VwH`j%%wN%criWSw5qLm3Z8?s>^Ru3oOAwp-HkuRhVMh@6Xv3Q zRX0@=3q1ZoqF2o6lBY}#_)~{(IW19dIgdEoy(&6Ww%0};p*Tx(JVnz29bl#5opzz_ zSgdyns)lczy^h@LZYcuqWq7M_Y@&EJcx(<)$EuTxPEBIQnAo5%6wjb;Da+$Z`F&<7zc;06oMD=|}+0+Nh|2E_D^tXX7j;Ub{<=ye7Gn>mt)~dEs)?>6xX(kMaU$(4-Ue zhl<{@s=rFFu^IKX`<g_*&OJ<(9}^-$8pP7JE~9l;*6X%fDgVFP zAia$62<$&|^%RJlfIqv(5c`rHQEu`j!jCHr5~W%Ii&_70eMd8n-mS>xE@#Uc>?N{iyR8)MjnzPGzMFD&H(1{y;{bi{Jo@jGcOL2=!M`#mkTgLP zJV#Ojg~L?{?*`M~5WTQjesa#Z?~fJvBU|WCRvN{Y{%ln@994^jX2T^_V;PF3DHVsErPWJ|w&G4X zvkqmp{(d5;V~k-Pyh^jGCh`(PQk!OeF?#CpwZ-r~(YwGOPWq1BUGDh=EK#E{a5U-i z>+J08?Cwm(3graTADO-1Uq7j6Ki26d;2D*sSe0Q2fI9v3>PYYOZ1=DR0+KVtT$HC8XS!d%&2lcZ2pw~Tb2`P6e!q`B-Tr87lDg~XTOa!Iv$JU< zFk%BYFJwoMVZrdH{^Eof^NQVEwnc)17`^UF3M~)1DDWY40w$j}7s%(ajF>OSc^Ux~ zgNPf)^L{{zbs^Or(_nBOX{EQ%eEmZ7qq;S)q23VCc-jh6o{I9ZUdeQE!8oi?I;e?^ zlVTmeYBbF?4VA9({0%VRS{~oV_`5db+b4R&U0>4GRj-Q;Z(Swz_*B2WU~6@G^l!YE z8RGTZ2KC#GrR3)($aCCbid@Y1m;1Q5SmmsAiON~&64l3;M=#EL-)uUQ>0RIY$mFds zjz?C>Hi{=#(L4Y+Jp@J3ti%FA;#5b&0pIUk(=-K6d}A!37&SqTK}D+{4f$CcI?5(qU`qZ7I%dhvqlayHYn?zinyb(lj?%&)OUPVU&gzE7}~wq!yR z+v=T(adwf#`fk-R$2RPXx(Pxv0xOdmMF|qXZGsfraakl-ElXGYG;%JC1h;4YGI8w9-jqb7$}oyr zLxc`*{0!^I)qXCTdj#6Wb0TG_%A%rDB*Sq`z#3omo5u9qoBvCI3_Rh3P>ce6-n%2N z95$=Er2D2dMUBHnK}aAlx2L|;D6v6T>I_$XR`EgUMb&*UPV+`Tw^TrqH zRCjwz#iWxJVNW;Es8y{4j=nK&p8&V1XGPeirUb^RxC^~LPqE^2djKi?=^5asD66 zo@DSx>Dtj!PVZmnJbkT&12E>P@|92KQpyAKHQHX6n;bIMy#^TTRztoUd}M$D1{e?u zxbqDSf@DxyMEO+HI=8B$x$7kKbqL*-2KjRJM~dvsvm@X9 zI_pq+tiuuYb?14|V`)yDm4Y5of7NS%3xQ`g)v>-cYUhIAt44{k$Fe*pGQ3Cn!c2&qq89^RSsC z<;g4K_a$xE%=+KK zIc+|>2Rhtdmk`#51vJSA{1XlYw-&oRpsS?hfu5505wG*Zn+jcN9z&p9)lm1|0DBKK zn&OItlF{DMy7t9yudiAS`W8MME|vYBdck<$ndDR{hGi95V|j|`(x>IuPfi?>UOh%u zi3l$3|K+`aL%++y7(>mbR@U3UI5n5s7S&lcpcR^>NR^`$7UwMcL<+9;+gQ8NH-56_ z^ICY_!5=Y{;zf!i8BUYQqeqAT(XQs(NjR0EYAKteYMD8wopsfLpWnK{(ilIV2kSkW zsu4Ud0FETt+`NaM?HZ&!tWAiXS%QhMKG67OU#C@AU)>ZfP_#b?Fhda}L89u(UoQqa z!-MhFO;{Hm8?h$)CKv^<0mfmqE8hn^PQU`0QW!xINxra7pUgsKMq)jr?0@}NHvWwF zyEE0}`#;uu{qBzZoHLc~JKQ)TtPNH*&3C%f!h0@XEq0Ous^wj8iOvib6KUPtq*li} z)8VL@o)A%!q)CYqWSOK?A$!!}?Pr?KOUl1Stq_J4<>IujvAY&|%#zlHH$4A99s}o! z?HKWzFZudPMNWp}Ig$mMNbt;tn;JAU)$L@V^-mj>^U3yvz-02KOiJnr$po8tjqP^f@DTWc@5 zz3ZwB0x3w9rvkVQCc?nWdZ>~)`v0Sm0#_pgcvtVxKQi^=X-FKHEz2V3lXI0{J%a6k zwnvFZC|Y+m`SH5dJ75Rfszk9I^mD7AuUjq72RR;h5%BmXjUt=AwcB!+Wt6cuV}Jn$ z7+`<_1{h#~0T&IDpDV*%gL~YyCTw0zty!G_ zjh@p|Lb1sdnQnpXj^Kk}_q6Wld~n<2{c8n%X$srrV-!tRWldywjzIU_*CgVq-OK9j z)AzKz3^uF~Qf;5)Ds*~Oh2bck5=aT~U!55|BRJxAA_j-&)Z1EP$%nW8_9fg{o|PC@ zivn8Y^l;cNPha56bE{pY{QSprJ6qTNIWqlz*fo^{I8U!$R-@27??lQ_+Y!!ER|CBA zZRF<dD2G^B*5^jBGMv< zp7hyg4#j9@gRB$YBYnSq0M8-c7->EdG(lwu6)+kD_RL=M8%3A!}T(A#fpa^yS{Ohc{r3_o>p`e8Ur5K&Jw$ppy@ zJWvQ#5gyyL^M%Ldr5Oz{^~#U&ysU?8@=N(%yzeh)Shn(@-{DR;4YfAinp%`FI=-`G zJgYDYp|QNm$-}0vH%B@_|bI1A7p?ObYtf4u~x=$ML=@-vnFFueQOEWA_3p~Y&B0-=!w!KmSKZ?y_gjM_g z{*}i?Y!(#7bD|*76i+cM#d2sC2|9nQvf#29V?FTTFCS!HUoC>zECdsr9#!QeKv68A z(Cn#31Mdx%E#^{B2$FS0r=$PAKC4!^`W!6JXjbA#mElQ>N&fKWJF-?Uwn7EEMx1#` z)@enb?`d#X1AI>7rTk*x1h+axPGV$2ktm7*4XRiFqF&_z2{1nv!T7o<);}^H8)YkQ zsy`VXo?`(+l1^8l%h!D(P>Ox7!rUCU*X47$3tI^T=ML|6*qnl_S?cIGy^Kw+SRd)T z==M1;z!^9%`V7o50!>RY2SgPtOW0#=_G)SxJbzg~A+#v4C#oRNORFZBP2;S?Pp<#v z>zku5*fdoI6irDCPo5qgEDDQ9lJTbA+%y9$7(zgGGB z{r`PY&@IP6?B_t<&da<#oR&dQ-6q$EdO<)4U>~O;x9IT!}ocSb=`_>_w_%B%}!u7p5;lAlm)Tb?x9mpuC7)Y zT{TarxV_4%k7~g&a4g~&5LrTI2u74O6}&fQ z2H~93&I%{eKcLY}XV>|eq0)7n&S37A0sKMh0B)`q;l3#WymDmddnks`PydeAhwGZ= zq5rprlZ^R7ySUhzc2I0`XJ_oN0=&RW3`Ix`Nui^-S*f#|YJGd^rz5`|T|c@8TObuV z6;Tsef#(HMWRHzlHFoBfchXVyb#OWD>%~f^wE1*Ja9N?NDL(bpY?^MZ^vIFp#}o0Y zA4dqRDDu4j`U)j7=pY)eU&EM#of&6Z2fcsOg!Zkmtiq5aO^BK(QLKWpiuyv}S8nTg zXRbS}%WmB`fPO9dQdW>Ci3Nh7Xc|#v-(MCfC{O)3ZcLBxFwZ-zXKNKJ`JY8M38*Z^ zX(Gd@vWl7nhNOhfYO52X+aS!QyRFx~BD`{-H>~8u$fjqK1`5fjDx-oO8{Ylzo+VTU zoR#5_XcELCk-%}mv%zd@A~S-!|E=>^o+Dm{QDsJxRe@tviA+4HzTM%QuM$QfY0gIO zM%@BCO4py>9UZ`jn}%wNP4%g6zixV&u_}Obbivq5n4W-e}!WF;}OdS)n*aA3v+B?BiPn@a@wAIm}-ZpjWgvhtvZHXY>ZCtM~6| zD2(O=@C|x9yWn#3-9U%ecq5b#QhkB>b=Pa5i~_%dH#2V5)2*d5KhrjVp&C94vm>#u zf_=*3#q@pFmgKWbYu3ezX<8&ikrWA@B1oR2MN~|mC<^={lW@cI0qg&{5<8H8;k7A7 z5Ew#W36W+*f<<$fy+c=9v3pwctrO2?|1a+ss2fMPUJOF>k(MQPj>O z04Is6Kxza@qn&5Q4x44x>HGFKvTlfAH!IN{HHqOEk|YTY(C~PJQ=R>Ai?=G(KbZz6 ztG5yD1f|P9`#fzOe%qqvExJUvrfQ7D6O14bB+H@J)HibC9f_-~x2ER4(P9|P>u|)p zB#l=oiBK5bW8P0tal>3Yx^y z%jrA30D0c}(=#UqJ$NSCX JnxYV--ztut=9&I<0KfB8C=TCLU2iRz^7`2PJy4z$ zMKVK+wA1DFZw+*xsDL8>w!q*?MWB`Nly~VdF0m4_GOS)fv+jE9>&JFH*yPqWI-L_< z)IfPj+S%Fb^Iykw<^rE{A}u*3Pj0dQ0Bmcbhg0Z^9z=- z`j&@IH%`8=CjS6yoO#tcuI+RG9m*&0ijUq2oO2K;kP3Y+Wl*71+=*V zKDP&*@eW{37jIY}x$mZ}zdVKC&-9-c6a|_U1wjxP0bOXc$qW1%ie1WVw%)(zfmWMe zjB*?#AizAnK46kl=4pafRh1?P>g?IG{qI?olLRx%(TzYWlX027Z5?+1>K$DN){8cS zlZr^Gj7AWG$~nFH?g?*Yf7#-A{o6895P1==xAl0P1X)a5tY7WkRrl6kqIv9_s1W;7VwCa<~AbSpNhvs0b*<`I4Lu zhWa0M}aLkx}3u@Rip8t5%rMPD!-eApDx=vG>&-7>!S8nui39M#ny zOA56LFRvW=y)fH9LcffBU@s#B;**=LF{x@UV z2SSYzx-ojaK2z%FP;Rags;%GgzW~nE%UMfbP9nafCYn9-B`;w4MfESaP1IhBXrwkUZrqEH21)`B05)&PZIcA6a`1QF8a!!Y`Nv znk7Y+q)A>B1U{@2dsm88@{g@wY<>Pr*Qc?2KZ3{#Bu`NULvoZ5=DhMBK7f~ESAF}e zi}v)&IQ2^{{9@m~3G8=O3S^O%2sxW<&|&kdcV#38ak;4sXW>dri~Fs&FU;%vz``5D zc_LJUrUd6;FFy!)y)JiQj>nzrkqZ0@ue;czfF7=V&<$u(alS9xC3_^#r0Hvg{eBw+ zV)8UOSs?q`qn#IG<-AaIRXigoCIi9R-~|V1V_ngAqk^&c(;6NowKtm$sd9% z<%wY|3kt_Oy>P)#^CV9Zyu_1$JNsbfuk(BUn_3xDtLbx-$|T*UH;cuspII00UAt&* zR#dmQvLrLA!ceRt3OKj7tFT!dwc3W-I+?GJe*94(2~}bgK~*IMO?x3W?eDDh-}F2? zdszd#g&QuHosy~$lqPYK!fVqX^u4j_;5@87F0qu;&BgL-9RJGsn04^|F6Et5wZdnU zz_~##FeISDwwKmojRwc9lb-nd=-W%88-GMflm%Ml zDN?<@cXGVt`13cxR8-hcE=(0=bb{o`9wmzPR; z!g{)Rz`CdT-~gm7@48AyiKWQkKmmS?!yA9;5Us$Ro{T*fsWET^*wvC=7PKF7_WxV!;P z7cZ86x6Unm@A&UWVZ*PpL@Vg)uZ3+8}&2=&1s^l$O6w&4E{95^T@9M^QAm| zgcm3OTF0_)ez|TMJe}DfLiienCj%#?y5>t>Z)fkMLZz1vcqE_Ob7qk6a_w)XRvul( z6>=;{j`RQIQhuKt1Dnb@-(*A5T-6Xfz>-1+*cq$2<*XA=9)VI|Zn#1ke9lx-2u-9o zUQspT>6s~=7xer$6(^F9>Jr7J=lrHFnGWmpI^7VGqv!p^pON)v{a%kLLkRre94nJe zwx2fEIn7jq%EzJ_5tI*tae94j56E^aQvMKs-Kc&)-S$E2?-G8gAXn8xE|O=wRx$o> zewlD%%W_UnjI-Y|DyTlHj%QrvQnl3u)~~#vw0xL+k@(0MuNSwSuXH{(`2OPdp9wK_ zmR1`###P0_mCip`8{hnWf%#a>S2v%;TfZ?CR6auvT^dL;UEO?bx<>e_teY~^DyW~} z`ZDJNzsJ=Nu5;mgKO)~{Ap8-pzB@EVyX2HIT{J!I0^0?qGp3B`lyzL9`Qjkw5{<92 z)~mV6fQu=Y(Q(zXg5+bXQm%t8l^jkeCpm7wl?^ix457rJpJ;pZugqO9ME3*$G)`t% zPT?6U?1~VNO*+js^~2M|b+zHufuPfTK?mbRiDhM$kYtUCcol6-+xW1_6q%0YK;M4; zzsj~D)+w{5F;NdZ@*2SqibnAoC!q%(m-h(0MT64eO_XkHx3fj9Cx=Bp`Xi780R)~> z1qyBbp?6y_NoUx=L%et1f6-k$kSYZPh2$tDEE+&pY^gJCOIDx!xOOkNqZQ$ZmH(Kv zA{F_HJ;2FoqTd$*$1tkawD+1fik56Tf7Z&C7>Z5E^O;B?X2!rcC@H+%Hu$}Tt9snn z5K%2cH|0r^k{DSLX@9^W2?M`xm4SK$YG1a)twGIF!#3Ia@v^s$M!OSEaimPKG)?jvi{1%uh+rEwr#d$4 zfxTO1R%siaQ|P&8mSLRP{!<6Q6F7a)!<#LsD)9KDJ$yK_=Bi+awM`P@t!YEsffcQP zQJ2G-9xOSy=E+qw5AgRtPlrKE?VIyXWOE+suEPTFE>W;?pz zjh9wUj_#X}U}%QcXq9ATs!Fq;oxvVi(@Htcn%TzJpYr(dl~8gn3O~CS5*a`uRGK9u zk^>GS@<-00no6TC;s*0z7dfXEAMh2mZqAIcs z$C51WM#hWSNu;%H;jY%nXTOQ=gNRURpa34IoIs#Hh-xlPj&s5yY&SmRd1DO|bHb!p z(V3m!y};%32TlM5E}tGSUh??7=!Cf!8TS@!{*+B=sy0~E9_ulcA_Q3$RF)Aq2KAU~ zPb^*EnN5uC+k+|HR!oVixC#o%QVb~qULnz5>w!(0vlWf11U~QC&=jhVA!pXXc;K1j z6aKSDPP+a(6S<<(0B}Ovin9vF~o0O}uS=f97b>m-x_XJSA`x#|Z>O zkyMz+N1Ffe(GiHaa@Mtd5$AQg5wC5=EFR(+?yGTwL00<(Z6K= zRy!-4wMrxv#XB_@$XEM-FnK`VYr@FD8*B|$RCE75GAh&3ln5A>BWXsG(WEP33Vnm^ z_LOA}ocp0~*5H71K98gTr-u?nKoK0^8A8gDR}yzTS7UEg{Nk^G;ka`b#*+W9eb3<49)!#|=}gT3@e7l#wzd!2SFHaGDqQev zCeF#HB#9i!3NlBsJHDyi^y9Z5HZQr#Ch|0s0R|XgKrG?e-S*RSZ~xM9YrSZuGEyQr zUI08V(c(D+Zq;uZ)4@LQKC*{m6yWo_v7VZ5vi0n<=$<;C){ADC6-ZH|7(pczRXbOX z??=r3w8rYiR+#L@ZqadgzI|-p31hFb9Ev)rGYqB5Briw=6BeJR9;Rs%Oxd=f?@lak z{8$sb>92s~S!bTx<@NP&6-xO%{5r`%AgSKg8*Z2yYy=!=j*OwE-#^aiG}*R&&n;;= zs$+w21uWRJrz#csoB{_3N#aPEkQwymRTq6XuFEttCZDI8FZ`x;wOD~_6`B%Rf}?~m zWvS07oD$jJ)~wG*DR-=c>e8$T>Jp_;qW@MGDT^#U>-$5mynWt^x#`Q4h&b5R73@>f zg6K`C)iYk&Y|61c)abT_ZxvwmYnCD@LSzX+;7M8t3w%N7UiWXnUf3OIOP}L?wcWm` z+$_=xBa;HdD`J>4>48|MgM)4V`=#AD;S+c(#Ac456S7W4%FiDNG~n?&+xGU?K7L$0 z^6flZD(aVF1L9nI_{8{ZRH&nTG#v=rkjKlBz=diBpEAO)vWD1JPW^r1>NLD#eUhOm zhG7Xp;6+9ZbA9sft$|;Av3)hvcFVy-lLj@5;7(WXDs@t!Q}JK)a~65rMZn{mx?^?DmdRA5ft@+Y?!y2@(f*nPVx5**!LC z@4N@-a-NfO9L3TUD(m0Tf6O)EtWt{Y<~Gw$y}C2Hrp0rlNJ@%K zkXqOwfc1kSYg&peb=ORxZ)SAWSkVYsW=Vk(!;G{CYt+DqQ(#N1wd`YWa#Zi+L2jqe zo#^N4R*Q(kb8^{oN7-(xGAS-|u*CYa6i9*Mv{>fgt-V4q?*GyP?z@o<%V$m-8D|7% zv#G$=DebzoXQ#y%!i4Xg72_-cEz%;-Xe40Oe>-RJ!Dk5)5FA_a9B;d8?HvG2 zs~OHHK|+|7G)9pC2^baKgPhkm-mE6tnzUcRRsXJW_^Kgfyt_T!B_+=(2!bSuyezAn z(4xbRTb>^`#fo~0t%|8A`n>8zI=q@)LFwS*aNKUYdrHxLhaPBv=t99GPAYPa1CzW? zPL^0%BmmG8u#dMdg?)9l(f_tO8DJ3|KRXBnc83DXUFJ1jskegD6{^^2-Wh=v7 zwom%pIlI%;Xv!ETGMYrnq^hwBI{ggB;BbDp&sOuZDfgYB>W1f3GW~FjbArczyrJ-% zZxJsl(Due1i2Ht9Jy)-e2WhN_X%&~P|e*c&c!1?Gp5oTv=YjG**b@Ii zmbtdu&B}tCi=v+L7YUUYHH{%;fksdHzdA0^12^6tudJDG6CF2pzv)QS^JoGus4Pos zl%}b;^Jw+3SuC)bZn^KvNAHF+#jK=AGlk46jKBya2PAD_^E-2AZ)=+5EN6~a<2;Jq zAgO%v+Z7|7%9n{GxX`eX%`Q?oVSB<*ZP0V{9lGItMpEQ*&DZgRrOa>Q1(vtG_5&tb zN+d<{xqvE_LY88Ah={q_5-xFqp+*cEwf&>M(a}jCHFIRd;Pc8c+6kr8FtP(<^|8p- zHuwFWn?AuCf(eEtXr7`3ffi^!%%^v;{^J-{`FPZ}YRNLH8XHdxO^HznpopZ*$SfK| z(-xzVN(;6M5@MCg={wxo{ZZTL)#}RwQn5#(1&(KBS&}qKW1HWdjfQJhtNH(du;LQ*voCy z!afUE|BF3D!E*#j^Q_1UEYH$myXQ9j!|9kq6w7Tt^jiPP>18n8brBX^G{ppNtM&Bx ziUtACc;FfADgf?cUuUn^m0ReK6FSm1vuBMiSqW4*vHNp;WNbkBtlrF~Cv2;JJF@Xl zcZ`_RfYbmB7>1|9g66KdCon^9xHEts%EWQ;3R|TI+HKtRe&ZMmHr>{$YMP<|RRAi% zx9tOdjka8ySta`OnN`ZxP^As7g1(PiuC1^&xZgAW=^vvm)) zuX%rdNp@+wG|3w00h;c*X1G=fUy}C8@ z1L>x~$h%7)fDLr#iLZ8ue0*(pVracmb#GH$bY^I#`xv-U_Z-tx_vE^HVxS+7)eCUb zCvOKb9F-k862{6u0|oy-{rtmjy?o2{^l#O}8LZcf`MUnyPddF+_j}>c%bWCCX!_Ud z&-G?9JyJhVfoB#5@||wjBd%#R%xv0iYnk`;-@e=6+`2B}+{&{wr;#)*sJyW8B-6k9 z(zc1v%|^2B`*nGf$XIJ}aq5(f71Vodzg|D~t>2c{2(KdyBF0N3A@Leu01%rU`QL`$ z>_R4-8BR1eN5v2(206P)I2TMTRO@Jp6u7)-v*|-yqLB7utEOq;3X-Zg$?!mS&tapA z?X!9R?zE=EE;y;pHAS7&1X#e#Ol(2iYe!=gaecEWRPS0xK|t z$kH6a1Hhmi)&iUMVcX=R9cniED}v&auX>ym=PZzXz~hqgU6X+`Px3a}G~%h?IErob zVKKxKijQD)W{c^#Ox+90+V5a*4B@_KJTIs5+~U&`m@ogY3v#VOnS zw`DwQ+8@O+Da-H-BTzg+Q5r{NZ?HHoBH1_**C z$PAA<3WX4VRD=I)|4eN8*_I`EN1-f1P&_3Pv`DZl6&6#hlOBxs2kabq#>TFFP@8fa zoVANWbq7jx%5HTMTkFd|{Bt|%P<0ow;Ct=@2UAJ8&f1O+G`-oWZIp0oJWU8ZEeI4x z5Il`my>mg}*XSm=EX?*L|9;UyeX3zJ^#F*p#!H&4X#xlRxYwGnc`>zSbwXkc4Z$2` zXqrRJLWy=d#XCQ7cCGN*25AT)Bl_)I3N5Ju8l@LTV@}7kXrleS%EM|N^hVEtP*}hy z1O-Tzpa1DVq0_3g^IGiKZL#_+#omS3xUOpx?C9VblA%}yP=ZXMw$?1n1Y^Pu`gHr8 z4MQeBwF8@-#tR}PlMKrWB$}NGF_Fc}F5NzQe4SoTe+HRlctpvj%8E)6G$F7&F5=_E zmQaks!_FUu~Ie84*>Px1hDkk9Rra>3`t3CDw@ z4)sh7#qkeStG(@ve&5%rQX5)*k|LBt;GAAX%9lLnf?s477d_ClcmG5fZRHESW+R~d zMhrgCtNan&j?$$;eBVKsfsso%W`oab$7vv$g}IQFf@w79&3s^->90cQMhIOE(@q=o znffba)c*qwZ%6}tk{Vi06rAPMuJJPY=Ac$!Z1Wmfuw0!WGNtVWR-jVZe#6GD;ZsUSH+CL(n5L}-@{L{ z)$H3g&Tm+F93RA;=4giF{4VHuiXg)@(WS2j521%lP}e32S&{JF!SH1mHxT7R9&keP9$(^X(+a%5bPY zsTbx!OIgd3W&dE^;BVGQBzo!resx)6+-$U9XE;be`x{t-HdYT z;Tds@$!7MSK413T=U+#OI1;>bKnsjau#BXz0L(l)^3AWa4y9+rC`{G`R;F^yS+u!* zeUh`WJrqRM4sZpOk_v&M&U47G`DYmQe*Bu&nN3Fk(ild8C9ZKfud!u z(a4E!wMj@$99T|08gER!RZv{vwl#`-aDq#4had^=?(P~Kf_sq0-QC^YoyOfgK%jAV z_x{=YoO|kat?GxaepprOt5su;IfOC0!S0JiT8;c@?;==d^%o090n^zEsOc zJf8~W6=duq$e$3NpP6N3w4+vnc`q35{M=cxhl9&?tT~BZrggvh6|T_U(M)k8fS}X@Xi!Bb#18)pfT{C*y=eAd+ z|6xo&MYMXJN;*Do>3AU0w$VTL{!TTAZp4D+yT%8zmnOxkE11?d0eC$5zC@lq3b56o zDz&TvgW19X=dITSwy6+V|MDFwgaj_im*2a0(lks(eA-#chSAJp(D)yf#d(hFJWFb!+sErLw*(V0_mhqLCd%mm^)c+V+j^R!T z|MQ6O9B_ok42;@qj!<25dp;= zZt9_*&0>m_aAS`c)(AY5uAo`p&r-I1|GHOebOu8s9)C%*I%wa3T0DR|SfoUBYTl8C zoy|-qBlwyYfHopnhh$KW223vDbvKii!Q>Fv;F9=1E?=RVMCN;_5du^LMf6%84vOuM z{lDo@k9RA;m{X+o@k%Ioj|>Vbo@JaHt(=O4iBY?;{Vanjm#fTo=(CpCRCy)$Wt0<` z?&GePrMi}(0(x|83O4M4cz0Ae1AL<4En#BOKd32zp=-CpD**auyt_7k?zYd6EeIMK zRbRzlnCKAAV=G>pxlHb+i?K?Os#0bJf!x!N<4Cv+`dzJKx@Uwy@9RypL}OWT2QEz^ zUj5C!#0a6533xsgprLQR?>Y5G9Dcw+r%H>v{ojo>F7YIM`jQ#A&YW4S z?;k8oTR>fX1)nCK3&_&7T+8)*nG;X@85%i^Oa*7J4q2I;#CZ<`90*}6dfnQ&=~O_@ zhb>e`%LjBL{cB-VWwS@mE<-Al&mFI~31sJw)9z|mK>rxm=&rI+fpF>0@6;9d3;b`G zd`b`$aTNi7a1B*Bc|y=9Iz6f!=*^0!sHZNvQ!_bu4td_k;MgS)xdGXr;~P{Mnfj4a z0yMaNX>q@t)~XqQ3VAk5)0V128rZ~(2qH4D8W_y!gQ}f^336v8z2klI2I!dnW)m;D zm5T@W2-3Z{$4xf`?Z~}zSHGSPTV~)${e!)+tx>9)&DZWfJdBn(m~{*rPp?H+30H6t=X+7O zNCr`(ci@PuNF324Wc$yS;l2uDyyymP2F_km+%X49WH;LhA59^2!{pGpN*FH4BBSs?Nk}Amyi2whoWDrqhHe>OFZk01-)(`TlCeMbfH*}XFoaQpDt>4w3@!Q=mPRGALzb3a<0qhFXR zLQiWt2x-?Y!_!T*V!k5Yjb1CgZ=DHMsbw1pK3_lVV`+#bSFrWTQjR(MVTos{$@kD< z7y1drQmMyMT;Q%y+^OunizDzaVEgPWq7x9m7NYA2S(?ss)j|ka;s@Gc2TGvrtwag0 z2pye#x4bI>3_k;)Cv(leNvPEAq2KEQUfiYW+FmdU<~QveA6EdchuPwCQH`OGbOp;)xGFs zq~J14@KOq-W%jg1#HOxcJGgx*f4ohrz2?S#dA$$M)bdK{jc^6|62vz#M4Z8^eO%R z;iGHi&@-9j&Hk5n^GD0glwY>7dD{uhDu*}uQD z8CnUdK8L;{EsMvs3JX(GYQ$N^qnjmz*a7KW59anrlXbS`p+oeq(21IxZYusLsTp6V z9XR^9Za1*}O8-j_ecWi}J%wHJSO|jOH^yA2sG)1fy6>$+sm&I5Yu#v1m;PzxJ5`Y3 zox?A}>x;tB7y)HjOu#?rcH4fdOusD^1w0|2k1)u;8B{8tqO^4!Bx2%NH+MTcxGt9S zz41hwN@?sN#C+8XF(2sv3Vu^D`qYE;nq61<>`S@>+pQ=Tr6c=Y=eK4pv1!;C84&m+ z%k-T_4d3$T9Oh^541DD;@nGF3W>&H=(#;atpS~;2-Cd`!&#&gh1M81wOSK6s z%Dkzm-pyn$`#WqWnP266YSg>AxThc3@_vnI->7erq(t%oL-fqv{3}=X*#sJQ812W! z6Jp13_-bh71of&u@Kl6Bgb$evgduvyH?yBM(d!7(+Tk|N&l_=wv*hsqUv&HbIl`!5 zT}a9P&gowfb2nNgzCWb`R%^i|JV=?90D=>=1jhiVRQovcec7B3ae>rMrEx&*^V9i1 zifH-OmsD5;;4CL4fe#IaH0gZccaDgOj+P-~)hw4P;P*Tizek;&L)Tla1`DxYj0to`W&gb<66hNo>oG&Tx<$Sh8(uor{DrT?#YP(- zWs*S)75U$#YO7{G61AHnE^fu9d1t5se|v|{zE~7d%v9Uu-w~!4)7NdES)1R14mAmU zInMPtmLkU5z1En02`tNJ0{ey?^1t{gxAXV^iHWjY0kCz*|EdjNnjp8>K6LT3v>Yuk zSjS<7A8@6_d4!Wwa*Rr!t2o6E-Q~RZVz*qMfVU|!<59jjq4d=y^+v*kBITFpL7?@* zTzJkfyW48EPOb6qylb?HF&W{_m$&&#LY|=a)LuA>k{ZTq8ntRLnaPUQmVu}weNnef zqrkG6h8~R<4W*O}V}RdPIGa^{0h&~n!KK>r(Bp54@JQJj#o9)l)FVe7`VW^6wF;f> z9ipYBq4EJWBB?WDN=dxW%iX>|4;}nKH6v@Q7C^3o@n37UhWDTD5IYYT9tuH>aCn%$ zbYvv|k>WdOEI8R5y{*pO!skxO87qt6>c7QVGA-X3IQTjA^#t_C+lee?&s)4gh^+;H zI-*?<+mI2?0kZe|+??TLR4Q<6N@35rsC;Rf<<@X&BJ(>b`=o2#x+GpLkq~Cw{;B&( zBC<`?(rmI}^Ow7rXiD}aoV}mM;^ZS2#;|)|s|C?~m1HIF{NurtbFXtIKwJL!@FpwJ zWX4FCVLt>8xoQrmEeL30YzsL}$HX7G0KGrVs1P$;WI_k#a!N(ehM9^Na0OGDLQM0k z>VGbSBy1EHX&u0cZA8)reNr)~JO-7>_3;=wJ6SgcGx(}ikb3$6fkBd&%>a#jkZ9B? z$LJvZ-Lpf@rkn<~U+n@I6zSc%g4tmjSWl@G`Y{j}p%o7fH5rG6?pzcz2t0)2=wdHIY8KNL<4Rx?xM*wdE>+vN`#upg&$IlKU>|%Rd)@oK(3XwzeTaLi? z`dA)uUF<@!D-0;dnHr%2p?BKd+n2Ko?L%LmpQ(dpyzvA;F-^SQGf~f^C7CJ;x6P|} z(cKeKhl)+UK<<91Z^CrkecF4F;tB9~iErB!9g3gWSux?ZZw>x*@BKqaVV?f|6mY+4 z%&H<%R*_G?E>P4@nqCM91(G6XqDzDvkMtkHG8_Q3V*C*{BaO0{rUHrtvFJbsC?TM! zsJT8%-opKs!DoFdaL?ky?nE(P{ne7jr48}JFgLRR{jOJ}N*sicc1hLer_J^i&UW^H zFMZ#pRhPw_Ss(b*aza;@RBWjf8i*Ol9LYzg5o3x#l1A{4lkL>lMgQfv#VZQ57pQ3T z>}-A#E0srYC5TU{m3IP~NA`a_fwP(!i?AqkYW@fGFej6#>w=w(*Gg2Dh8v|c?ZsxR z*j{(M;EyV9Daq>aF2S8?r56yd$nkjCuU8(Gs$&HHulZ&{7ZJ1P7#-p$9grdx-A9-v$Okd)Ak0Hd#;joPF#8f+f2; z9bdxPLvf$7m=r->{)EI=uMc6CJQ3Vn=nSJa9XWPD1;OmX^-0zeXpT%coF=j8;+s*IO8w9lfe%X8reol z3TMA$DYx~3K;AE(7R#s~?dmKl`&^MJVN7c?4_Q15A!@GTg49&d{_$d8%M#=YMvCmZ zJFT;(9M*EU&z{Y555@Sg^PMo`taSDfWQ5SfDTi6fLoBSE(hQ&9OLN8B1ksIqc*y#> zEG%m6(_77P;(bfk7FOR^$H=>XMeSLZM4#RFVVdAgxW(Z7UV z(YyF){HTZln+$$l?lz=1`aOheBddals0zGXH$4<}fDp;~EiHK!${ZC0o%zp{z1a8r z9)4qCB8NU!teu=c3B+hoV~u7T@RCt<-i?iZ=F+7+<*c6qw`WDdyV2C)PJZ2!WzW+y zixn!C_Wm>TX(!)H711@E`J7aw&j!Bs?ESkQ_URAT6xH`HZAxZ^2_|xDEwJaUB6IjI ze6R`)kk8UL0gbXiUHp~ezsa>wGgUOX?-wgeOVgz4_>JAS+;6uF{^5Vzvi+ib6-MGv zSwzf64pMl(+6n41{KG~sQd43pjPD{xt{^XH@ncVRSdw)=d$a8?t^J3r328h+q#a>_ zU_hp2kf)U)rpP@#5w+xpA46~RQxMQ0dNA(Rh@o%>h{Y$AxlSlmw1BWnDd`c+zTZA{<=U=iK~kh4~vExiaT&{CE+J52Q73+Glt)JSV^ZARvOKFnh2|SO};&qniK%@Bz z`VWAgS;6q1*^r8m6_m2heDXI*?2ze@2^b9fAIvdBj0Jt|0kHDfwGJdT|6Kk--QF{u1|}47;_NA zYY1NPIXu5FsP5$m$)glcf*fGL*q<60`wu$h$sI)_2mgjhXy|kM9?fyDnb0}VOFw{W_D?v>~9z|Ub#RWk&tN=VIO1MbV_{a-gY^J{077^hZVri z?=;!mW!9R)AQC)?1-wMN=jBZdAoD03a3$bqlZ=wN?wwy&v$iS_isDRr&s+I+3Z zs`h{VKJ%y%XS7aQYSfrkA29TLhORL-k-AJ3~R&d%W#05gr8I`~*ICRhMYpH*VBTzlbKoqGo=| zi&!q0eyc$FbMTeB!lq#;uNp_r!r&JV1)P3+l1s6Z-ONNaKXZ~oPwzui>W@qwo{#VE z?AI~P+S?Kqv4&HQ&&|XkSg^v731w{mi5Lj1ZrXnvJx#)%DI-=`G1J}W~2r@hfsR%5bKkM<1 z$9=c6dn)5~5x#;3VUI+=bzl!8HS0RLUj$}YGWgH{+`5#o8HIQvWUvYO{dYZr5Ac|F z$zi+xn=$se+0KuJl3T@71KWH`y;oTa&|ST_ZRn|N7b72o;nP-7Dz*EgZa3>ZyFMj` z9DmDqiHd3}?++J;q_eTC8kq zB@?zN?=P5d#V=<3V~V5RM@27}5L!)zZR07pdSyA#@yI7*mXI>9j7D%5Vp%Odimz>< z6n|b%`zDO+=4{bV67MCNY62&Ij)AoD(8~g8aQyp?X>_JPHAlS~FI@byv$kN=zlRJf z2v6`Ir`JMUnYPgjqDRG3P;3`%#Z}cDi^lISl|OCddNVDoG>W7nFgydu$LeH{xDT|r zQJhU#9DnQ0x9h+VhJcI%B#p2|wkZ7*C`k^8? z`cYd7k4||#wS>~+6!o2Hs38qetD1IDcE+O~`mHhzCC(|%DKj#Z7ttzrMcin!wNRT7 z@ozr8v}!WO<7th#O7jip%{1pDW7`|4lh2Ks8(Qz#;3Npghkqh&MJyR1YEBjs2 zgO)2w@awMoi~Gs3mne8-!-PB8Xwu)zpYu`dVx&Ett7gL8w2tD{jR<|{7?mP{W2laZ zMTuO#u)w@)Q{9I(%Jw@BPYKxR>|xTIE4Vh&I1@IZw-H5noWOF%y)E_A2ea3UGBF`< zNaw<5OZrYFxrm$_6u>xuUEYu(B2^)k(})7zcy5O88GR~dtI`+A!h?n6M!-DvQyl9Q z7|AmG!Wa7wZPnClOcB;>cA0%IB29z$iL;UsuT z#qe8zY803z)!A!snJSX_)xA3RQDlO8yYbIhpJ3_4I;Y{ypGN&%p`UqsvLs`d+n5_@ z*$qr|FrkM(`sy2f`dmYKMUy>5VGJ{hM!EDig`b&WiE=ucBfR$>WCG)M!2XOZBukF| z0!X}8#{2s%5eA!VBB&NUSgGsN6cIrW$ zLB}0X3%mIczWF|Rh;~nmLWzHxPfspMEio#Z&gPF_@4bP`Akth6`ioh8`y<-}+8d%j zjap?UmR~%i50U?7ao($DG2{=I>}H~W2u~4ZyQnr-**kw!O|0a`9{u*wCG+P-jDN;R zKQ7)h`|7$e4B;y&m=E1u z?s4y(X*5l%En6<0!uldMvYFCN^6ZJ_Hdkd`B&X6OgIwiMG1u$$E)Z$-b1~Za2mP2` z!tN^WDa&*wi5WIL_F=#C0Se7GYbe<|s9J06iV^@0d+S(?SKdTg$-F3ryQ?Ha(ZH$}jKXvnJ=7dWUAvlox#hvpG5h}jWW<8rtd*8cCn zV{bZ@!K~Y9bI78N^@eoFf-WLXJpLE0*&rx%$siQq|7*`1o>L)=vBiGj`J|47>Z#0t zxD6})AEzI9;SzW6U*QBrcbL57(Xcmvp<^@Cw_O|^&X#X*6D=-9v_OSCNJJ|O0`^FzgO@mLMLB5(lJ_v+BU91 zHttLMK$IIdYq3ppJcC3lWegl~o|Z=KTTOt1LU3nH%1>xBP{HG~oUW;!hL$O+y5g*J zC;U{1FNl5|IgnXp%M(oInZafqaD)_~AVG!Ms-r7sDV4*FXb= z4cJCX_RJEVu&GZo?)yMl=(1RH5q7e&Y0o!-`TXXei3Ph3IoStwoRzmF`-G#7EqhXY z3|CcUZ1gl(4VDv`D<(6mimD*iGfy0-4dz6rB+f7$-jk$%@9Q248>X3PK2cV)Y9x(7{=&3QVQf>HUgZnSJBG33=9>HYV zLdry^YW^#Sj*3nd{}(X?yQ+$q|JO7^u2twamk00kM$C2M$Cynv#(5Yn8jK3INBXQ9 z68}y!%_pQn>cgORa+J!qXa zizu~nv(g$E5Fs>NIyfA6F)oc<_y-0f*Y&TcZVC07Qwdc{S(IfBQVmDuw_E22=v|Vy zHttouoEz^kITw3N6CpwulMzQMX{sfF+xx%!VjUM<2g$^rrppnBu*S6lB|ZS6#Tf4 z*9lAJQPXNHl0d{PcWfea&EPPZ4uVa@+`1o*Clq-^tfIiA5U>~gZdCiC3i`TQfeXnK zf3YivoIiJnj!MDld0yM=o#r~XSPM_Ya-EE;sD;`0JJP@zhe~CKGg(DF*sTL8;zn8g zEYn?;wSCgn+XZ~uBJaLspY~iR@XSJWOVp>j9j{r@Lo{ki_po`z7wWk_dTFeIwML>FyY)J5B*c=^`7HZNT&fa!$w4h4kbOys8fwS z=~PM?Nrjs7=cwnx>28Z2TZs;pHln|bkw9->Lbl49?-=jZ>q3<<_NP!(*EimXtSuK$ zUa#)nUEbElzi4t=O0Iq9A6R@>Yw*@*V#keQW!LU-!+74?dfm&{hAM7g@TR}c{+$#j zrj>SeVY<=~+H8ao`hswLM*xA&)88mmiRKEKO9e1AULy4Uj^3Fu_T!Ci%a z-qS%{Tx1obWZJ5x8RmG}+iHtwO0JY+azUswH?W~kvx6D3UoEyh)6Jz4s610>=)Rwk zj5=)Ol5F?xUlmZ;YGgllo(EL%@pBe1%a2!)C$cG8If1Cr1f41$Q)P~aD+HJ=J<^jX zj_GYyCD2`EhAc+=tZ9WP8uUNO#%mn%$2nFp1|OS{u=1#6RJ~fBcE$V>bmth<1?7W)aeF7O8l8 zV!nvGx_jn0*=b7nELtdf!7wBwT7(EB@&E+dJX$1rqyiNf>^`J0BVkBo^a*hAdb=L= z>TjpiK_Ztptq}JhK_cbdF;SABJw<#?2qY^ zl}(R#erM~bFjCHu2QNQ(pM-j!FwcciGmH&}IyU(o%m{}v1opqwf&HcVX%A*xnk@gi z3))mOLUn}@5U)YcV(27H0s%qcJM8LHE2a3|mBj~BQ`OiJ{WNQcM9k%`F$A8(lOEpl z#m&T;M)?~BzVgZ`dI6L#{FE<5H3pu~cCq$;UG`$j6@RQ+GlfHa44HTcc+!+#OPkg8 zrYOEd(PDAX+_ce>Bk-~MBz)dO_k;K_hrsO}g$uTSH(iPadOU^};=m%57g`rDP)@qC zOY6FC?(kRe8N!5Qoykru8^MzKMcH7${;nzB24P=-&Q|Gx1wo*&KnJ81aswxlnz13kUI9u~cN1wpwCNF!2E@ya=al7N=S z)ye9fA~RaEfw}cXHCBg2KgesLG8rB<6s>%BVKpWB%K#h9A46%eD^Imy*RS(DC?P$k z{jSWC5bP!u4GlARmnG~0FM&8jqY=w9inqd)`q9LdgtrN`__p=^HI zCKmcPrX1hGsg*e;%}FJ6o^l&XsbxTqV|dBd?nzY|1I%Qo{|hXPc`cX zkj`S78Zyl!iYQS^X?2j=Q?jeA>l~Ej&Q~pC`0&gKU5ZO3U!r>AXKw}svYZ$!WgRi}uV#o)IZqSnwszANQ7+wQI^V<_F1DrMaK*3(SosF`g}=|pf0+q{ zPLTtco(Zs@NG0MMx23Ay{dHW<8F{9R4Rq$YNI2t8K(iYx{W5_2E;$UHY>)?V;tcs= zpI>jHy(qA_eU0NVUEo1&z}0p@5kZ-NO!rs9n&%13zPnqGt6NQi90t}E)!IoVW}ai7 zz@sT-lR#?VxK{?PSE5cKw+q>Xkwr@eB(eEwu01YA$1C?(R?gNA(m~?sTK#lgQz#O|T|T~ya`So- z0srxh9irPgcX%@IiCjO#So>TV4ySLYT=Xo%v*&rjJSaNBjI%>Aqw_Tn{Z~cgrVgh8 z=(R9FSpP?`8in#!b8lc}JmRs7L#OEz^d*JBqy)Tj^*RbEje(emACv$iF~}&d2yW%6 zR)N~GSC^6BHlJ9V3OrcJKs$IZg1B(RdavjI@$aXj!AAlUoe|5=`}-}|n&*Z3>eK;i zaouLq$ir!gqIak^l3B9FZ*nBU7;eB(l9pUuMh6*vBIdd6aeT6$o^GeY~h zD#Kg8N6*#A;xeOd^lCpOJyCr@oNY(3xB@cBe^AE_7fO%%x@SLNydM`O`#Md2w{g?L z*1gSaeZcNRL3nImR*-6{wUu0f`XI(3Fa7U)lOGS&P|YPLk{+A8oC0s0$bQZRgTy5T z9|=(~PI1m3d5#?Td*8p$yZiN&=!TV&?tECn;NASavU7O1U3Q z-Ontu3NQVsNjyYwYg`52vVH|EyLmFbUHZ~%BbWTRKY-hdet2@D&q>Wqd{ z={GYHj7Wzfd;xJ>=RZ4)%j%|sB{o*{g3Z*LO%Z<*u4lnLisc|W>xdaO={xXdCv2Uv zc={x8CEa45b9dFu9h4dp-0Gd^W!s)QKnvl@4lygjr}e2fb|tyc`-qFw_tS6&^+|u$ zwwSm(dM0d8I2P2ozD|@R&(J(5wUv@MDZ>-H%nJYaEA+1m*%A{P$En~|IK|f1XI6J7 zN;-8l z;0bseUud1zMtjRPPEPr4;3WlEv5rw0%pc#;x_cJJY^x;^$C^^$kY2Q;x2C5e9&dJh zT{3ecjk9%FX8sy?a`@Q-v-iy;Zf4hE|1QzH-L9VD&_$!bLJkDp6<**U*Ka}e3Yhat z{3s?6Ks->7o^faOI}miMqJK`)1v9oRkQ5VN_d|`3^|99SNX0vwj!?r%QE8- zIAT`aW0f2~VXR}#f|??DOz0CpB}*^dJDzdJND_ox3#8mQHj>gXF^5H;~CevL4@eQ@={f+ASVGy`*&3;P~yb9o6`{hKZx{<=-;-Y^-co zKzmEqWCBS}l$^3_3m*MZR|Gj2ioeXjn)T@yvjXuWc80MJwHU}$Ft%2b^RPp*X!&!p z*+wLN4>@3BKUHYQ$~BURv$yPy29I0Vi??|)Uh|wMh8j{ztBU?7T`>FdvmKc(!F#=9 z1Wu(XALKeC!SyW^(uIO@#f+K(^Yi2TCA6o7>OI!y!Anw3Vc(sa{rbmdti3~P>y+iOtAJodRYEc5?!mj3TlZCm%w z(8FUS=D%CJAGy2c4mUHxvygcAt|EB^vtS_3pWaHjuPFx=TcsGWMj2y76m4C_P9-tK z@2Z*TF4k*3hXOA*dmL#a9KL_HM+td~k!kx|7@J<&Jo*2`KhcGw(9hxffNb+89+qs} zNsE&><0oRCULE+xEnd5^-JSK?GFOny&OE2?KXQuQ`U1=vkiy_XNRl$GAXjJ0Ad7+Kgix4D%kdq(DlSmQvJB2)n#tW55fVp$n;~rc{ zy|&1<$G;GP_QPMBfO;?Otbv^))|s3N32!RP@NIfx#M(C-HA>^N&9eBhRl~em<@N1E zyu4Z5$UHeDF%4yk$)0H=@o!zaHA@}iL>))q7D})Z$Ct-`zGwqWyD`84A!vwr=)=0d zlR1G}Do2D@L57hjJYW*@rhgq(g9V>aY$`@_0!YI-{wRa2S)u3J_hLR#zz?d@H6(^P z>*8b$B1z?#yV!;ek)-k;M%1?JusBL;thBX4q_=tAGG(f;vq^XS7|pCEp+%)5J)smp zD@1(dY8~J-3fQ|w&PgS9bhld06K@QJSM5w~kQ}!rNJihkL^NX*T)ULDnX`(xb;mhB zQm!V2rVP^(=y9;?E1Sm1Ho{<={?5p<%Ry1W-B-b`_XXBCJ^r4_^8@0Fm-C08$Tz3Z ziS5Q+%H@Zi&RDy3zCyOAuakKVGDH3yO^r&m(>&KwxJ!u4_5V9~cmS0j&gxTt;3{6t z-A2ya6G}$b6c*0_jazXzG&jmj@MOthhE*74zHtyYC>v(!E#5JQThsFBzHn7q6deRY znu=%+3$^iTkcmrtTIq-5oI9?GHtCO?f2yZH z?5>>P&<-)NLO#4&Y?nL3^JX>dv5Onc@dD?+lcu^Ab?T@XRDzD1nS{o( z@O_(;h!t}8>J#x8;`ylO*reU9?@87^R;tuyyM_pep~1i>B+qi+zj_^yh40ME+PF+u zArtC`F`e2m4<~C*g9m@|81IimDihB>OiIXdmTY%TM1X>&0-p&r zZUSLeuE_6P^5kiTo59>`Af(gmUr3-=ex?gaK=}rBcXmfnEi3NuCZ{Imuxt)VRxeI{ z+V3&VEyri^BFGyu$(YDn9FHQyl1t#y`a)vOvh+F}Zn4FaM>U%dBii1Vs)aH*4>rbG zIORI5$YjJ74V}2bZ@V1(&8jG1{rRL`(0OVv32L4SOv6%SGx9C z4BNI@Onl^NjYV%K8xAW=0+aXz+hr`arEq9sP# zP&E3B60I;fg7B*e4!dcXR%3k_vwPYi`ak3Nl=JTM(557%b~WrXyx`uOL%5}mm^v-w zA8wf@5bGds*VLXMTIsSuWVUI?#j212G!#5rs5Ezpb|Qo7$AVzscY)ZS0{8pAO;lf= zoy2s-^!)?2P$J3j#2XRhu~^l!pKC40Kp&yf!PvuE23pj6QAO42{?7o=ezGCz}v% zFLI3WoXhyGgo%~apMwk=OJxUl0+M_Zoi|gmGHFs=>Pxf?ectfs;>XBwGA#q{;?SH@ z>ChlLy%WP?fCWRjbh*oYSAm*}SYLm)ILKYKqF&dt=bT@BMo^{3?M{6@lq~T*QYdkr zS_@hhGP>M}qN?=f-LBX-8Qa?=A>b#i#!w;wm*=?^O3LM8%T!($V7EwMF5g?xRSvFC zrOjf|lv8`~$`+x1RZxwvoVPiQ*>ricGE303)xp3`#xS2{!}q>)||8K6qS z^9=d?XWzRq=R15Yhqm^tzvAWAn~BmUbn;KAakRjjEAeIlB_LK(X}-5^w6`oR2<6ja zSPrKI6iBK?-ai%ctZO@e@-QlJl%;_}Wilil*IXDKK!SSeKStvZxb6HSb^+$i__laoMjWwp{4k!_W%#*vwd5 z;p7~QuV_k~CIlAT{C~V3$}_D)tit9o(I^TKMA`7jZ#n)&^h|CqD9JHTFuvyfw&zu! zY7lJV8^~`W+rg{S&RHJA+l4x8o;>w%IXj!G_=87ej2`uk4%5OBs;^EBz82GRA+-Q)=_GIk(zDRE_Q}9vzNHI>cN7aj;1k;>vfq)&Udsl~Z}77XK_zWWLNgiy~q&l!F4iz%JMJuqoCINLY63XdV`L3r&0hRObR?r?(GI zo<3Q!asO_{&PeOpDZt-pL{wRH#P##ssdY#;pH(j??g=VS$z=(492vt@CrttOUIFgJ=L_NZB2bzT>-?BeU2{TWmQ^_IvM~nk+R)2N!#1 zmb%Yz`>!pagO;3Zobh|8GO?Ic2a!Q;5u@j67CGkfEYR26G{N7VzitxTUwknAZ|7*m z^RUB>u|tCBb8n`$Qx72`BpGp6$zm&IdBU0d^P6mJ)^DPUX>@1 zwX83>t+v9pMYw%i ztg7~)E780T6gZ6>mQ8(&F23p!|r&?oBB<0iezMR9TC;i{!XrJvm0nEt9Rx%s?> zS=B)BGv#H(OR!_mX-jWcGUWlP;fpG#OPeDU6B5kQ(wJxW1{djy{0$O`Byi@uT)<~! zyJu4`^JN341#M|c+4X-0t^6H-pulNfAnNBlN)edRw}Uy+0}ps(dS-EGK_A`{bz8Lx zA?FI%7MDvAv{mW6TuBDVt1g%s9N99f%SVM2r;MB^fDiOKS(mFI`G1ZVWQi5Di7AC` zoVqCTalZ8pe0DU|AYq%25ztk6u@dAoFLJrK^l%{DL1MO`aRr%&O{8R|v4S0bEI6u1 zAeJ>Q@@>O8@L4>ccE78G1d(FUX@Yk~MZZtxoQ{qqcL8fIpi(bOI7@vEKgTZ^t@F2< z?_9yTsmat)Avh_34F1*bC7Lf}-;o*+ZaZ!%dBw-c$w%3S^PUKy1?+G3HqXbJ)mTfG zkrbAEV{sONyoq0PSD{l2H=QdPD>d$=1v#~r#3DuTf`*i$BpivvMK0Kz$Rg)M)ltB5 zA)H^I4O9CTYf^Zq#+W;UoRfs|*+dd{7#e9xyHx^WmIX=WXbCft(d@NN+K*pj46Ey* zrVjdr@FQsGU?{$abcDWDICwwQQ2wpDsi9r)U>h6EGv2UF2Ro7VI5|R^ zjujuf%P4|?CtrvX6TH7`%c+8QNtn`^JF4G$bWtUxg_uAC8zn4;MD;LjI37cIEDwiJ z*gcJ5bYicIDxl=ESa|7=6(DMy#xqSOE^|7eA*PRHs(pjvDuwl>b@`^lT&@BJCCE#N z4&rH^8?hL=u*Q}Z;zNaLdcgiOT&%sg;&N(wy-ye{qKf!+biauX-BN>08>LV=$?Ksc zaHgA`b&gBO^Vx!rt6RmPyo8w|1+GS8a^})H`6sT|%FfNCea|S8DH(OvU`WCP-*obB z5|0-KHg8*@H^KNHKXHGGZ&4*<%b1;vdfcu=Goo|+tNqm8}a0+*LDIUgG zYJw?to5>~O6n)THC3vdP&*}kXghLyy5Fv_^7Bba>+3Przz0!7Q9vqe%F<%&QJzw5p^Px$^e?T;Z&k|>=;!6dIZ)6UhQ!E+?oDY3r|au;d%%DDAHCcq-{91=3>`Ly^C zX@7yU2)i-lgH!1-mTA`}yZUh(EqdutQ#u~0&n!R~HY(opWHdWOo9f)R`2r3W?ssf) z-?b;2hRNa=Tm;NgrGx_IEHgavL3r^ERBo|Oh?MeE|-u@kY-T@I056j^0#(I7h{i%UBCcJTlL{mVw8 zVOztcvn&kyw-lFeLIsQYoN}ibRD)+;ScQqc&~2z%e1c7$)s7osHX(kjO;OJx*L_66=ne*J`%8ie?jJAJqw22Kz>A!D% zBh-RP7S|9F3mHQEd(mPap)=Q`)ggD%#ePlbnjwUUV)lj`gpo0*gsq3$&45d34qcA#DfDBadH^Gfl%NMg-SXp7)c>9 zPuvr6u8K^SvsC2WPDOHKD2gh4IKOB<{427bE>pn7ct(I<_^h|KdA^cK7#OJHFmU3Y zJ9}5^yCpr3zJK_WH!QUK)P_M{Dt4q-d+~}RCQZ5QS6iPq5f^4Eb!}cYlE_gclsF`| zEJt_hbXVyrpOLxjpZ97bYnDs>_}G3_WlgcthKznh7P2Ka*;CFL-Mz^e`SPk097olW zgs>C1e+55;27A38Dx4B>An1!gc`?8>dKK}}#isJTx4m63=aya-KV<$*(AdQy+W;zx zuaL^j0JKleemjmlJKM^RE>%2EOC&Zv?-O=nG=v36T|L2JcP za{^mhE+Omh4I^2dWUDQhu^DWljXmHMDH4L5ImW|}E|)upV=}<1$t}fV`pccRnhYnu zRkBW#u@Cd#R#aYR2lp-DB|B0B!l1}M!^^H=;6bwAk<|N3@|}+aCwg9ycmRgY`5iCp zWJXsoqxo;FS4k_gmpLKx(Dq?Jjt)ElUqy`4bsOTFPTT9vLL z^~iRms+iuQ#3Y80hiB*KG|x^O1@5@_im|alE#|Cp8Atsc8YzXq(*fk$u4(Kqe*Kfse)k?*L&Vzh6!b|B z&y(I>r!MVOeVEiP!uY_;6M#XIm+QWY!OiEdv&i+cE8?TLucj~x@g5zm|B-&(0==aPIYC&Ip{{}+7O^3I+ir}BZB1<3 zJb9k?yytwGHEZopd##zhultYR)t$WHDWe`4Bd1ST%Z$Tz*(U^k(d#@tu-`+uc4!A( zcnoFf;g}gl9B6<&oGt-ku(sSNQjGS#KC_peuZG*RLlPU^3u-StH__>P0pP?0isxlx zjPL@W7KK!*ReIg}>-9o(DTB&{u3?BXs1f!m<_@D$Pc{Lv@w>LQt@w&%X}0X=?9e?F z|Fb@jOjzMl3rKMpyX}!!!e&3VT-VPI9@c$ze4F~h#bhQtQ#_7G)+(dB@sGHNkjJd4VA-Tg zY^H?5Ff>BH6uQyrmbSXlNte^SI*0YXF4Aa~y$^pWf7cu_)~W)6xW1K+H)b>OyZw`P z+#Mo=8AL>hw#O2eOux4smRcGCs0A*FYcxf=a8-Ug+%Tj{UR&5(NZU+2O)X%SW=o@; z>34H-hn=95Hq>*A`6RZ`eM9l|dK?jqZZoR**{%xdsc;toYZ6_P8(WQbRe-I`Go6kr zkRW%Vt`lj3G}F@1eD3EYyK}ctbeulSmng(^qW1h1IJnj<)hPsZu*+BTtYoFrowv6B zJRJCN1#Sv3=`vd~$;L!fBSRF&;Ms`*ABx z?*YBDF?Y!M9zd)NeUMy#w=p2StQVF-uTcU^YmZG)4l3(t)oJe0kfY9 zA{Lb-Eg)ibICu!?sbS^Pb~Mx8)>f7fBv5vJrM zPp_#H-y|+99oU>las{r?-r#w;-hd7ewyhQt%iPC0gP}=4p{s3>wBKdY6da>{9WtN) z^L2#QF0pYn7#L!jpPO&0DB$-@YksuLLQ zumj2FBIUxhCBdiTuoK$M7`xDTgG_pVIphD!KU`w(HRYtOW8uNpgVnAMw~ z$1Z(5$~vsbUiC}k5V}sJ+8}k+Q!KmYZ>y#mF3#!z?XQ?Zni=F8QAv193wwX1304|< z)M$@hiBILb7fgzgz%-ZAd70h7KoOqMc)-(9WHs-1pNwqVAkRgdVjr1?^y`_Cg#O(9 zii!FQU4m;ZXcvw^jl()*-$n0t5&Udzd)_mkSaZ1*w=TsP(BDA;sYD_8x>o743me99 zlCs5xP3RBBJhtpA6$Fz?1&rS@4lsN=`iNdSjt$r1kuHEY{@?~1Zq~sz zQcPGA6f5A1(ZqpF&MQA5+tKd@v_H8=rAw1o2*fpJNq?o+p1X^nZk4?hcsCC&%3@_? zgx!S1l@II}!xZL@6Pw)WK3|UV2T{TtjZ$LFWsQ2&BkKrAWbL~3U0}5PJ+vO&#UC)n z3LWK_@$;>$TyA*@jF%9^X-=~b|Hh&FjWY=vLvpXaAm;ERGCJdMou+=7kRVYJ-Iq)g z^$4f#>)R0U4cB&ZL*VK@uOeWn<}5L)h}I(ZHF7iECD6|pbEpHx;e5MztkdeLs^}DF zCY7M$@95K5d9rBE`>j<%OoO_V!{pFAcOl#$@X)7Ol^V1c_yr|a>AaGDz*ZT6jepHBi4 z_n!jlyn}Aog((9iZ&0S=2w8hC7F#S6B}MK979Qg*auh=rst3~(;LaiYrpW4&P=;@C z`tuOTovOb3l>G~~K&%O|wHRL-bMRzZ31DD=j=GPCn`Z9+{&W!go+3yVm>zFvCx;M( zg%|EXA@gquI3^j(kkC82Bk)b`uL_2$F8*PbsOXUFBsZonOT0hm#2x+?Ke zzQ(-GPs2cgfcBPO-^N@09eMs9ilx}}FWx;+MrT3eu7k*6gw)cR>|pB}kiC1g?wxxf z;dBT`-V=9Bb9z=w(=#a##BHsoh3#tQBh_GeS|bt#|Yo8fSdI0ZtZkiNIw|EROgAnb_nmc z|G7!!o&@HuE-B=A{#q#S#s#w{l6w2|%O1i%R&_1FcCPcJ1?YzDwoG~FE4gtIu@89<%-~`0e5cg)6BOaz)cLx(%yOsY%4x z4gIJ-UJ&zI6Mu8H_%QasK+lm{IYHH6Wj)fqYSv<&n)$5sCKjv;g-2*SENBMU2TBGBMfT5jOhqFT*DrUkq_8CUbbMz7c!uf(SCmry;}YEO7Hd-&`i7B} zlP*ud+%DwD{#E?eXcR=A@d+=T5qe2*r;KYl~f(kZ{%K7DSEmV(5h zRhZvB%1cToZb4QYtm2jgK`GOCfbwlmSy?JH zq?Jk}4gdZ&d>w5@eLfu&dyD6y(_1YiBhJ-HdXdXb);@;kg6Ixm^-U?Ct05zIAAIp= z9kGfx^r~(@aZorqJ&4-6DeU*V58K3H{8~k-!vwKWIweUb$+V=t*OC5(f7cXNa4D%< z=z@WkwK+{0uVxFDF!-Ca$H@v@cn9J?Fahk`IE`x zX$K+qRsCU??D+R$mCH(9F`62IA=Rh>UDz}A9%ap=E@Ek2g&8o;Gq@UWbhYyF>uZs% zB05@^If67~=4?Dg_boGA;iX$Q3%w+lD;sYX&HHhF!NV` zaplq=)NPpX_WbyrGAY5ycYqm{KohYqn%dd;;5>Umh1;_{CDrpvCxO5xhUyzPYC)eWHsJZ(~DYTWXG@PnGF{uTQa0CWM-!t*E^7l5BnuIh= z;Zlr-ZpDi3-}GTN=lm^aS^YQhJ)XV@N*wEI16$u!({#hCP`WTl{ zHKs?eyQAi5OHa$qKkA_?!Yy%FQGwfvP!f`u5+MXhl#F^k)V{?`s4mU(#>!EDE z^sxe89HiGG9&_H&OOd0a~B&?5Skrx(>C>J`WfgPfeTkWFTEIC7CFy zQ1JYuz?iV3uASmGRg*dan~zsEdKaZd_`M8$KUmWqu=&8O=~%6~%QNr2pvN_M>%1mJ z`O9W^bEewu_*|~!o-0feeGDTwXq9B!&qW~6WT+kO#^}wNN63am4Fj_!87@{(V>>3m zAueao{tZ;TU;N^_VI1*YaffN1I}Vu_p>wz*&`V}&wX!t*;mv1 z3B5S>ZC-VnO&`JZ;^b8Au?cpfVG5p zUUgrdE$Jef_6}~tqNP8hC=Va<#+0_8)E!O94$9Z*XMP7)(k#p=Mqz7Xy8+3Z9hb%B ztb3*edu-k7D|7mUdqy_I>-4&3nRn7aaoyH)@U^2tVOYcLf5R)eyPgoEgpC{v`HST+ z>*KH+spFxdLBk3+*DZ3gehVMf@#NFl1gPR7W#s)tOSFF(Kj>EC0MtR=)AcwapG`b5 zR?EB(0LYk`Ky|N*H}};Yn5{GLfX#kCK7eTTx zMcU~hF7m{V>YG+DEvWUlg*TgINKb0ZWQ{LkF-N`d(0E9$n%{+GG&Y$2PTS9&rFBO~ z*AIi(-htf#?mnjp8 zifGxGQoU70^fXMN2}__TKrMwyy;@Daxz9syNj>-o-K9`y)de~ z5M_k)U`c!P_7|GNYW8S?{_~%-U{bz>t-p`6uyn<$x@2)9pt zY!0W?cqwn6#;AVgp%9?YoMWHWzd+3LnZ7ySQ}3wHtHiQL<>JXwa))oDNYw+|3p(eS z+n4Y)lw!4IL97q!v+DET)Oy8IcTMDsO$b~uA@%({ozGYZd>h{FZ)DK%xr&3USo8EH zq>yywFKI2sGMnTy)F%b<{9pttNTh1yoqu=g@P^E=GQ{Tr4PReD%VCbnl}P~=j0Aa= zZ*~ht^88_Dvam8sQX+j+QC~5WF&t8RB%6*rcP97klE0ZG#RK#wgH&@-MU#+(>%M{T zt=<$%xd`>h<_2MZzg2Wp*L!7w4pJ6g3H?dXigy`qp9|VM(boOYy*GYWs8apfuZoy? zc*nO|5j*Ha(~QfoNnj_Zgz3Sc4**Rx)w?pFriaG&d7KPGPhM;^GC2;>;tEj(V&dWE z|CQ3av?HP0GV#1468n1xBt$Vs=Yf-+9j}f4J$)H=M}8cD#sm<@d8SGeh6(NUh!NV7 zGG{ACy@x$nw)Gd^`?a*sh9c%jA&X2;K{@|Gz0b594@Lih2BaCvW`?&a$_iV^ zqc{0>`3J6iiK|L!3M?DALB}AZS99()_jWP%Yx8+XgTs;0LDkCMpwrmRz3rJe3jOld z@iTNhhNzT&Q_aU=I3Kf}$G_ZT6LeEM*&(x_PD=XL3~PmOM4E#d&n|S0v%(|V`})pF zL$YVaAn-hq*8xtLA3N?DAH5^!8bq?(K#j)M4jfekf9TZ7t@^soj`U#8Q?iP)1pCh* zFokw~@wX_*Rlc!185DjHz7iE}=Nlh^8JEe3qA<8poBz}^-i`R@j^9uw%`XQguxfRv zctgqF-M56#$5(Y9@<(x1R2jc~@ycq`9EN5aya0=F5h;JN!HNRo(Nt&B(5XY-1Qy!0 zJRmmksWzl|1T#3PuYF$+!XkXT%0-+nKng1lYw&?!_=EXH|Bs4EKZSAo3a2a8+>u0vgGNk9=C| zVMZ7)2|0#%4hAG0Tm_Dpc-)C2H#IBMOOI1e&+_+{S`A zCn2gvQ4(9M0ZF#$f9eFSro!S>HsbxK-yzB=B1UVDd6lU`uKuDV*^H8SdqOv1XN11u zet#6I?PU%S7EMlMPJXo=(c}NGZ&*S&?+p)vmz`d@o%>m$anoLB({z?sbH2YI{cc>( ztRi%cA{+r=A}_H1`YIIJ)|x3Nbs0u;3m>?CIY-B%lJXJnr%v#YgjC1}DMh1~O7{)B z;YWUp;LWq?GtHigt!Bp&Djk*2ztP_hwRn^C1_sM&6FabOGK)&d`7Q^!h08f2IL?*( zlsP7eZ9I=r!sF^jo->WhdbM@(bs5JF9WUSIXCL`RvRl<*s&1d13Uk^Qi2usEwu?8; zd`XIJ4tVxiaH%^_PT}(?+~a}Rl)IFqvMJ1-(7}9?35$Sy&J{5(Ny6K>rhRV3DE9@f z-(pw)So@Mx?dKhR*yxS4mR&-ep=))6eu&B7kT&Q?-FHv4*|GBc(>~!J*I2A!#s%5H z^%ijN$bg^=ba$jO;DHo-vk%OlA!6Kn*LO_Z<4u|~LaU-K(Tm1A5+s+q*{cM;c!dpk zSyla6$EuKj<2ZI&zytpiZJW%{9=VgZ(lI1%p5EWQ*lcLu?W?Prw7gY=^%PJmsyNIX(ynhtg&sGtWuaf# z+r$`uqpgSO8E&VS#6n6BM{4{}DT&oO^_HJ2mZz8i4!sbVL4hAwh)-Y!TndxAFBgkm zGFAfV*GErIWOqDbD}*rDGMl6jgD@2L#iA&geRCatZiO`1MHnGw1sYeQp@Pt5Agw$Va!Y1_S$=)Rub>KNDJco*OMX!h{; zxr>dcMrK;F&(%cxnLbp2e(r9z!}MIDh}$DjTuQx&lN?@6Rv7=tceAht%w^`9EN8%L zRY`9^iXWvata<=Kmv8vb`I_aA6*pV03wa9%gT{SZ9wfhP!V(~4*yJa!g6a6}>v>ua zriJ~jt(H5@R}V^azs~a`#i2wizL!MNk8jf>Pl|Jpf7i#=1us13Zasa>+2W6AHoSI~ zxcl%lQ?>5TH!-X@8qV;J10{v9VD|%H+h@1Nllmq|(AC|wArCMGhhKIVP!9AdYp{e9 zr<}YJEytBlzuX7p+-UCAkHNrISJ@oJC-kOD`ybpAVTaNpj-5NpC-QyDxWhXqzGjdI zW5?ig+4mHnbVi@hhh)Z!qr zQI>9HWC~fHFVYUut#*VwQB9Lre0l9yInfpin7K|M9M7prlbiXT-37U2St)o0$G=pC z#svlFd*UY^`AYljsdfejDi30GJIc9u8AWEjWBgBEWJI%b@jqkf)15f6-RI+U(V(O2 ze-isV>M4Vt-%?~gKQ=B-a&BZlFXtL|Cpt2AQ6{RX*e5Hu)$OQsy%)Y#p(Z9T3N(7q zoIs0sEL${$v#P!@YAKZaX}%D@cUnZfc@<_s1DV<`#(+nFFUQT6dpgk#eau6pu29pk;rTvn|Hi3?~p~K z`p&G+=levM^S`1apw@eJw!~-FW&D3q;h(_Q&%e#{>wj-+YD+xun}U`WZ*jZ>ru1=q zROkTbKx3VpfrC!4Hx=q#jeO4isZ_HRqnB?0S99pAD!5o)32}4N83#vu+XTqK027GO zP}32VvGJZysL~+xcuC};)0vqE8=Qofd|-e7|J~e@O|P@NZB}8ey}k&u$?iPo?wY!J zLR&iwTueSU4_kC!xZ-W|6lFcKGyfMr;(T+F_`AP%l#1{t)&wR3Kork5~SO2hJ zeNtFW)e5&@RO;8HSA=*Pz9(St%{|bva{#VJ5;s(z0@PdgI`|7g+FIV~Naz{x`tO~< z4vkXD?`Q0^k34!`;|=T>X*bVM#h!XOw>AWQ+E@6>Wl?>!*p(|$;9N1xeRnwMAo z(@59ds=a)!H=C9knj|OZnjxpA1q>9 z-7`VY(JPza7jlaiS4B^=lO@j?^MA`t$LEph-|u5_b|RBtREz;g)RIs!SB-)!-UJb) zjW*pmBK-WJT2!rc&cA-(+i znm9&$=#Cibl1?ItBEH^g^3_QenzP=#zY3m=GP|q+j%*x#P-t*ixxri-TyW*)Z@#WAJava2y$NGwMsWVpin^9x@F*)sp{J{DFQ%74EV00K92i`5<4e361YrfBK7 zl$`MOO=*8pAGLXdZkU8Uz->KG2^TO2#X?)X)Iv{0K-&2qzz=j|+o!LI*{GjQf~Uot zLVcye&GxTpUOTz3w!cr#{I4@}hvzXz{19Pn2%-0U{-j;tCaiya$^D3iTWIpG8ok|p zlsqZ!yT+r-VW+a^YxjKzvwIVaAcsZY=28?#Vb*q)w#e$9|Z4N_o4|?YOo4U z6e$<=kpQ7b6WE`o4sDc78;H$z_(h?GZIwRFFM41{M!JR&l21tzK@)%(+=HpIYx;j< zP8K-o$S-zXLq1B;5>Id*#Wl@tSF0s|X#9~Nn} z?Yj5+k!xD7p5%TJa;)BOGhDoS4)W@Y{GxEI11_;db$%LO3tbbW@B*n>5ZdBjbqp$J zJId;93M!K>QfRo&6xJSQ5TseT#SQRe>T|&}9;yxdH+b4_g0{an2mgKvz59A2l8T9t zn1mM(DCoFJsoN3vj0{TlGeD6;4ZVDV7^!V4>w6%c(5$0iC=MJ_i2qT87lCy7vn;=i zded5#)b~-%y6#yZ%K$&V2Z3nNA=FeDuvCph1F0_5vmDCa^I@JD5{R8SLYu}BPj>3_ zhWUpoHaq*PyH5OnIz>+Ig%j93VNCt6J)di{!Kzs-;%;fKm4V~`Nr3Vn<$$7LxQ@?A zGX;O8`pVK3Ur9#B$7~&--ab_Zp_#zdOtTWgfQyywO!S^Hh}69a%N&M5QX}VayHM40 zlMJ}rg%q-Hn>t~O+{fn+oN|YMQCug|$@$XrE95P@=q}zdt!)Sn7&CsAs~yTU7oZXi zOA>()=|Pt_=w@uh>V-H~Hg=1Bm%AB**=M((gzdknBr6|)n5E43U*~7A@x@aoH{3cr zL5v?4e~{Z8Q;SE$B2PtE)5@Af)aFP&v4W?vHf!1A1Uo=NWe$Nq7%#z>ceq zIYBytpTYW??B{c18Rg;IMSe?eohFG^&yDN`O^_*+nL>aNbJQ0RInQ!>Wje5pQ)k=j zLl4D2t(5Z5)fRE?QMmSj-=xZB(#xH6*IQY>Xw;;hO;?&yBF4 zWZstF@V*5N-uEZWl9a_4*!~>JYz(l*Y${l1^9C!H zsbc`}4L&_`*Z_)6I)wZ`d0N$D(8onXuOQ>g<$F|%29_>Fh1)V6h!*RD9j*^bv0!6! zx%V}m^90PcfwcK^OZKR`gZcGxY$G1a4N#70`$4_WYrkOgTp8q?V@rUqLaPwH^D->; z!QILYzwLjGb*KYp>3HHo)SHXLI1MNLLYyxx)U{F93jRVkD`={I*Csn1n5b_W!FwJ$ z+caqg1|nTNfBIQ!umW##3mltmhK9rMyu@pSbER-V76cN^xcT;Q-+WBH20i`xEre zRkM4osc$wcQW<~!2A9^2=AVLh;sf}di}__0ITOOe zn=ngN3L7Awo4ExG1ELZaH?ES&#vM$SCCI8W@re0hQPEW{Bu?X@A)eSiYW&xt-pzf4 zvjzL#=DUq0P&OA}l5$Pd$rvI~>KE|wV=Ft*rrd6=qnBBQ1_fn2={t-vXjW;5_p(is za)Va7oq^-BiQeQTt#8Q|0u^PCjU>m9AjzR!D;d*IIOHq z2nk2pAdpPIVyetu)t_sZTS!wCR22vSA zn(;hdyR=FZv|6Ow5U%U@qk23w~<_gQD~tnJc&AlNrc`+2jb!cJrtP^EzYZ>Po=0* z0(U7U+jLi%OTs>zOo~4GH&KWZ1?&kDVfx&He@N3#xxD&HXK~XtAj)WzFGJ|mJ79H< z>YrE`o!QDu{Q|#gZ>dSLG`MbLDJ4Lfu_^VU<7_o@KETyMV=BG`$c`BWP*gJp#&rig zX5jkYn+Tx~|wbJz6{irp@ML$j^hEaQm$oq_*(a zkyvZMZAp-hx(hZ(=U?aXV6BV|??DzZTN5?+W0ffq{;pwhXi$fNFyUh_c(PHg@*o|)JuI;0;h3(94)od{VD0?wjZW6FFSPn zq3)*J+15>k=8C@0_X%;hAG^_tn&s?n&1x(}{Tg$pyT(k=6H{QKiDgV81{9VlAuO#0 z5ot<$s=YAkdKa4{TA2nKw480*(Vg2p>-AB}L+#<=7)WJNaBD=B7PHch+B6#%`M5ua z55Z>v2OPM=?A!4iH}M%R5Xe?JYP2`xlqo52sMPu}0}7Z`LNQ&f8*W*-+8-|Rs81{f z@e|R-G+w|BuQq~n2&yR%Zc=>0*mVFWU~FzpTS3=skTOKIh!O0CpQbrgan5IBijRK1 z$lSa*fdhB3Z@aZqzr5})F#WaK7O@J|2|B(TdbRstC?R7=7VN35R@4*{!B(={_!-)= zWmjH>Q1=;b0(qK?o52y4Pof&2rexk4(0k;1W=i1RzP4f6v$3_fdZ&CWopcVk<{rP} zfbeT5*kcnHkkV5eH_$ri&K{wc-1|ZIj!GM zaoY+Kr;#O#87@#HMDxKk- zOOH03|HKny99q}r5AKPinsFH?w5*kGW}0-Ss~=RK__^3tmamFzmLu*#UF$G?O)gs; zJdjKac=Gz(vif*b`F=m`JFge~U6Kq@0qyb@DTMKOSxp1?9RwvOuuV(8HcoC?eWKWY zLB>FQY?ip)KkD_Z^6!LpnGiDEv8g+}dkrIzUr3j}B)cvm`V}wKYFp#GwUNG3=G*=i ziOXZdG#MV0^ooV52Hd-}-E1Xbj^GKPQms5#+wrS@tWMIYqWU1FbpuGghz5SX>o$Uc z9MF)?U>h&pe>=5k<~}aEaS~*~YyYveCeMyWTHu}vUJvmH;F}Ko-183}+XzAxN#W>~ z#o{Dr5{vPLV364HLP`RAjh8IS`Rz~Ale}_|Z4~4Z@jh(*{_Ng;U#${iWZXzUxnCw; zw?(t-1k;85Gvg@DS+(F@=(b*GRdk6G#mqMr&XbBG+hTeC!7}p7YUy`L3KsBwoXQ#G zA^<6M{VRsK^dt5|Q)d^TNZfr`rrrLr)KV`?F$ z*)~aco0#zwKWHZa{I8dYLYpeO>epCy=F6AGjepJR^~RFGjnufCW3ViQEV ztkN}F+e)D1Y8E@lUn>zHgJ34yWiS}}`O>f7Xlj&y5$bABB8m{jIfe12htwDT5TJc? z>~6Y{%gZk&*t|;UEpt(jO2iKaNNJ*o4!d85?}L?2vSlG%N>eZdRG}Xq_Ayd8%G&MN zi5fbZ?U^xkRWw$s9*!vw|$dTXu@zYtqjd$5QtE~LtmhCbxya`bxzy@)?`t_A& zF_T9rd31gsaI{+(O2Iu>a3!Zp^0g|%5K4;|Cw!SOX%$%Rn7(W|i`5NvLn;@qOWHP$ z!-8G*0|Nk4vqVmh?1{O=b$h1YoI7ro5w^2G-V?g_UXb|WiG|FC^bu33cxZ24`DfS9 z_oQLA^{($P$7i(g(U#9k@808h5X+#L2AqK7*uKyZM7ZvPk%}N2R-#(emYaayIZ{f? z6P8eMgp7U!aWu?T(tp(6(-PLGFD!OBXk;wnAxt*S``SfSCI)U#Zcf5R?zM%*tyIrQ4Bt-@ z7)ZquXez}g3h`G<%*r|zjOA75p+^GG8@?#J^+mEy#4(~n2W^8)FKaU0rmgs*p>R|g zowLVB zuY35mX7DaR?XbpTlhRBb9HFkJ?=M!|_ zV6i@7;`KInooJ|5Fyy}xO*2z(!>b5T<JZ1LDm4bcZ0k>4&?4Pa~Al6%?mDKpA zH5@pgsc(<@GRRcXt!OU_S{Hw1Am+Eio`TqmLHec6Ow{4M__=G7(!s-rxP8OI=m911 zbuYDY_(NuPc|8eYttDf9@_WajeMfY;Z}dNFdo1g<+o~yYD4#=nA5c=-hYHWu{9e)t zVO8S5&*J4`n%<=4!Q!;><`kK-H3Pc8bN!>+Q=+Gmj73cX65 zCok_jC`!Hb%}tm3b~8qq_vhX!Y0-)Idkg!D;wO$=N8O`Sx72~{7u9>NYPQDg^>*x+ zy{wt&ybX?{yLR$D#LSq_erMQ;&xcv%=9#pL)F=`aai=L`h87;JV51>#Kb0OoS=`X@*PS~D9vdX{ zI5Dy0O@}BmC&%o6FST>+I2hSEsT|xKc)bVEhf%2q>m{du9dnI5|#zSp9T6dsd@r#%$4^u2d1Kt_PvO~HWV!u8%Q zGQRl*Ab0up=p6e_rEO({Njwh!{h#B0O9SYHYrOhv>1SWB?=$Pp5{S(NkH`Q2TDO?!gFBuFDcSC9glFC?Oo z2mfpPjQM+oPb3U`A~nagJw(^S=cPZ3SR+=UkwKY(8&Mi4T^{vRO6A7kKdBzdv%I>K=}1~^ob(qt7`or2B0vto4ao1|0z*gl zsCyZ(N+**Ojq*P4>CTN}c9M&m>;wiSyW%WtPPsd!cY5wB#l+Ed+{4oV224J_Gv9Sx zm4c2kmJthe%Gr($uiITC{RYQ*BT7WPVX}Xsj)7#Vo=z?kquN|WoXP@Tz>z{e-3xx> z-%LabM@(fIqNmUUhb2AcC8|g_6|MG3Xk zlxvQU4Otwv+r_6gx^z|V^o3$@p@K!T#8|NM5#unQ{uizSP0mg^@}*L$=u%;UFpNno zYHo-OPq;Sz_f|a0C@=}d(#9gW<+a{L>PWfgTt8PzY(h0JO9nWqOho;g;{S)_3k$&H zt&Eqg{;No=G5#g#CcY2tTLByw>?oNGkWD{wxh3*YgbnG@3IyF3_}0KBy&h?1O`f&b z4i)hsGyf4pz6DzA86n(A@>iP^#H2hKgke9db$;S6~7dO?QJW+Rium9)iCbK#nKXm#Pm;& zflZsQWj5*BqJhvw6d;xS0|D!c$ab0O{6cRd+AqD2m=w^F+x00BKW}3>XT+`crxp0X zjp0$xLE)lChDCfE5NVhf%wdkQi(Mo6KaxS{Y=^G>v@_0n> zgGszV-fOeiZL{3v6?#G=c)BNeawSOr{s~JWNbkfgT%E&5b^dn0;MWrJ)}67IiZddH zkXCyj$v6ypTFvUTb@TliCgc3i5!agkd+lP{)qap-JaIcW$Lv>Y;%$iW>g!J`Tn?xa zK0p>GL8^VxcmF0OO-On(zMM=NNGo#maZ2o#fi1>0>~eniM?57a@k|Ux@Oy^ypCY`b zF~UrzX-Nq&-X; zbGFSMjC|5`WcI{^?~5+^Ep66S$ScJX>eEnms}*o)=(hRj;u^+e{))$a-vx!0XPCEJ zn3N4*=2Qvib%6XgB1J*p25vqi*Ub-&eG5G!r_tY|@#)~Up=Gk)DJe3uL*4( z*t75ihqrGq!mz@qb3HQA3gLjH7Vk16RdBwa6sMv=jq^&mg)|iYag%7uL04aK?2lXa zV61aywBqLIMPxo-;ygy%;z=4~-a)9zwq`xvkCw`E%ucKgevUsy^5K>I{NrP8F3G>1 z#S%D4rkV%*s8PkCCh_29fvi1>FtW;ZcGiOuM|NkR!qR4R=gu4lGov*8>iNgj#3=%fPFyVH{;}>$f}6B zrb9D1Z?}|s3oV~Jse?b`Xy&4{{s8>sBvJe!v|>6gult^L%`u;0DS{s5uPb);BpjY^ zV@5VmvnIAnbY>a1PJ$Kw27oWuBg;hO^RXvnH#^nLULPxkGz-S$rLshBw{S`^66b$2 z*=S?uV06P~8G@!!S@l+~HfvG8e@&Y#393UU@MP#>GJPxe8lNq8K5Kk6N8k^#Uf$`( zAMg`|;|t)w7{Y324A`O%t7Ul#K+j@?;txZW&bv?74)=fD7o4-$p@jS}UU=PWW@c`;{oNW|dGTVO#N!&V##yrTVSDzVaXIv*CxP%1>ZZk9C0} z;*YOjK!4~#e*pcCC}{bo9>-Hz@(BDcpfOas*Vi9=nk#*%Kjeu1<&UsBIV>6)C4Bof@+ski+{I!qX_F+Kqis{+w5wXRcOoo?CsR5T| z&q6t)JI(=~p7Rn9(F@z#!7lv=y|^L5!X5_tsoQ$@)WLWk3A+TGZR8P8=xB zx%4@$ti-+hllNl~QhmWkj^X-n$q;-jTY``U!KaB%wdY&8!l%q0K$C(98|?tBT=L8@kvc0`fc;bZ(AEN65Zdx1@HYf{zX_qh!Sw-YbfOrJReizXfd z2Yz#(i#gXl^y^qQI!SiDxyE0r3487VBNVY4_L!rgsfd7EoV+Whzkb|G3{Lu;t~|w7 zo>{Iu!*#Lua=MD{Yq^X_x1jo=SLGApjRbi1_GRp7t}h=p2$Q4qp;C=a>+dre&4Q9; z@L#;;sYFVD6gX~uM6SgAvgomJfMkRp=VIa@Y<(a(N-bDh-1~M6O9&Xo#kzj@G^!oF zjU0u%dZ_u*D6-p6j+Vn`Wz1et`R5AfJ=$qLmr63yM2ir;%dBq6n2t_&Kr<~O_z6lP zp^PIW_Cdr+uDe_qpOiiAH7Z;FP-<|gWVL&;8S4MKpD)FqH7FSfFpue%LRzdvpL~WA z*ZCq>Kvt$Otvc;(=DnvfPh3eZM&>rG`KiiMeU)i}a%$_LtoHt8k#O zm+EWd8~vAWU^Oj;)4mPabt!#6&ZSzC^c^D^%Zw@`Pl}8ND?h*QO9AYqwyDW}t)RC$ z=*j{QT9nMF4#A!{J2_DDj-$hd^WE^m-dP#vmQ^1Ix7e5@0cX3!V7lmG{Wt+>cGO13 zgL7rz(N7Vq{8bvYpPrhv<28oIr1RiyV?1r03)hnZ$E+!vW6xMAPBK)DImBzLO$=0-LUVbj>R{Oxg=Ivc0zZ7$GJq5hTy0&SCq zd~qjG#`!t5LDKq1%#K6w?E75P&0I|t6@&;~)UIHx27xO>9AV*QQO-#0Sd)yzI5MxL z`Xo6c{V%?g%YCH$)iu~`C&1x}$qO(4=7nsR0Q=2)Iii%}2-1}7l4%u}*%ETBn&>+E z?~EVy1uz;h3s1j(C&vIw&XI3?z)Ls9DdhbOIAYj%qnCgIeH1AW-7eSx(phkmC-KL@ zrxWc7zcG(bU}}y-Q0(rc)9sheBnc=2whMP2jApphUNzONRd1zfrfVQ&wbP ztHZ>RvjnI}h5@#jL)$z#F>9Y((+No|lXyC`qI2E8pbdQ6D< z?>l)?s-L{vs*9uHVOWxlzmTz1%r6gaKkG_J8@#p_n$(!^;`#T5l(Rgp%ZtqAGJ7SF zcDSL6rEq7ncVtvn-k2Td9?}S71UUb!#Ev zAyG8T(`bXoA4Yi9&ElX>{2uxn`}wob8RvojJDI~4&W=5FHTux6Ox?N;V@YmcFSh|5n$J-F8&# zFVH98XX-8W{;ETho(dMYO5MS!*m#iSrSoXxURM^&;aiGK^X?3xo)58Ti`cJoHL3NoS zT|zX&JdLjF9+7o>575j)jY+|O$|Js zv@5Wlw*C>Y@VQC_{%`nv;QwAKGOhS`z*1Y&FW}Ym^8bSC|NO5&x=~$DOH|`@%e#Sw zb$WxyOP`t==>5m4an1s5hkPAOBK0x{3bLsep;y%g^i}hkZO;Jc&^W3N?r#u%xk+u; z1$A4URG*Jm_lI@}ecK^W;4F2ZX|;SDy5RV?&I4?QZR@0_Jpu)~$_Ey<{Y+p49J^A3 z=En+|eQEb!84GUBc`oho$Y`MwFKS1hTNEhgW2-T$|H~zf%u9C+P7Ych+Vg#0Yta8% zzKp5?G~nexc~|Qg!#+>y6G+|Kv2tH%wD+5e)|W%lOYOVgAPJ9F?(!I=ooET&59WGT z&R+d|V!qFz5N9uikq#-x?aj#s-GEC3o*s%r60y;*)Ls0;+G%O93|&?i?Iu$&ZM_N6 z=$B9um+R2t`dvjLm4g4b|I2Sv`;`k$BMbX~GEZDKqela?c00ba;C9^S5hS2pkkgbS z?{2*P7`NbZO6{HAT)(TF20u*NaK0i}P8-)zN#Ros<|Y<2$d4ebR%=)h8=TL$#q#O@ za2~SQ)hUywMHuh+_{YcpzcBtL!_Vdp<2HP{V0T?T6CCQMksxpkr$t#JS$@=KV{^)G zcs@}pthF&XEmK0A&(G$Q2P+M#^<%h8Ie_y7#_tzj{>Sb%cm{kytM|ckfg|Rf?$-Ff_#H$Om&_Ard6NDSjh3g!Gq5&s`r%qlE#`8iU!L ziV==Q`0h~q{*syQlvwR(rnXszgxV}5D-ZxE42#hOYO~mD2d4%@=zJuJaFzW3{x%<7 zRNv9Mi8k1yLk_lp@qi>1MxsUV!GkBJ`)aFZDRFxJUIIQ5BAdi zis7fAR6n(;md0PW?mG8&SbqQv@5uEyT~5CiM7tcf*XJ4G_Zr*7^H#N$ZxuEmOofnU zwDB9Y?@}$z2Tl2A(9>b{Kq#3LX;#J*4&z}x5Q{OG9BYUm&kBVYu?*k#Q`c*C!X6L= z!{H3AkQfC-c&~O@S|w!BWh|rmOxVAYL2Gv~9EGtW&JdggFPhk00F|>ef0a4;z6D(! z#tJy@kK@7fu(4@tHAIDGj(B&(>Z)3#Ay<7}zY9B4G$ryJU>P<-yjsz&Ne zT}w`jV+%fBi`4b-E3@$&N#it2leF#*JW0jqKOt*qXgM;wzj5+>&ER;2Zr96W+z!bv z3U%bFXx2f@`FP21KOe4Wsu+P}i1gf8>dWDF%S+o0dzwFfyM8%D@zU0J2Pid_(0(x$aa`j;$^RQrK(I8o92xb3Z|e~>Ny|_a z$&xt75hTM?I>~ImHv%-vFeLMWTPA+dEBRF{oL(n~u`EvUJWUXkPWyi3jzAUH0Lr@? zWBIfF&uJ6U{yYedq6r-5a2lgoN-qdGWr{RVyd~3{`t{77VO)kV7{-tcOED}-;yTJa z@pK?PQ3h4TotDZAeoY#GsV=mugNR3t%Zqv2$sd8jDRl$BtQ@(AyUxV!zfJj@Ew3pd z>?c%lr=~Ed@}B+L2j}&TXk~fH_rET-0qA}b1n$?sB!%%h$1WQJl~hD7 z=Z=sH-{5;#Y98U*9%5q4YLIa*vFdSezh8$ecxaEZu*gu9; zM7!Q-zdM;2+y7|kbPr4KK23i5YY48F+(40UNk<#6&mZ8{F%-Ee#MGsd7aa(jANjPLy1 z^7WT%LkxQ;`z1k;F(A^IM3C6V9{cM(zw@IM`%QAPb$zLH%0Vxx-j-KeO50!h1#RI- z_+_Xhj}g2-;G%BRzM)>zR6|ZqAIqzK-){YO6>X3wguYe5WLzRSK(T;I?3?-BmUpI< zMR>x}U5jZ8XSH@)1^usGk8F#Qut+fUwY;<7;IIStXeFg4hiWs+)2!q-v6mR}wUghD zde-?;`S;`WSsdbU%fSZ^#K#~G#b-+UT6$MVIhk@u^NWKN;1isqub1r;Ab}jf?H%C( zJ#vA2Lc_#iwF-&viBoq(S@8_|@ad0NXh+rxQ>Y`05d7RK4`fCr2VOcNcR6x8rB?|L zMYMX9RZ6J7w-Wyw@|GLb4{bd6BnH=O`iHiZ(a_J*w_10q`GkCX<4~cbWC7Qx zr%{%#8jGo=@O|%k*bX8>ps30b@|7awVTG>>#mtwFF(0d*c&vKzvF1~w%o`2&Sv*Is z)l5#UrWa(467zMbNQVdH3LY=$<8%37Q3l$SYKW!P{n^7G-l*Zy+KHiXDMhj}18`Er zIAYMa^oLG8u^?drR3-&UVzlVC7gV0){`&*zGoriVD0;zblMbP`G=WyP)tz6i4FIWRf1{MYyIyWQG;Yx1@ zQtKu4g`a}Q>43)+)c*=9hS+Tky)LwLs=6Q#BnOw*TE!?@w2Uk2xL6TRs-9~SjF=7} z#BlWz3^xT73Mydv<^f^im4RW*GYe}$@w6xbmSizTAU=6#!>%^}tDmk-b%jzsiBWz- zzK(KGQOzrVYdiZ0F{#5XeLfH;r_Dr*&q$o+8G^-dn#XWbcdxQlMSu?=)wZ%NPHBv< zTbmGGKl1NY7|dUtGq*Q*V1&~H+6i8tm#sHu*NcS%(`Z#zhooe%w9Fp!DCAh4tubKf z>*d1g6mkmBasq~nGNHgag$HjBP7Q|8`9corE#D5<-SoL08kz`s+eA8Y1!=f26m%C{ zpoh{^@MXF0K6SP5Rg+k$s8jE=oHw4@zVA70UUuqRP!Ix376?kA7{Jjuy<^D10fV*; zNkFkbuI%WVQVsE5Cp@-tqWvB%L>^_iHjw#)s8>_JL<^DU7|BQsiIW`9<%MtB()Dm7 zzfnaBgmogNC@h6;-&o7ilbx@8`-)l5ljj9aqKLwQlW214H-+`*xNjzjRP9D)= z?@N{Ri?XoadYmH~x7cs->+x!R=&KGwtUBijhKZKHxDMZ@MF=~lP&^GtfhI+Y(8UHr zls(Zh*6`Ki2k%5GCUZCmc$&jGS!7{7=nBTr41@t$*~TY2HyMM4MIkZ`I98w)N>ErB zg}BrT(;?zM9XVT%SUyjx>>2lS*eWHO;|P`{S%zZZDl;HjH3>nx7F0Re^5w4|UikFw zI{L{fs1l<&65|yT=P*&@W23h-dDPN%Vx2~hn>BYz>rnD1oZ%epgk0YU5Aeq{bBu6H zS%P~gkR9$kuZi79`29m1>|B3t!r?{9+LG3ALK`T(J;oG<+o8j$e36gjDVDEKp8fW# zV|DdMaws|rmkFK%1VOVD23uj=%LS+U${w@S{kC_5y&q|7Xr^w*D{cp1X`G^D7T}^( z`ORznOxZ6QH7dRBUBN}aZoJVX%;T1`i36S=^rj8s;DWj!af-&b-O+ghi3k-uNMu586VJD=~$pWS}qvPs1DK!jHzVtLzC&S!Q+O)s(QP zmLwSxumUbqK<6g*v$WvUUn0z zdBKowp3(f5>!~OQy}nI}Ve_#?$wxmo5~s$D#=eOso+`GoXR zAmB7ZN&+R23YN$DR5@Cg^n0S#mREZyg#q<{0k)fGsZeF~U5&12>@Xr8bDjbeE^;C+ zNumUBaon)hkAw=xJX95{3)MnDFDaN^rg-n8k=O#u3qK5B-6r23Yv)iiBZ@dHaU4%z z0yXu;)xVxC#FE{X8NB-jKusBdoQxMNk6t+D{Z-bSIZQxNpsF(m(r=P4awNxK)XN_a z80{C@p!lZJa*$RiFq)-n<6~L(musXaX$vnUr(4P%Ap@_26-bW52%e!S(A-H913n-6 z?ALd;FliKSCSpr^jG_P6o_nC@jf&i(#g^1rS09FU*!xG#pJX5?5YejJ(8B zvrVga20c_kPeFO|4g9@64@@|vyAOBqr^63*FBhuhB^IT2ZED)9!mAqyke<&^abk77uC@`zKf!9hbzy;V?g-%;>M}QZDW2j zRe&zw{%JsC@Ku(|v(ENt)-kNGL;xbqC@jwlxP%gx^g+hqTy44Fd91-JmBIvapalY> zXkOCW-fIq}8$`Xe)>wuvp0to%jx>*E2!+qZQW&)7^idQ(+-O<;!pO((>ZGmEP>x3kl5>o5$^pj|g4?%q z)ZrS*)}E^*1xVG_K;X`&|0LLgrlqi-bFJG7^Ea-86nl>fhrl*zg_S#}OJgd?x zzc^|lPsIXDdP&I676oaz1FtsY*Gqm~{RBg-i~nQG#uYnUkJqY;zAq^(m*WTq<7t_e zIEwCj_mPYK5Mkz0#znPnkkHL!_{uUe=Xq&uBkfo^Q$fE!NDd2F_O~751KvJ9!RM6P z3EA1{=Z?Q34zF6?ZqiR{NixAD#YF#CAa2%qb3tN!eZ=Rb6z0vf>*i<|j9;JH_1~Ir z)VU{>@Hl9EL`j`z6{0N97~^>wtMj7dBCfOl>Uo@ai}oY2gz8DY#m6xYafm}4;&5w# zCN25O^3zLaXU-k3NoUX&mAv1gvQ4hjA!X-yLGrG9CtFv|FRzKJBIzj=h^qEker=mM z;PXk*DL_y%PU1ioWFFuNhFRCw6jFe=Ev0y4P~7h$!K8%n&nHEBeY7%0&4+S^qLqvG z7yrFw=r;M;H~H{-u!M|Dm>@}jq!sv@U_R1m&LPWo`@ZJ?tFM*aIds{1T0}UgzpX zF@2Zt__Xz5Z(n3e0W3jCl0dPj+m{D!Ga;%tX1O$=+kr3VAa7p;g%U}Om3fS#VFly9 zQBti1)t#`++x*_p$4{aCjWZhV;>Z(U?rIJdLp;yY%#PT^iLtoYA3# zg$b~RBkFmq@Z~3hz)i0H3Wq|A?Q4hC^;iM5@;t#SpZi1d8$U} zRA1q$E6@D7pnuq#ep)6d5yNDR!f^Pe-xOgLpkfOp7%p4d@YBo9nH-)B0ueInxdtqLe+-;d)6Eqs#%PmK{cUp7y%EEay)WJC&-q2dVm+Wed|t_>A&;P zr1ExXhBa2a3Df}9DT6_~f#IQsms5vnuPO9%1N1#m(_HocPoergE#VW|WVm7>s{L3Q z->U7I2}ofB#W=cV_ZPB<1CQ4+Oza>J1^&1`GoSz9*-f=`Q{c!tu>uE0x-uF+uaJq7{os!en zHAfP%JG%Y8qI)Q&ug*3)yZyCP%?!Lkx!^Qfc;t=nX;DbM!pDWp`oE2>x;|quqmM)v zPGSzx>GSr`Nt_=O0k4WW2I^Qlu$9JcxsG-z&#@Fv zau`G649}3dpkE6TLKE)=RWz}Fy|-t}7VR6NNhV1cF*M7k*;x3!fag8#;PK>m=6{ys ze&+P-q>?0IsNVP9DT7@%) zz88@>iQ#3ER`A{jHotY}tYt|Mb8WZ}%13hF3A@ zmtRuAL2Mnoke*2iEy^+)80&(o69+cip^f#rkaVFa`y(9Uk|+`aZ}}`km|rthv&v~- zN-S}#<)Y07T3IcV=e=_ANjeg9OVf;_i`z#M9kPMn-+#Ltx6dO;KCiEcRq>JVJOTXO%p>9RFhu%4 z4n^B4CPO=G3fClI(QoLpm&9?Fq8J>*Ilr4B7`wBoq%;d*@72ya@W^*F#vDdoppulo zXp!I)jDs&w52vFQnyx9++`-8C@#*!?~@(fSYI{UR;cc3E!?Wig;t)s6$ zHh3``{(?l26`Ep1S`q=*a`1=^?|xt2)G@}33k}w*hqf8&6wJV(oN&XYr-KmheY_?> z?9tG6okRtHr?p4R-=U6xX8RAvuy{=qjUk8+UVT0qdamVvC4y79Toh>U{|J2t=WoSr zoDg*4^|a1C)BEzy?%GU$7J|knF*Yy8j$h1`C&TcL{IJ9152E9K5iCZMG)D3mLzAqI ziCM26bCZxuu)p;WV)y2OZ>NS}h+Pg^#}D^ImNdYcnR%sAa=uY{NCRn{!g)pDC>&$* zJ;~cgeK4O`vnt=~nH6zPSfi$-2U*hxre|-EH7Vp`Ls%&!MG%4@kUYs!%BJ;?^-%5{ znN%T0dilm7jr*;iPL>z?KB$Rigw^z}v}94W(4raw=isIJCsJ6RWu0Aj#?LLQTJ%MEr0g8G-$k0jh?Kxdlz{2( zZB`%(&axib-rrWUS2$lA3eR8+OVSizc=*zN&+yQxek`2UI<0RXc2{rhcCa=o3Y*90 z5HN|uBo2@YLCigVYT{LJtbA?5tp)pxH)=Nxx7M0`a&=oSOz1xsPB{XR4%92KOmB`m z?;lFyF-jnLT%d)@4Nf@<`>ep>GX6U(VPz@Jtw!FLv@ujA=tTG_f&<>R5z4z!ZuGks^LyT-5%f4?E@CHiYMh zB%C58hQ$CYV3GVL&M0*nZG?5Asdx7~SSy+hrZEm9Ig%rAlF>yTU-?lW-H0?p8f6{Z z{n`5UXV~?WeBtXQmd9vZq$C+5VH3ENsC)CEwMx4k=R9v{mw=(vhZx%7^9XLQlH+j+ zK4*@5NySdP3+peooEe-(KbYgKPnM}pgUM(^8jd3{mSPDSXE_qrU0d5ygHwGK6RZas zKJnbkd+LP8H~|OIERM+>BQpxYNWQTh9}R_aCPSnM{Jf-KY%$+Qqnt@rM^^RRE=Thh zSmlt?caCCkn!<5Hr1nhM@KfsPOXVa;T3lSUd97UyqI_P|@^97e;}C}%A$o{I9O4j% zTLl`HFr9FtAReN+WWZR&brNg;8s1{_Vl3tNZ&W*s|4=*P5QjL#Ar5gU3BKBBm|}hQ zXjAcWqNY_|8y&Dqq-g@D0FSdWc-_|F!7F65L|EWa~(21LFy)IRtgJuw2|T=4BQ zYe!3~qr2*9c66b1KS+)bm>=}@2$Iu1w2f!z2*1sR_t@GtpZ=A8I6oXva?I;=ruC&n zr);@LlL?0~Q7-ctMWohdl|1*%hfZxBSB4T7FO zr#}Cv*n4n}b;PyI5AOt;mEo2`O9eP^p7BTPk+S;sFil_CYFGxWiu(wZ8W8_iK-|$L z8iUjPwPXzae_{lxH4%BSw!qqQ!LAP5jdt`J^4}auoIr6T&GCXbv16k@&K6P>8lwI< zUpv~tU1)u{My1o&u7?c@E=UrEOT5A|f*f)mK9Lk%YA}S(qw&NdYo*2qCbc;mZDc48 zNQ{6n6w6>bLybF4kw#;w^_h%s<`FHzTxgL5LtqR~6C}ZMI$fvcYDmH)WH6(bt#uBJ zyHNfOq%{a8i3}k!gv^T+Yz^{2nb8;5_j0mR{bJSD;43uZa^UuM`JaJK*LpFKzR@Z0zqbE&t5@Q%s=NV0a!CkWi&AaY zQqApW8N_?L5pihB{TR6VbEr3fEuRS9kW>2v(s8>3DSTJG@vT6rS#tE)9U5tCrIvI! zoKA82?3RKSP=0D+!P>}M`&Mg^tCM@6_q#y(NG1w!>~>hVS`W&@L093}q4o%Q8*mnTz`wiajX$agGZ62lxZ}j-rDdu?p+s0qpFBiUF^3iD1 zrqwVH70Hyt5+M&8@D_JGB8^WPV3yt?Ox^)z-mn-?(-UTBV%82#(|_mf=C4X;~GHShZ{X}m>GDf8=UR-}@ z^7h0YH+YFz9Oczn_gPrkDSZ~2TmReeD#Q(v;AKE(jpwrs1$XG z>o@OP*6+@pOkQgb8-rVvd4P$cOmnmX$KY;@1rS4TH7_nR1m*QrzH4nb{J8cmhZ)6dcr) zst1GXYJXo%3C`=Q-fHcR@7UPqO^ra^8Oowu5fzrDNgn433T1@5#~7R%459OpjK>q> z`F|vDG<;&s+VSaAm-2Pm7l(BCAybru^8_Z-3c2@fZrRE`$MZ>(I0VD0&hat$XNl-f zt^By`Q+qaQ^1`T`FS~PmK=s4gB`+Sj1ls#8dF8j}RjInW!EXuL4%w3&oc2}I^(slY zM%_W0MC_ zI^DoK|C8g3%Wil+F$H44%M^1;KeJBma-^&wv67wwDC{DiBH06LxMtQuad-NvO^H7C z)aTZ-pMEeqcPx_o!AXuIS(@f>jAJkz_j6wLbubLcjBK+GG7y)4d7&YixG2oPVl2mS zKp;ti=bJl8VrbT!)$KyCTL$EOer287bz$B6<{~d97{5a{z)1w72=K+kZuNq^Q$|NLn)O_`ZcDVcNEDA#6o#=RCF+C!M6B%_ zYq@6Sd)x;!N#N8B{m!5WJa8A>$L9fp>mIk?@3xfV0d2f~H_M$Hx91Kg+*f`^Tm>!; zafm}4;&2N?y_>xLC)v`ecPgUXSp8p?baeOR_>zo69O4j>nK@`(hUdQ4u9+s-*cnpv zM^k|1DG3NHLEdq_`?^7W>gn7&-fB?(KQ`x;Z_s#Fj!=K)n`{r54BuIM5>4Kr=b-f! zIE-USp5QTz<``DzHD>U(03W{wp%(g`_0T+K^74B^R8YuU50G$ylogr=G%Viv@m!#S z@7^^c_VHos=sQ37Qibj39AR)UKy0)#-J7&zkxv+6|et zS#UZSs#1HdL8NP{v5*Fq2<72GkFb?$eJEcS%EJ*qszZ5LAq;MxIYPc_1iVlwLcU^z ze1!;jHj;uB?367ne=d$$M`VuMx$!gYteR>Fbyjg0P7neRC0u5I{&~QkEo>hqX>}w< z7+aWb*8IF{O2R|5QxbX}dBYsHHhA$YR_~BzX_SSwG$L_U;uyaoicDs7NZq_<^P6QF zMi}%+P_dZ*7qwmu%Eb7*q@YZQ%@QhFsL^o3y0YR^6F%<}MrjTA8y--U-{BR`lQ2qK zqJ}VBg|rR$&FWl+Pi(tf+wlhyb!#BVvl>INl!6fohBF<%_U`(`Q6X`T`stt&8XwJ& zWSQD%Ol{J*qzH$7jQ38ORm?EN|EtvWD`W7+ssUlgGwT1SsD~d#V1PFuHvCV}^OK10 zpGAzzS#=exOA9{^rU=jbAl}og0yOIjM^bS6o2#h|E%+~rNgU~jxVU5?4kb!_|86Dc zE8W1*t^M6Czv^JEl0p*QQ{u*>t?T7LYL-~PF{uG1_ zL?~Ym$|pkk1Sr1)!mb;k{F6}rekk7^%2$K(7ZSC(IA{H4$%XM{Cp6ZtI&CGW3do!z zFDL@=bQB~(2HKK<*IWhU0>5B=?XJ_-rxYY-#w#Q%vYg0D3Jc$bKcW8E7v+AsVEu8+ zyJZ*7rCu^aB47w&NqULJ7FHOAgtUU*2OqlD#U$P#x)R+jiN5L>rqQcWGhT)t7Qr}%QbZsGq$sDDI zMM=G9E@6!hl$5nkDILuHqGp(`Sn;us+Lt_12`SXd)5juk9F}ISR8P6RMikC8*`BH1 z;gtsUHRHw62u$J3_5qKBWC(`fM1LR?I>9jOx}RZgjF5K33Z=9k@}XRW=h22*j&<23 z7O`B^dNA3B%6mwmcEBn{H7PPg2}ErZbxis66=7?NdJ zVaRNL|AcRhWltDKdvMFnQA~#>q7r z5^cZopP#7mz9zV6gbHuu0uCn6{k<#^G!EYLWzM?Tcw5<85g74EU@GSK(UxD@Vjh^j^-u5 z+kSWjQN~zTP`d5RoFyW$U6UeD2q{IzMUtfiULX}rxqgNJ;3w;$j0{LiVMX=?dbR~hrqLw%cF6hk1X{Of9^Z9-P9XQ^EG= z=ht38Lxm5>K#>f_D>#kwI5W5+x!F=EAj7;U=b?h_i@8&0&i*v44H{2!yeuf3EN~2L zgSJ-XpS5tK_aUxG3MRg?Z4h7Q?KM&({bq!$7zo)^gx4=cRz`S%`|eX$ zr~J*9*Y@ZgM+52YF)zU?wi8t^e)-35;jCv_0+T6H0W1Yr*n0M}cTz*9`kJcRR!_~| zG@yS2^n|)_oOZYb_t0$6Hpjoto$jH28-!fofun64G)3D(o9*J1l@;cXMMpy836>#f zp2IkrWN6(Dz;j&a{a#Q-HCx7GpUv#OG`vY6PK%hth%AfYENoI}ir9_nwhP8b&rbMN zGkh7HfLbyb>j+>8f6O(XAUd<1zA+9d$CW$62l_i@;0AKn$86s{ys=B!;fVU1L^x-f zAn48qGJ@$u)bm)u9_4$-`M|I=QByk={CvG?g|J1vnb3k)u7|7>-c!@>GlJ8R5!y9V zMV7nOj#q_5b5B6;f1{#~wh=B~;Qu3eci5d0-FQX&UaWkNc3<%x;*i{kc>dZ=ei5Oq zW7K@TkQkimdEgp0EbDZuKhMTh%{QJ zFOrB)%h#|~zfiMM-xcA`2tr{v3ZpSu1UP&~%#f2qrg|h-%eLU5Da7Wkb@l9Kp>Wu= zOkgsLGrTCH2d6J0MM2*vDIlYvw#{d*nU#7JN!St$E)q0J;;cf$gl)937z@Uvj%}H@ z=7OVTHG=C1nk{uOEX~U}#!5KACp_)jwDQ0#q$jhcQ-kw^9t+rVJYXE|B<0B{w;bQ*hR3EL(CcBmist|x&_||4`}1(4SGF4=ftl) z+9$wb;J=fC)3>@L1nn#B`7G#0WftN-yr{=F{M~>U^C%c#xU~Q+c5Pw1I?A5bdRG*3 zaln#5^qY-Jgp5)oDysc(LMgu=t6a0R4Jo6RgBnO%#rG@sc=+c!;iZFtkaoDj;xtDA zl9!boLkd_~{fP6IVD@sN;5fFKLGJ_iXzoR9>827>Hxx^lBCH#pT| z+0~T%+t}$tqpKc8s%32Jw&GRW#MT8)KX_a1sw>|5q%WG?<#>wYaE2sV3d4Dw*7F^% zz^~=yP$O`M4Q!a!DRC&e3YNxcnq@GGqA8lh;3~>MslJMqwl2;C&1;^njUHqd>yOWx z4J2QW5x(3JzMg_F%iCyF^HV1_RLL)CZwVogf|j+iebl7N!{1id5cO0vqE-OTh$O%X zoCDub^ZCo&ca(=EH2o??xVWoSLKlFQO80#YzBk79yrcjwAAQocw)J^)^{Ek`M-d!A z;6%)?#UZj3d2HayVUKM6I0JTUiIue!slk!HD{8qKy!?YHC@pR|k;(D9Y}cMyxPQ>Y zHT77!kW+G;lra)#d7cOa2dHwiF6obJt0CyxJ(7b`7_{dTg+#;-{h#ev<){9>B87Lz z&PXgz3lc$c5>a{YUnZdtllDo5WIMcRLIQMLi9pMB?oFs?>PjjNk=>$n{m_n!en!4Kl z^c{0P+fKWJ`K(Y)z(EjLAo|3#gM+;?d|<|YbU1k&BXNS`aFSy&M&}2LI2B0eW}s^5Z=18B z+M%!O*9mtP(llUrz_A=bawwhTUy2}E2tJkv+FCn&uV$}AiYsx2qbXLT8H~eWapmKV zfX0b_d*3+eG)3TJx)1t1&@jGGjIi*Z*jyvxk9FrJ^mk1t6 ze6M#~bt@e3e!sD3)16?DZLT4pX@M~`LTHC^5=M)H2xJ@<+HFh-UAAm%B3*iY@H2JM zB?r`2JifcZ7zd35L1ty1mjni`qN4#-#rc{q)mqd!jKj8qW1r+OOkfxu2aK4R`(dqt zU5*%2ipUC~)%XC$Z5 zoaTK7{e2|79uX_xI0G<%v6N1aXm3KKI&y47sBt5E)UcsR78lMEj;y@Xx@(T4Gs8nN zUAeZkVzVa63+q68Bjgo{Ovo(9F^s~J+_d>WelTz6M=ACgabI!wps|$qv@vQT{Q=u@ z=bzmZ9i<@^5l$snfkgAGtXcS5Afz>j|AOwU0qzc zIUXOI{AZcwzmNa6oF-p6tS%YNLq-c#!=N={6ndyF>zW#z*V2h4&fkr;ExkW2V7B58BG^ znALRhuV_ODhNdWjq#2IGc#5InD&}*cW#R=@JY=(Of35BZe`^D@CFB5+GEfAD02Ga3 zyS^J>s32A_Rf>|#{-?+#wA_EZt$HW&f4{E^CwyXfngYBcvI-%PL67 z%~Hwp{JHKSP;8;qr{EYN;<8N8y481SF^ZL)V%t#>Uy}Hu4K3St5e(;${Kl=F1#ed8 zNBJ*LUVhZ5G4vU=1E<&}RSiW;wdZh^Q*5i2Hf(iethQ2SCWPdjafWAc3>R@0oSvTf z=77agSvNE@P@fDHq7?JO&RCj43_)o#XmJLxXz<-l(hiLPd`X?iwZ`Ciy$vzXGK4*0 zr$W_l5rTM%ZS|JFzjv$(Z%84M93ue|rvOJh+B_9%hcoMPgK3xK&&y#yY;J{n_azXD!>s_Z>Kn!Bi(7-< zOI7jxp!g8|tLgAT#cWSWJx@9I-Xh73fsNezt?sXxC#Tx**{O?ZYyNG!i65IZi3XY*(alGC|}0 zPum40MA&DkQLfkG&S$jyFg42mI~5bJloWj&shorxybf+PJ2X>-{SS|)F~WRl`ShEt zpYis@2Bmecs^PQ7CA5z^-QxQ%)^WaB&b3?3p4}MZ`izxMiglg-Q~NIx{gzrn`x1u4 zTVx;JEc?p@@im>GG}`v1MY)dam+(F_-sJndwB!_V`tqMSAO9)6%(P&#vu-ID1H3jIDYQl6!Tb8&*Sj#q>JKo zPwBR+Ye4ef80NF9O4j%IK&|iafm}4;t+?@0+d}; z?LxB1P_jdlVW#cc{rla{$>GC))4U`BK@aG*7N)1gq9(~FBilg zB}EAUz{nhf6FX-w;)mVqP0^Gme^IqG?=VHjY#9wt+l;5UA%B+%8}gK*2teSBAo92j zhdg~8DmqfWpz7JSw>Eb?zhj*?eyK*?IJyXyaSV6ojBR+Qp2kpXm}7VjFoMir6p_5^UaI!s*{Rye^WBut zM-KelY7A~e1?c?^b%H1Do$A5)JIe>ZFG2dVEg?B#7G$qc70Q#4{_ro59YQvg-v@Qm z1?8WG)?6Bt{~hY@2WtAFZSX6Ln{1n^nR6EkqpnVX)9v&*cwO}Ly~f}aeble89s6^o zVLjhmeKJ6QN(2sFoN@F4F2U_{O5Jm0&{=TH*}&le-kj`_U}YEZ31<5+3ATe7(l03} zjTQ>px6-yl_@ULX_v%6vTA}uha65C{4iE4NLmd(=MVT=ik9HkqH{ z!Ii-I$c=f`wqwWYKJTtGx@LE9Iek9haY#9CA8`BnjB$xM*>l^!Qa0Jz zGgT|KA(YxD1*fqO*=x4N4}7q-bBEjY)LTJFcM@|91#aMRN^PBP+36nI(*wM~ErCA1 z91rOAZmVuH@4cF!&04H>I0cP)$7bWkufJYTJ8RE`XbjHsj*&w42ynO4Egx-v->hz} zj;3l@N88F;Z(LAS^&zsj4uZ;s9}iSh?FiSlP9NyK=hQzgsCf-_X+KWCaApFf7Fq7)ug7tFK^z zdcc2z9Ip>;zYN~Lcl1RostS%_d5p(tiewpvCg2*l4FRk;BL!8%R@;#F%d6hD(5lC~ z9zULOx*Z;t!FidIfIvyokJtK6ER24yR1KZ#Yx>AGc*obrCe5g$J~?0v!D$gRBaxJ> zP@Kp+WZ?CAa>i7u)9d)2HT6@p8SgqcAt>G3AsF>*&o!6}9l2I$xc8n2-Z0mo!FnA+ z33owx4$7Z|TpIig<@Z7P?NI(bD8CWPuZHq7pnQKQk3;zvAp9^F%6p-_1m#;p`T9^k z0m?rGoy_+ewJrLoZT2(24&2pUI}rY7L>)2|?&v+br{MAW&zLO1(=J=^dMo~3h3RtS zbV@x0-hoY{nj-RAigIPsw1nu-RA`T-Lc97#fmZLQwjZoMdB-^ofvqSD(_D(%!BBw2 zWJVEj7Ju+^(_YV?+f`cnI z4?&U6nF_baxL5d(uG<^k?+T^7-Xfht<)SRFs5T2mL^;|yr!jUJP-=i=KS_{4CenPO z-jDQM9IwTpB==i^uw}b#)VxRfj<2oBlCumUge?+9;v^?gI8BL~u*D%ypVMednOf)4 zw9Ntw*!RZB{Gl%=zqGBbH|*WIOKNHop%fka@4|ULkKu3QyT|SH^#xwv#bx53W$w=? zd$E)T*K&i9uhDpG5Iz5tVL=iCNtaZ)8n}yD9xNSl#rS>3O)}4Q{v|>Y6x?x(2vgV+ zDyK6vz8gwe*iu{}Kx&LajA)73Rw`9iUCmd5#`!rlou{UnHo~21dKdJ*$r!wESwuff z5YbuGewd;hk7(g$b4*!4wCu2r-u-=tZAyFf3G#xTrYJ|(oKe67JdROLUsk8U^=^Cr zrM=u^lpA;g(rcMjc%V>#&C}P|)A8!G6zx>~B7*Fz5k$3UzwcP^(D2Sns+~{F|8JMA z17ZH_vsn?8&lIOyc90~`34%Zq1SU>++P7)tfmui`pOetSxii8B)80QBqetv@#^Sw? zrtiMCbvh$-obyvOBO`INLJOQsQUn`mWd4ATj0O=}tiAt7j7Lpz-$#$kKHG`OT^7$< zn16Mgls_^BH!lK^2!+5Ip5ghQ(_8gv|KNtQOXJMXzk*XMDHsc-{}gc*hkpl*zQx;L zKujF5O=$7w$A5lrD|DdiP%KUhlqg9UgDW&_q>zj(`e$2*pFg~9czf+hA>BAxa^Tp5 zC*OL`uObLjwD+_Kq>rZ;w{ySR7M8tG|Ibz7w01~Y#6^Z=a0#bit)04vRA18x+rZ}a zz5#X7N=_KX(>TNW&nb#zFjxg?YPs;8F&TcdrN2LJ?6kY=`r#~WPeKbUE-E6%)8x$O zKX`WDAKnaYil!71&7mv`ezQH@h#qm*N3}HRnK2~c;u3NlK;T6Q=K;--OzOL#F(X6Qbl+3wFQ-TkXsE{bRn@7DH|HV&G^2kSvB#9H+c)>+s+evRPtHm@O}1hfI;m zi?zN=P#P;#GDJrNn+z9iscRP3bklY{YAG~{49U?FhH)$}NHA_`sd8cG5tDb(mhklT zqrLvNLn4vj8gM^mp(v4K0WDDOU<4dT%fj5YNiubEz;8t;KKk61E@c*mrO@`>8l2*C9)ypqwflNsq7m0>{Cv=F* zO5zZQBIChGc&W6t0!&53SLNbU7!u0I`yC&TIK&|iMS?C6pFRk|KaGOYL`aTchVrn@ zdpRggi;|vAfZmJ5N0<4snRXtpl*`THz~@!n~N#kYqPjYBn?_% z+^<{N!nazfs*S_Lso_%?c^VCA_D9>*`^vE^f>wnh(1Og9BG1EG72S-eqBHCtJeRuY zp&{B8Pa*e51tu#jg;NAA0fG|=&1A(s>Vx^jnpFwfaG~zK>|usF__nFM{qyDIYRj)k zw<|bJl01)dIKh%Q!NB*D-I+l8z6sUYO7@q!cO6PK)W$Ew1$HR+O2Fmx^%Oil@2)|% zl{M3QC1}UZ9%K09qM+s~c52~ux${|zeg-{Kc8(i3JRB>DB*CyO!wB&F^-o9$S+L1q zwO1)`Z{^ycT>uZzEsZ=6UIfV=cuZ*Gce#YJFe>WVZne+pxMxw{&Dsvh4Yh(~AaER` z@D!=A1x8Qs8ma}Q21K5N)(|}I@NIb6{CDm4!NLG>_?0yMJjjxRU~zclJ5@0tM!uGP zPQzbXAA43SVghn8QKX}*Qvp7w3-ks2Sa<(kwUgJV$)-9&X|%vW`WmJX z{}%>r{%hG+wcPZGxG;>9N}3^P9_LvG$7oikW0b3|$z{k@-_+iC_N+mhKGAT%PW=!m zZih^e63fsEF0n+55caa`Coey<2Z_}Pn!pH(<`|r1 z30&t2wA>qk^2#7anXorqr@T`+IlNzBoR$ej1iVDBEWE^LC7|Yyw2vD4ce{&3SkJ(+ zAo3zkNFS8AbAv zXf%FmbjZtSVC{d}m%O>)Vv2tJe6jSkM`{}XWN1jHb0G=Q?0%KSX=O{vd zZ@pL-%;a@Bj4N? zmO3Ia2BSntBsh+M=k(jdsG>XBZ!^|h`Nu=`!WAvZLaxt2$`}DSoD^9=I-IWDY`B)D zHOnDRl1%#;Q1d{88MO?-7^LvZA#uVX`Q2~z1`6=_{V00peEyc#`0>3dnu%4{J44TR zSIrHjy1+C;pGc`8#`kU%XakpNU)%fH4}aYqUXzVtaUcr}PLeDIyEDv)xI=o_H?I2d z<3mfs>yrVJ=KU_$C7cj{{%Z``;GNX8a)>%$+C<|HR;oT&?}7f$sT@dIH65-lj>JmQ z?($ytUJq~HAm{7Vjz+6e5*WsjG{ee3WJk1HHkj%0bu#SlI5WcZWJWoSnK4d#W(h5y zu_Y`cD{8%HJ==kHmp5_RiyP2BGzo&mSb`#17AJU;gK_5h+`zAFBu6>O{?*x!scQFW z>v~enuig7qwWkL) zuB4}4f>nGhl9Zls?JWrE2hA}utawI%s;}OpN z3J!_E_GfGC`LRl77{8}DPf-MklYXk5)e(~GRUv&vge+jNedydnwYcO6zMv@z_66NI zE&ON|jFMn4KY8fv&MP%c^j8QHiDU^!Zl0@PUx#eqr|-NeZAU(^_sd5Tt0APmFebdi z7+kT%7~OXk#P`~WY5&((mxrR=dZ*cEi>|Wm_g*;ok;N3A9>)oa zl0{CEFiL^xak`Ny0z&lIuRgz|YF1`=6C@62BvRpIlHdU>8E6_c@*QDcGv)2C2Pahx zA^mWJQGUq>&P$@-O*c+A*wJsCU)Zl<*!k*@hEbCcqwVsmm-qjktZlv(iD7RVa7+?u zlHn+U1owP(<;=AAhnIOeezOx6n*2-panDWW3A%B7X~iKbB=?{_F{6C5M&)aVJtG1V zHG<>?j>9olmRa~LYIYhD*%_Rd)q-0S6A|OtknX^^i`9iuT<9u;;V28i*Sny6ip(PAK0S zdY^&vaO?xv+aRoOj=Fr(AQ{1#8=$>pG)%USt8?-6qJ1^=2q0VbIBraK%oRCOb1TrV{iol>WCqj|$q4H?HZKnPGwZs0Hk&w@fCx=2fET?b;#{rC2 znDxK6v^4AWsA2Q}**os|D2nb~IwT?VawUWgg0j6Jp!D7nq~oz=HaWNw?k+$C0a1}A zilU%^y-<{*f}kK+;S&)=Ktu%*M8F1C?BDNlyGd^D=1LNp?)&H6?B34ozL|ON&6_vx zsZZRePT&J-+K^{dPPAY^06ze|%W#!W#i|YE`|MyHIj2(qH-@r%EJ}*JY}1l0p{$I9 zpz>fX>7H7iq;l`+PyhhfG4ic#!Q~Yy9szm24pbTCIdu_myXw)YeMV;@0YSY(vmtDdI#+RUZ_&R&9rl^1rvBetkoI?lcttu}(_xvS|CQ z!S_c)yCNuo@+5*0UF>muliL41)V;KN>Bb7RThaF7Z3liHZZ}$cFns-efM9kO+ma)ZPdMY*0i#wJF8hh)5@Kjxg$U&8?s;>uDqzB>b>tZ_q z{kh8S2zAeOR2BJ`Yusv)$+6ne>dVJt*pX#fj*u8Zp@4vm`1_g{x;8$a057-OMOmt} zi*YH_F3u%SyDGBnwutk3RY8jglOT9UcB!e`Q>*y3IhulXIBcd`oCcgA15uE8sw;Eq zBS)8)Q`jqHf@|6K20+&bT;}&0aFsB~vW?i~rnbuXSDGwCZ^RNj$Iv{>F&It}6b~;g z?o(y<62$6dx#^G8w;ylv1==f%B^ZB*Ig(*`o+m>bGv8N*c_XAF^a_)&@y8D&=W4rl zb3$#+P6o39m(No?SdbU}2)J(eW~!*)K-gFdb1t=T< zG$rD&G3ou9(d@a8o7}}0KD%*DX&AK7Hk=e;o9y;X@D>UZ2)Oln2PvOyur7XJ`Xh#V z2s2qqjMw+8cppT9Si{^@R8K1-!4syH9b3*ia#(X=>77_+?DBa8$)BiQpf~_z9Ft^* zCg+?v`tlp4E<%%%!sF8ZO$^q&8bX1?;J99uEvM<_xXL>HFHhNMc*>L_od0#l3mWeU za|&GUFj;`aS28@&(F!ZW50w%ngHi?qG1apBlB#s}Q!`?$P?K=OIJ$TE-z zd)$S<9vRwJ#S$8a^LXSx)+lDf@+gPR<+_DTK5v5<^tNYz}P~WI^SX zklWTZmf&)0s5}I6RV{!{!9EJfw}H_99R`&xP`PhC$ovkOCv9qZ&X``=&bieLO_Q|Y zyULW-X>3F3?Z|$cvM5m!CE*;;O0eJN`r6@#_A1lR39l?@eoP}(sJpcjXL+05+sEZ2 ziT-vU@Cc5a`t>{YSkpg4n`g6Bd>b@?W+55mYSaD&DP3lF)_i-yY_>aT3bQ%<9&UL) zkKL8uU-0_+xMVP8kkUbyi6m4|)Mz$CDoVM7x{4fqI=pt5f_R>)B(I5f zeykmY)1nBYSg*tNM!T-VrIEl!!x~d(kdyY}u81e)D4wH4Tw)l34Ds1cLR!V1GktR7 z=sGtRM&>ILNsux|3Y1KRWPVWY3SakgCerffl)Ro=%`?NV@OX|=1R3M}?>+dXlupYh zRM%t-l^BfYA_(iPP$-4MB!YWw!(D@vj+2r#yTB^!N}I`K8qKChTk+txev;NLlr=_TB}B6wU8IRS7Oj{W2qzh~6D;}TJ4K4!(Z zf>?l{>@RWFgL*!ag)P{*Xq%~9+eLY6duy(Shr=jX6hqhquNQcHKvqv}^#h>Ldy`8B zirod|=N1f9>Zhiq!>3#SR`I70hT&KBowTD_JoNeZLF78GmbEmK@c$O*Gm0IiTTk86>ekVSh%z(->p7cKQEdHn z)BQca_uQ8iL$m>gWNDlw1X0AoM;oY))J@(r)qVECXPY0ggqk4A4q?(%iA^T3-v{JB z``o&@eBh({2prnlJEk@_wOIY@`o?Jcm6N1#o8kad?4kqo_4b$qT)s2a=HKz#<`YPN zHqD#62ZZB=XwQ{>$WGJ$&M=!qr3qTWEi=S%oWf8FOB19h!##tx_5IH@qt8>zAKV&4 z|4TB#)3QJj5{HNF|KUiD@qN?in@2o+;bt5nWyrZOoU?T+0MZ2D=_yEp4E)D(72uN! z@`|0J+u^M*j=#3N!+c7N>t*XCF;SvK45tCXY+7=*?ZEWe zrOtvM#+h`fTn=25{CPVpQ;B@6j0CJC0hv^AjDmwo+l#I*ZF^YbYWB2PwQjQ@Zq%?qSJ z{@gTo{z36$a#y3}J znD}J9pDzhqu55FA)zdWE;W!%da?n6cB3koJ!Ab4TVfEy61dRY@+n3^Z{oR%?1bC(O} zz(&z{V1J;i!z~U3lNZ##w{uP>GF|gtyRBJp?JX}7VN8gxNHm-@EqHI@Ei>PVay*0Q zIDcXhjKP&ykA9pq&F?hk)-fMy4t|5tv5Y7{5FEuT3MNyH-)#Hzl9yjkh9fx5zepbC z(dvcoSM)swLAbI2*MsuYK54q6>+z3gkJOy#3d%aTEE*+75G>D7xJbyd7LBrYlObmg zJl`T!+bo*?S}^W5BKQTSL$Oj@LoWo=?WWRuUz(V=+YH)$4LYt3gEJ(Hu?&gxIL3u! zFu7-Qpz>NsH=eId*L?EJlA?7HlB6)4Kj9e8F${q-A)Z37s1pPr-Dkfy{V{Uqy2G`! zg8_#PVaI#P!2kxA0RRFii8wiZeCJv8$0MpkCpSxDL!ho#71XWd|Ir5o&YNiGn=ki$ zCt{XBjt~T%C29Ww8#qf~wQd2doo89nB)A;f2|t>XMsG&G_+0Wg5$XaT1= ziF$qPkyEzaH>7HA5=Cvpe|()vMv6z(TTu{eR?Ty0H}(srW5?cZKI)gqr|dA2P$XWF zSyGTbod~*jJiRm#j)l}sZ9uyY->+;;rm|7+yXkt@kex~WnxUs+`7>S^i7_$@7>ZYf z{>_^0x&IzA1(lKq7E1|;*|9dt%WA&R@Ob(Xal*+GrM~}Fk4%&V!=I+l#&J`({GpA& za~Yxf3eIt=d;ielLLlb?rCUjy|EP{qQ0haJv<~o{<#!KKwn= zt~Y{kG){V@j%N+lCxm~Wj3k2&aX_^gRXz>BSSxs*+^X^sF-kproab=h@p>4+7y3u( z{}t)+eP6WeA=*@o^JsCDdbHp#;}8y4mn&v${}?d^m7!oZhTc$FhQ1g6?r7i38e$s{ zl`T_I?l+9`eyH)2QOW+Tf06sWVH7%46|k3ieCu#^0nFbQ$7z$34Y93$|1>SW@N54z zFKI!)ri3$K5E76TK?DRL0-M+2p1h;~vDV)-o|dUGv;D@{u)z?%9LsvZZyR`IFIG7hdbo%mGWongaxe=Ln2s zMI5ldV@rm1es`@=ll|k>XgeB)A>Qk;FtPCVsbn+Re9^#rPMIL_q@^4pnV?;QeBirv zgt*F|%n^2==MsW(bQ49YdF7ACcXT|YwY@Sd9c3U394B#{py1!Np=XVJ@3k6Q8s^&} zQ_??q0X*=o;QQL}Xr%`KzIAB$>tlvijrk?PkZyjW#-a0LCTX^k9F!5x~y)*L7BJV%HeEzsm= ztvau}IW@gBb=bv*!g!fE;(CHXhf4`*7pYz-0;i#RCF?_ti`gpfYYbM>XG8yk*H$y2 z@@DAz48)>P1Eva23`-C=&M|z*wrR2}FwXWO(}C1C@7uNUiz%gA%ABxT3MH|OC}9-N z($tqZp@HTHu@vnGfdfopID!Ne5ceM1mbc{ZHJNZC*S=8* z(6@@_O5aQLTph&q+LxF&)QNFTXf!l7Z?E>%Qu{egMQt5nk69#Wfd?WS5W4d$)g}@~Pmt+TCn+WpluBnHn_iVUkS(esD+rKh`Fq$=8 zXC8jvx^_2R7b(~~&T=v*OEOEy3LNZx=fLndv?a`J*)k}lU>!QNK2MPhPtr8YlPn+N z_A>TeHHs@z3en~z9k1KKJZ6E^y`XJng16aS4!aBFN&^KafL+L@%|^wH@d9e89=-QpoxrZ47=P{CEIf7#-j)8k)i0ZXb1L=fxgZT)*j$0GbpWQU<$nRrzsiostw4?rMZ!5*mSw=R z03IE`s4kxUeSyB}9(8o&nzwPU?_RhqiYF#RU<%IAKp;W_2=@6CRrgS{)Ai98U4M?n zEehua8skYp5@=;h->L2QF1x?fEox;IpP*WKkxpb)v|ji*w1^sGp3-mSPaB6t?k!QE zIac5lQV>bFw>~$b7AL@L$!1-q$(p;vz2QfKfZ-U46CBUTHql+=l7T$jPiof>*T{BPGe)j;@Ncp@}K5T{2%#TKx@{QR5UUTXdpavqV4!poA#u{1%!^T@64 zO?QoXX_s%lr?&C&X8BsrY)%MAJ&xybPUZ-SWr^KgKmN8bcT;8_RA;P%2;N;DALDb| zDrhsTAESE%?2ZFD=o)H4%6~n4~yt&6wyY)lW7iQJP*?N_JT9(|J zDqe(mUKev6Qe2~;=6fbMbZSGJl|LhNA zO6}ia+ah@dNSs9TI3*pv|Es?`j_;L>T2Lm&IqPE`74e=EK+Bp!^YK;8AxBDN`mzF` z36fSMh7(gh7=1_irAf1xttJ!A-rdag?3+JkIMmq$n$Mqn&K5Hv%R3>~u3x!M@n++;LNG0&NFaM7}L zk%^82f@1;3VzeOLnVYh*yrwzB7MPeP24qJeda=3i{Qa}%^e00na+Ju+`M}rB?ec*s zz9PZl_u2hmCQWYd<4Q)&=KWm*Bv{idwh@tNxW)XdWzRJu@78?o&xHx|Pm+`kcsy>8 znlP<67YH)7cFdFZ7tW=RLNrm$STz&v-vSjinRe1<%VmE5CQ%sRCiOzPisweDZ+49J zRyD+Xz($)Yx0vracEcwdGc@~N4%*Nwh(HhooDex)d9hKCseh8!Cc#W%Q5BE>X!XV0 zp4L4NRdB8KP-1M$hg;0~!}pHge1F7KM6|%-JSk%|FNd5WGRM7jaH}~9tUR(MrFN)v zQ9IJ^vioc}MZw!_s}sW;h05xlZmwOoUfQ>(BiJ@c3L`21_8LWDOo-_E;P-&u`t`3Y zVg)qAd}{6D;oEC#EdEAg7?T_pkR49ZpM|BTL&*2qBoTXKzp-s&${SlCOD>x$DFGdyT50DV{Tp;p{5B$g>>F1434`BlD)ouP-B>Uz4b59*;oS zfGGEdvLwTM(QTlzC5pxhyI94WWgT8T!gk7}N|ucmscbYXGIzc6k&cHtXsS;bhn4b9 zOfrel0>LwaEE2Q+8MXaH!=o9Q5K#!kcsX0a>2NqH+W5QA3?@4p)?1)S`t@N=516QM zG%n$SL~}*Q=Co+IyJLoSsyjU)7*`d zd$`X=UsF0ARzikqhx&<=lY()mO=X3dV)*aw+^ofWX&p{OXK0&06@9?1Z(xxVcvOU++6|c&jN2Q7Kdg6@Uecy}t}z&=Kr1c{YHK+cFOtCM4Ov3r1txq3 zKL^2XR+xA0+1|0$waw6GHMcn423$Um->HM5NQz<s=5GDhs>G9!R63l{-}8(lf5c+7}^67 zXBnQxXdI(iHq2p=Kj#I0wLtoQJZ^Ts!CpUUeZ+B@4?xnu>e%ax4 z4b-+=w-`UTQ6;aVg~NJtm;LS?_gOS6MC$~Eo=9MXt-qstvCC#>sQzwu;a81^)E
GvAr1Th-CVj6+gVjrBrWJdT*0?{;^MYkL=zEY1;K4 zUt~#{B{7MnL**F@R9jWltK5sT4fV2l!0WZUU4uRDe2?IS z)m*GWKzkWpFL=XlO>*SdNHAt*o_F+B@AivfI`9J1wEz|dUl`R)C+6$ zMzS``Z*E*QEB|fgZg2J(wdq4sDACI=XmA#AG)3|>K`cI!%XMB|EkUFDHBp7`8)jdx z{Zlvph8DU!!SXbTGZ;s+6drOKVt$QiO}uH|RCD+u9~H^a{|wZ{aa&P+TmC7B8Xf-v7)$-y?im@PO9Mg(1VyN#V9ak9!hu2`&i?7JLQXwQck6 znYWzCjP)Gr!3utNnIzwBo;7SglXoierXowylFU;$8M>+17O@+>+x*1i`K^B@=};9R zSX04CTNkfypj!t0?OtEWHH^ntAOVWQWvRrk@Qc5PF3KPCL5e2r>YVk3#nHNH+;6+) zTr+(CmJF+Q?>%$BPdiLKw=4!}h!G?qGbAqHyfUH5E&Z?mX;v024YdFXRRyuuSJmyt z3I~kA_+r%$e;<0au*Ym!^+oRwxwT!^>_4e+I>FYu>q&u zm*;c(y1M0JSU)~?GAWAky5C&;;M{#1YBmj{yasevZAoCu15Ux^vrB_L?(snK!D?(9 zTfo2!<50$R$b4X7orNir8=%A&&(NIz02j&$((*t|q~aQE1R(%`cR5x1cH zmRCi=DN2#4jqH2l+OP{#hpUe^*l9HApu^P)Wt`xM8N560raEhKLiax)59bn1qo>^g zY!18ESEuE7Hh;jSBuJ@AdLAMUC#a8PiORzfltgcPNnUTmXM(~D2U!=>^{$)?-G1P zo)WL}!GcF{0w3^rA03>taZP$!k|uM7JJclw!VU^wFB=4(nRhO9G-+HBvR`R?wM6;)lYL`kp?V?nAD$IYHwGIyn( zOAD=9cG|d*qa2To8)$ReV!u8=MFg@-xJx?*JjK{{P0?ecD}Es5!n=D>+cvYI3@s( zH$c0-Zo8{Da1;Vh&$)g7Y?Hn|C7K&{QWSi(Ku)=77|upuG@LU(@UnH(#EuceKVXW2 z%RJ30Bnd0vGn2#PP>y@v{7_wS-N2``mByT;P>xITR3JxDce}UH?G;1^$ny!(gqys; z?`<-m&X&EOh+{MFzQV|C`5?G_+dfq$*T1yZCLMIp;o`3$&YM$<-z8rfuB{+GH-t3; zmUz6#1O{ZD5OEQw>;F_xcg^AZqsGdP<~f1-f2?NUc*7o6H^i#t{cN6Ve}BfcZM8Md zlyGV_gWJF)!BHgmz(B!gp9H$OorONT6MS{e)3+Bc-&#%cWlX+A{b@A(YOeEwW32Iw z#%T4TG%%XPVE~YnEYrk7*W%H`J~&(h)=lSkt=x-6dp_R!9m&BsUVxuN4}c5iM)e>5 z^X8`7u3WiO08&(-J@$J<_-ar?V@`pS8x-A)sFw)veNafQcaLY6q5 z{N<+sci-~bt?BTlX&qxQ)|9iND9hC4#@Gf%g5fW7%^pe575){`>y#0B87LHvi*$(B zX);u8k|Dfur4i>Z^PNh!Cwer{R`|QZE{HLjkO+*TB%VhZ_BTP;0X4V@D$jvv0Ivbt zgi+GV1H91-Gna6_i{UH_M#JCcJ07-s{`onAcew{hs&_f3$Q;44GQrT~rX^?F4osh2 zO7ns0DUeyUgK>1m`j5HmBfY-)--alnvLxD*!U-5b@T7<-Tzkur7oMClG9#L()HZrh z8(PusqoAT2R0oW)a>@kD)A;Kj?t2@(pH2}xOJW>Gax~2{AraW>sbaYmlK6?19~`G2 z$YnI*pNl|zNdy>6E0ioS*h2SuifsLJ1}c|`mOZaq6)$X1^;I-nxF=@4CB@=yaG`s< zm(bDoa1oPeg~Vk_hT}qYcs79F$qz4TLOUkS(lx0Kv+NpV$Pkib8J44QlBan#G-QbB zrNHaojf|jN-E!{dub&vXGjgg#k|!yOqd7$)q@JHQCgNY70xZynQ0`@%qhU$yzE?fMpp%P*@(Bwe(U!%S!$q zJuEXU*KJSaHZ*S$IxPL8QZ<^;l&$1gV)tT~;IvDA+lk-7sO8>a)64Hy7-O{jmuW^> z7GmTrAHV-b3f9h02hfmoE^$ECP!4!0!u zw>!FgoR;iQL8>|R-OC5?LKV-tF8Dh$)S;50GK*P_FV2+HKC+?Z$d@~&UKpX7&&NeJ zpK}tWNUVU7EJs#5@k|dtzk`n%R!ChF7d-vS{&P90ec6;?>&Y zNo~wyK^7Qe7l#xx* zQx69~5&Q>IBnj5w8meo9_AzU4Z7k`-b{rk?XvFg`xPmD_A_bhF;eE>s2prndH5PeE z-`m!%4fA1MpOAlDk-!KtElW5@kVJmLl4R0vAIf=O}i_$0*cNc-dKtF|vW$ic1jSj(`td*7P;PCsqzw!eGU10VJK z>9dT|zVBlBw$nXR#_Z4}(FN2Ia+1Ytd2WxdkH_W!u6$p?2QSvl`JwBBsiP6^)aMG* zU9YtFxmuu)PDhAo^{{-}?D_ex6=*W#v9OWzS&U_Afx&@DDdRf3zuWlZxU|9gaJr%) zKjFVQrh8b1ezj_0|1M!P0DtC0o{g5^caMq(qz2j4AB1N^I9`A^bG7#BUt;=< zK@Q-7mT4WAEMJ91T*#B z>p0fDeCDZY+U%U7>YC@EAaI%rdZ88S4Mtn;UHJI?8t-efb5~~Q8!(v7;TB{yCx)0CmXxrBM{Ltse)i^(N&_Rc*7Y^$DK!>Z)9%VF)wY>9uJ;VC? z$gM`4WC)U_d7frynhesM{>G+f+AlCu>${F~HozL>J@ ze~tPZVLwr0r9U*^(!EaohfXbx<-8V2OCl!9BoIX|*7I6KoNq?N`CLWo*8sh1Utsy* zz}-8W42>3vho)IuAD53L{2Bf{g5>kUCYF{69NHV>Egi>pdF0v2O`r~jdIL-Xo?@Gf z(+tn^3`PKK?y@gle`w!ZsV2l{YZX?Haz9=`J=dU$W@LkyyKAClOx+Xj7TU_NmiPQI=9bT!RObT~BEFI1AU!!MC+URy=ap@XPQwoIDxlWZ6md#FoHyKBqISr zf!o0pNV^c@V+L~wi?xmL) z@-vP7ly^z-^LSwM2po;!oCG9-U}<~(C5s=FLNTxq7uf%lNc_@3L}w{R9xfq z`}Y>sNH;}kNgFY>E6HF`j!`43_@pUZA6gVGvPe@7&0}X}hccwV;L(m2kT!qlj-i4l zANaiRAR3e$9vjLHjUcBRh>OAL+2Ij_x^FbxZ#ng4^{FFgX|B@y!n(=Rw#jy%3wXUl z{jo^cguE{eKk`{joK&ssMq2%gT>q60#wgc~6`cPGhQ*cx%WF2x&e!JgzOZ7I#Yq8^ zB#gpjnV;3_+n)~oTDrG(Ka}ctwpu<6r3;kPEZ+>>2? zCL;>cgaD&C@mg;0zpK$1>n_rW%(+A(zb9gip()JK(q2{nmve?Ir5{kB#aYm}g=Pq; zXnys7OMh1tU5u5y|8vdo_)fLIr>p<#4vho&f7fdZ&YHph*Wzi&S;7A^#`(VRn(L7& z>W7Qv_&y!M;~%iJnYbmV-3m?K7KW9#n2a+b#tJft$t#S%EIPZnZt2KAa(e)e9T-?X z+J${ z$E9fpwY6Mm3)BH^R0sUM>A{QWdaC!n=Hw1LyLYS}j8CYF^91$&1hpP54vPdmuiYXVhG38)x|CbkJj*in8 zDB%*#3LSLNK?fal&_RbQ4KDS`oTQ8NehG;B6Z9_DK?fal&_M?su5u{bIa_AA_QHuh zx3vxvD6)JWc0uGMnPo|V=1Q)~b`Owzi;v`<{^s{Pjiu^R_4t}!W_e`6+|>6PqN7xi zG(ln*&GHz}vNR4ml8)IMz`r75m>3O@SdJb%UhmL*5n|R51V_>|Lz5KE;$*ymu#AQ$ zEq`zGwCX>wc4!mjH$riag43|CR$u!dG7sNM%jqKz?!w=X9H?6o1e~WB92aP!`{>8( ze!aPFhyz38*}+{@4{_+XPX&jP(Xhti8UDZ28$LpZi(yHgW(bNPX@XI zB18OauxvTs!?iJ^S?KsGm&BBlD4YizO;e@uOl|VKC9QWV|a`L_y{cBzv&BlhO#uBy9wBVCx5us^CsGL_T@b=Y^({hug z(Vff84?nH(vB4qi$tl=fGlur^z1GK7I+|f0jaYp)%G%v*$>{XIr}AFaj*y2tqfU~n z%`JL?XAizWAe?b^NbOtsKyF!wA9z=V%%uSk9k09wu--)d}I*gnfzK_%~p zI>F_Wts>vQa+TCKEZdAV#eP=JX{L9mdzX{I@IZ2Dx5w8H6nF9DdlwI$I<2?MUhQdJ z;nyJobhygjf2iJtS0sE6jjwi4e_iSQrZM0*EF165ZPut;#A7%(En*TQvMh#&9K%UK zO2s!Vo6K3R`nN^aa|9p&iIp*#kwfZsA?i*v?6Nd6+&I48Fr*5>;~37e1Wr>FOK~A- za$Zq215}LdU6y}_zqqx_t(sH{lO@4P5qL>sg0q{??a3<;3PIfjrJX-l9qf0FIxHA1 zOi>L;F^2zF2l`*M-ri+dF#EM(^M8%FzLGIamJ}9<7%suL4c2W)k5GK4TkV#v=QcrrM2lRxpB|P%#ULw1t%yTkRgNS)2*3^ zul&sN;>l%gYl)^%234N2qD0{`!2^lGVFTiisv`VVGK!L)TPFPW;*qarAXlm+%iuiC zayWr81V)G0KWj(rtS>A!2jBeZU&3Tj_sKSbg9(ylL`~2Wmd6IB9Uk5-g`s)HrE*-nRYl*?mpW6VFMpjLq)tB6^AneR)2?2b{p=o3Z7ix5}r2nRq79 znQt)xY^MU)*)+T@^rU56of9Wq>)MC1vq-0D3I)#uyDPtEk*g#rjQ4|4jaFx;4@uO% zut8$1jQW(NLBsifCu~J8{uzeE{h9{PF<7W|1yg0hXhaLAET-zMgkd|Ip(kE>yB^|TvOL-qC3)_0Uav}X# zD(s)4CLl4S)rf_UY}7pf2Ok0bl!5q%EiT>{j90L1?cR@k|1L^3uIa%m5d%~+n?XQu{xZ9 zGT!9JT`PINqU~x>KhEn_1@$9rAqAh9|Jl;V*=>Kje(32=GdxA{3{CO8KY0#3-FF6Z zcNvBt$G|U^<;ONOe0eE)hZVyZ7Uv0^r5KvVF}R8S!vjq8>0T&#@Vn)`ZeQL1z++mZ ziE;hHNJ5IWIs7vrw!5=UI_Pk9%A?CA@8Z3#%eSi4@O98Zhs&?O>!9leS8}@v`ux&ChyRS6st4i9LK_t) zUZM3}Z2qJ3vr7Z*bP~iwpNfu0ZM(L%PRf7f_-}KCe_zf2n7*#Vze9c|TqM2fN}o4Z zo4ii=kB`%<4Ql>Nze^uS|4lm9)VoYy*Xf}1kN+5SLeM?^GU>~@9MR{c&OfRSy6Yr} z3z}Y`bgO9K*zT8UUM0m@UqX!a#~Tn=bLziT{#X}_?^HD7RHrEPzSR6U;S#q4w`cm7 zt!vxGNO53d&etz8BGmvEJRXH~C{av=Ut zS$~qMqGPmrxiK7KB+@2@;=`2Z$j|%rg7G%+sk#(q2hAHbcL=Lb*R{S zM+Y5rxcWfnlRD_2gAO|AaP>e%?S)!wuHP+tf2z5B$ym+)xJzA#1tt05C>xjpq+H-F za`?Ilg}x#W7$A5jTxTbVPiNmYV}W^gwVTumv2)0nqhYQ9cB|-#A8` z6{Q{&WW{)2cf7bVp)q12BpCj(B>sKkiNyS<>u$V5n)WT_!tTmW8%^Hy#F!^W^!eNf$I!c=?oQT=W>?^PV5s)m7%M?xmIG}Nk0rfqpR{oyFzxS=Kjft7z$Am)StOO(h7s+?q&PaM; z;Co4MCcST5RXdK+0k5lB&-D25=I;`tu1j1Y1;T9AY4(X=<{VCmSx3` zj&QX9NFy2-mS{LG6I7jJcV^L|rDLOF+qP|0Y&#X(wrv{~+qP}LaZ<6Jbk4bbdyM`A z`{UYcO+1s7j;UDE-hRp3X3VO8pps;>!otVJDyq|UX*D-J1JxmsGSPo8nA8Yv)OuYs z+iI3hMkAkJ>sw%Ac=5|riG8FiD-<{)?U|1U)A@yAmZPYkhVAk1!+gy(Ny&r(mm&~r zC`$BHUpaG+IAAUnTT^+u{9Fqv)BPm+NPB_!$*D=u0Cif1xArLBKWVF6idBXQ?&c9T z-aO(cxhR%i;TlAy80oN~!r*tRud(bK7HX~v&trx$kU+j!&{@)>GG_mQ;miP1?go5B@dRo*ip z#pzj=mX~*jIQqIA@?PVM1=!I7$(=@S<32jf=|&_}go?PM)U5KRvADh;Mem`e>y z$P|8U8n(l-)Gq?gf{*AcXcvP;Puuxio(AGz8Mt4`n8C$2iKI+bl8SNG+n)mm`;k+) zLp=6w;ytk)e(iwxddpmqTG0&+oSjwL$I8z(Pcj_{&Fs4}{hL5u0ICju=hEuAD>yj> zISMIx5;IAHh~6b@zBj$5amcW`_PIX1>+H*3cIgQNeAckCi9;d@+V6a+{iAFg^_a{~ z_LAy1(E@I^)e222j=x8Gr8ispGQ=F>J|1E@3z;hACI5U$mgm?zm(qNuFm?Q{llFQq zTlGQwKF-Pn*WcW3oZyL!wzp~9sy^ktuKDK_BMiGi(6R%*TBULz3CKeIY6|DDf54#E z&4@XjY|R?4b-%TAt!!9J7F$lnpr9i`jic*bZr?&T`SSl`wT9pWp2qs_p9BcT8ov!1 zn}n9{4|2%dO^v#OD!N^s1{P5+?rX$q+e8|D*1C1^Y%M_IJ2!M2wo?!p1{9U}=KVF( zrqj7TQ#-XJw1SO){CS>T63$Nabv*vg${*d)9klXhW*=T6Pv=&nyxj5mjz*+?sETLB z>$C@h^^=@+zIog9dEGp_<7{gn_p3&PtTVRvsHpLH6`PBfCvit)(SHwk)Ddiay$j*_ zkcH#~k1DV9Oa=9)hG$=SR^|jrU{h2M85g3Ah#F989+6C%x@Tv<&HoC>)s)KIG%Sz)p1uOb?>~8h`$>QqNIfBb>UDmb7@-?!>hR5&I;vu? z1}=votfaEAx6K?Evfv~0$^mE@4B>Viuj|f^8lhy%@v4@{L`%#h&QGhFbyJ&`HFUH- z8JC8Zhp0fKvWaitTuxqjUDbU#`uEnP`S#HRh%B_DU6D05FEoVSn*RcvBA9yAn+!1- ziggZeQR$dIOMGk5Lup{?)qC~nC7Dj`$z~DpoEgE!KY>`|ZoelWxu^FF)K&XqzM0=l z)s=ilCfsUS+n7dPU+h3xf@$8iVxagKR^A#5$5t&>Kep*APW?2~BnAGuMvRoyN1odt zmFekfes)(JK}F(^&}1{PmJ{}}jYtuwyh_g=r<%XG3Ep!)=DL5^{^#%eizc~!+@&Sf z_Nk=t{Bk3*aBD|;9RX5^1zBnwW;nuo96@3`XiZ(QG(xpDMSrMS>1Xq?$v4O-5>Ao4Z@T)+{K?Qt6KlrId6- zy;qay#itCr6|P7>>Ta0wwMKgT>mKRl5nGrb60x_KM8*8vB=@1V(?|kjGu|6gfn8af zHJk#N-iVY?>FMY6^k8S`YWV#Q3^Ke^n= zVdCF5(A-O&4OiK8tys2RkWP{T}dsX@V-G z*N#vp6<(?lhRghph+OH27#@UXGk+lz0iS$DO0fy3K*KIkM-Yky5T|#qFzgZG_zEpl z90nyGa|fmc&jS~!&A2K=C&KdVM6m{tegoPN!^q=$xTx4q@nn3C5l{Dz6{+{DB~kJ7 zx%mzX+d~)|;GM_9rGP+FvR#=~hJ7nt)VC|1Ix`&j zg8(E$4zM!PJlr$$z(A7*FzZa39@=Dxh-*%ma&|p%E4N^ZEvQA34uMEqaSx zH_Tva=l4m~^-7x2p!C{~2bUEdepUH(wSVVb+p&ctc4bE=gXb zrrpYaov~3j=1rB>hCX$PS)A<5{hWcM^)ubpIy{pKZv2*4%n5VGfmdz-+qL!ldY=gK^HXoy@=P)OOV!qTJJl@wF$cC9a*?CA z*N1Fjw(f%p_a$4pYM`--D-P{_*OULC%MAaGR|gX+xe4}tLY!WP2|aK9zc zNza5Jw1k-F#gWXRJ_Iz5HWd29gZw3KCDS$Jq#Py#%aMBC==@Lzg#`&+vKm#j>?&W) zl0IK3&I;$JHAaQH0hMhA7Tb?1Z8YP{dWxOqtewa<1FflPv#v1yMEy^~k8y-VNNmMJ z08fXVkgV`Q%_P^%B=DELcV*sC%y(2^#N0z6(PypEe5K zcA@v`809yOaeh+ulS}U*yiJAX;gwiMhQrKCHIO8Ve_0tJH1iF^Dz9tWDt-6o4Jf?z z?)3N^HD6i7{q3)nn%Nrvf;A0W=ib{-k|#ETmA^&Xo9ic2xuw^#8oH%%^7U`ITe+Uw zC<$oX;=qxD3E2btoVY_Ivp=D!0oi(d)7)vUMu_r~157bT+))7Tpdlt&p?8;jXj+vi zpR)D#&v{~ek5GsK!aA(P89NUarOZ0sFKfMl>Hew}v6m&u{JU`{yVGby-YG#g9(Oze zOk$sQrhzsfgtb7e8~7G&u_~%$6WwD}B0`SH0hbIk8x%Q&?wS8^mowrS_l))2-rFY6 z?>P81*%5voMks+4mQpC#iWr+yqPMsUS?f<8)iGf z8?=@zi7?UxFJsC@r-D(-tWmtw%f+N(Tl9Ro-e>54-k4}}JDtEhVZ;p=G%^(nR3B-L zhL0VtNe4OMjX(#XSoy_u8@fcXnbmU;MT>2(gN&97Njl1WVVXII7 zx0EWAe`RW!RsnEIqnX|+0r@0cIRfYVQ2^2sl*1R)h z=~I2*dDLtQxrDREDOa@dh8N4*GK+|Uoct9un^wb|cmoFA=Mn-sc7yBSU)%|-Vtc+{ZK+|d6kv;W`f z?3c0v|CJ}+i&^?F-#Q=283-kGea9M?^SoMGoA#;<&k}D9UN&%HLPrdVjZn|^|Gt5O zei-fU$Gopw>L;s9G}8wxBwRgl)KR#udChz1)@MLb=#P~;*@~J-HYSp!PquC)gcVwr z!_hGuc-Ix+?Rr`@)MfA|5s7~Rn>8$5`-rIU6ok5N8E>t$;+;Zo|5+|Np=X5JSKxr`N*~W4;US#O z>2Av+ZcHnAHqepyKY<0HHD1C0i@7Lu{oigbO4)f#Kc;J$XyXZtDOFmmhYpyRmepBokI{7b}UQgs|iriAcTP7 zr^~+NY}aaGFe$k`T#c8D(P2kk9Xi}rFXQ{@*egA7L{$3J0x#LXE4{!r%9!xI&7|Nu zi$EK$bXi%|W$d)`?svTSTYqSJas4-}*W0}gzpHV-?2)ps?HPVw*3uy!O5*`u4Ndi9 zVyqXe?0Q1De?2F;6zPmiwnC!k!_=6E1a_5XYKfu|5Z+H6tFymjuhs$&;gz_|`oJP_ zR5P>KK`7(<-^}iguZ-M0#6Vrd2q7G3nV#GQZr7~GCw%`=HAY1$)p#-Gd<{J+=S0`vaWpA8}!zS-fLr(tL>g3_Q zOh-If^H(gI+3@tEUIRK_qo1a+UeR2DF1;I&)I>6&6!DC-EPn^6SF5655qqK)1^B@W z);0OLewjLdV}3n4q8B4KehV=T*o;)$0MK#q-u!Bd?SxEFqxWjh7;=$o_A|{J|Hi_Y z()e>oo&?2)*LLPy>%n=5Vj#*H!6G8{xc<-N#uQ1>i zt#!-0We4=BtZ`@g(FTwHzz8Q4pL*0+hOTz5Q#foANLy8}_iV)=<_pWk!Ku0e+ZFE* z+<;B~9ZR_Ck{{V*3wgs%Gk%B7zp7O)i^}DUf`J98#9~?61|Dx$^*E`%BOGe?yMWqC z0C-Sx3e{#LOqyMFDliorG?bTc)B9~L^ph51&+BO#%4oW*8}$IjKYHe_Xo^RGf@LOs=BXg7llb zH%)`CP^>oJ)t6;*OyHNTWSAk1O%#p|#xI>WL{!#Njty`(rpi_v2%fW64<^UTXGq1! zH3A{QW?xw1P1Ak9Sv0f6H}G}LRI>I@R?^mVt$GBij;sW%;r`?*+o!6dCj2syn_NZE zR#CTMfiUD*Dt6TkDj5MRi-??l`xrh#QnyAa^^E+o?59Jd%Wq(tklw--``v%N3q(CQ zo_H=wCDHhMDR@E?DoV8Ki22rGYvcK8rpOafm_lSkGujv`w-_pwLgWr9R#C!`*=P>0 zwUpP#XnThvf$E>hZ)C6xRC5r-@$*e;ak?(+U_ZldzmfegQHY$6_(zIgqnM#1Qi-J^ z%qV(ArDy+=BP{oWT~-NNQYm_u$uGJMDQLx^qbZq*Gbxz+lVqN223#@Ff#`QZ}`EJYgq6{G`;4kf74mUnaA%p~Gs2ABjEaV91( zVGk#u7-fq=hwfGdjUr$oY8~cJ2bFXaE~1Lv(27vn@G8?k%m ztj6@&%mkadh4wFPzNn4SQ>~5f-y>}d#ZUlpe*`h1zo>zsX?ST_k^L#-*jTQ^$qt7a zUqJR@Hf}*ILlI&a*CStsG(q%(! zAGs756%!`8Ls!#_Y6a(*YA>#$V)sef-iMX#7Oo5JDCN zKTa27r>1W&_rv;JPyg)^qj~}%oMAd-LR_ROnaNat^U2^iGcT*C8a1GrE^by_l2i&* zk&rB)MT)rnVal!b6I||(uFZ)pLrZsIbVusNCs$c4ZEJ=wmNl0Lghu9}MqsjLs6Q9Y zGMK8H>(l$t?NMIM+{lB4$Xc%Q1$}#*41}_Eao_=*EJ$Eq$yCB%oS*Y<-4q1c$50fS zQL`=hci2wn^fZ#iRO9j2kVjq9&Tk!P{c3mX; z=tAgd+WU`+4&BU&A#@2}?ceCnbm@Y7=zdqUOD?Hl2BF9PHrJHKgqHQPJM}Ve8FrD% zVKMsNrn}nFdz)e)C^;YyStYnCx%H$XwiSLH9IYaCP$o#5276>owf(|eZKKXB$Ve~X z-Q+(PRM+&ix+2;UGhtd9zlN*O7|cz4!CuKUo_0hw%}lz=W&1Ow_DTuk4q!$UnPq@o z|J}-~vDtYN+GLh`SD18a(s9Tk;5(u$6*#Hk$++==x=Sa!_@N32FB4IXW-2ma2BY#I}gS2UU?$%m%Hmj81YpLC+EcCpGb**i5Q?Y!w0Nq|U z#xIH=9Ns(#2~95nJlQG4O08Jk>jJ4kbb6okxOMa-N@l<7hj?pbePAxQ zPYPbCS?S6lp|Z{7wZO;L+Qlrfd61KcYxX+A1RPfXWh-UxZ)^Lf006uV7G``xxwud( zM=zOt4S8>pRsS~Vdpn>HhNn#eI@M>SH3H8MvfarJ=ls|kGh)Wm!B)<5N2|1$B{ zdypnzuLUFd3n~%AQwH|Sv{jwa@RC~(nJTQ=*N)()VTk| zZyT<;H;^+qa^~4?jyTO5h7sobN`7H{{58_@k@L=4x^%@ z$NVwBMaQubr)Dj8aWk-`7#bHEQ^&}oZ!ooqR?@Ll94~aRL3gpvm?D!Sj-6cRXTBrX z8pQ%Z5>95XYhd7hEgQMId2c-iQUD+$Uma_Qz6{ zAPc*Xsv^LklS1#&Ta-O*qt&ODfFw)E0%;DjK$2D{SfdY8UIhM4aoxs}fWTA$%|XeS zcF++O7APzPmv93130ajT*7n~@7Zu~1Pe)5Ml-|IJ)+Md(0FjdU&|$k32x(2 zto$z`LDHwK*OjOCrbf%SCh(wSU;(F|!Jf?>@M9p;)O3-+Y4hrK=~T$>a!*o{_CQzCK_t1#%xW|nvkt92Es1vWo>NSjRcf9VNf7jF z?q`v-ikPGFHqy{bLJg8c_t!`2nA^IvL6xX<^iTrrqi)y}pEj=BNQt1>su%eK@YSHO zBp4_A!}qZ4^epz%{GCAK7lyE^{{d#UF?6O06F#B>8fUSEWs7KRC`sj})8Szgm!zY- zaaR>gsHV!={rSY(N4KAi0J&MnuaW~^o5@Bm)U^0jP0F^xvqZZc*BUXM z9cq$a#kFC@KWr$PvNnR;s#bF6%eVdpds=CCebabCd=CP@cDZT?OCjZO5kLsR??X%RqABuf|Jd?an%p>}TW{kP? zn@|p->A1{^->vFu{GE9;8fD}#SVkdd8;fiw1tIsny;}C?)aRnC;K9KWz{{h4B#c-F zX2v+Vc+NQna<+A*t=*T{8m~^;tB#@5`o4QZtK(UL=6bJ6{rUH%fSVU*q!f(Aec^z8 zx}dRoqmj}HOedRW_A>Q-b?4RfA}bhd^!?$H%olFjq7``USdI?8;R29YykF=1or%;5 zQLlv)i>{29=vMA%1OCfl*5?Rsog9Z6bc-BUM3Snaf++YOgx|aKN2whQWtJQpq24it z9Qr3R99FN_&BUAiI$l`4oyyuJIIP0JZOt{&==nQ6Oo z(XR6#@Y8cAGF#`vn#UvVz9X)_ zF6aDaRdT2PT`;Dsc~0)9RWxg1--snjR(&{4&OjMSNXbhCc!|RL)VFb9#nW!3io{!s@Xf3z_b{B{F z8N4ZJxL9mH{9jpTAGKDTmCr9Ic=Yid&6#wk*f7cAAc(wm-~yfXubh^trTHgS1sLO28l<;MYXV&z*4J5>Qz^fKv}vP&Mm zB_-f8#JCLIU+T%upsF)njsqfX-;Xrys1#eSVjH!U@P$Vye^1ANOBi zW@yOt3Ql#5f*%a=iQzGJkqe&sEQ-%v3lQ|;Xh?QwMUy)jrQNAU))p{_2qDy=^K}|# zV2KVRo>A_)GE}cG>PEdlqOJ^tWFxyTDUoAu(@CMD{q?)7oCbUSauX7@#8+7E4(>WN zZ$HwcD;EJnnhSgR#}0F3QC*}coWqQ$@Lo7NzJERESZ}P~_0;;)IZD5=CV0?<_Hoaa zd4{1=4-9XYcs8h318zX?^wg&HU4I~|K<;Ik<}kc>if{*fcaIG`hwQG2e;wIPvu_dk zi!h9`AF_i=((ZWL&PqP^Z@JmDWa#)R9ed=2_0+q+lII)t zI8Q*k#gkhC#aZ0r?w2IlYpl5@?dawYanzAh%xD>ZczObNfc8^Io({*bW^;zPclQmBDU#T3P$heznC zAGP|BEexIYXk=wq?W82&n)&v_lp}wsy)MH0IHPj^vf%js*yeamW2uuf?^q$Y#gD83 znqL1jtk5b$by(!rFo4mOZ#$CMh_&CuI_p>mT}FIHS-zgQdZ9RiS?=VTnuNuSo17e_ z`8CpoLV!!4aNnj?wq^Cnd83*Rww>t~j;8e8n*w`NBM9pbq(J!1nufA>2#c7MvZgLA~SI_EngROj4#{ zpbQQZLNJA=@(!E7kXvnfq}|84&0#2_%GR+CI+0E*pke?p&TyvpPk0MqlJ?BeS-$h+ zO5HMr!*=`TXkWoY@Wpy?bvPt$Lerfdl)W^PC|I|Xb;z(BoSBIi?85-&D->sTS6y== zqgI)O_Pitp*;V^>=y$y`Nk~EOGAmImrx;nNn%7|Mfi&gOu~E3P_ry|*%q}96n>}1y zM_Cnub@d?gF*V#(o^9FQTHa!YTeX@Z>as6|T5+KPm?ts26QL+QDI9NQ0nHz|kJp7K zCuG*J9I>F&x5Om(#ggD{!M|587c#1f16C{$f5S8~2PT`tU`QgQakVw@ZcogZT^6?Z zCf(ZlgqH=!*o!KXp~4l3nW;dYoWc7=OV6~+6*H>S=-3^oMlbrA(ed~mFJ?O8TuS}g z6*{fsn1TKQI&Ob=UXSgThedkJT}5-|zwT;cbhdtTn96cKH6;JNX_BlXW{_g^nFw6y z%cM8OldgEZEiEy(;)~pe&Lq-^?T5;SzUgZfC-I?ei1cgx*Td;GAKFOdP=g~HA1tRpGPX_z;Hc0t4l@(Y%Nb!KLfK<~&Rb08MMK_0BocKs$_r8DWe%3_&?)vk9Pi(JSVualg;$^}at^x%TJY z8X^2=&l%)#-pJz!iGH}wx7%Z=&w$^~0UpbF4`1lFQHR-KCH5@d<)%hrP1CdjeDMO3 zXjaum^zuyEOh?Gp`xjogW#q!y_Q3C-KDbYs;oTp!fIAXgyboPUg4(1oAqH;ML8X85 zJ~bsyO}9Mx>wENzsRKaMR_m4AO5wdt*BBPBNLrRboO#;KypQCEzRgW^wkl z742hBs8aUOHq>0yR`H>mm9qvXafIY&OU1)v!<({Mpi8_69=C6UPjTxNjtWjHD%z#* zN?mtnDhU$t`l)mZm=aJd*!nIX#*xT#Y??x`YLomjv2oSHH6lwj!HbwO&{Q}@t+kcg zc-(C6+q8?})LmIl;jYVK$M)HuO=&*muA~sk!&a?oN6+rW-#v(}o_C!vzEZ#>G~0dG zj<}Dnr+-}SSbr`Yyy(UvvQSbqH8f2fkpALI?f37G0y%5KCmD_T^cXgbl;Uhb;2c$457+I;PoY2Xtw(rLu+}DZ*3~~&ybPNY~ z=C%MsU!iuWy38pW)W$H@FSmR?YN<^zFT~br?3=t7E1t_SVnL=*p2ZZVfht%a7^VrE zGi7CPXI@r{SeMOWb)rsKUlY!<`^kV=MF znKpgF$fY_bBe)5Ipr?6Yg&w+OJMbfUA<=(-{<{F-e2#j!Da5f zXPzpE`*6ZQH)5T|y1mi@C1lIE+fsj>obnvHgCi=ZB!yO{8j@tFm{b*%#*r?t+o-U; z$qn1ha;S`Jm6VN8TM+N%zfa}Ng)Bh8Kj>8l3hHOve4`dHJ~#An+jcnBqz}xXCTzTu zLQ_sZjFXr%pb1C8lxiZCp8goDw8myBFXH}g`(t5k_EfL5hD^a54}>8GO}LM|6S^lD4+&+gy#?GG>CXYLFRH zg=<{C45kZxAY=@)S7VoDpdI{#g`>5{fH=k5!&MLhx zF8eDz9TiSX{jU{v)6lqAYSSI(nUP1j{++?!Gkb=ZIclg5nd+6O#ELU<30kv(_m2+F zKBi|k6@G7IdTo*IuKV)+(w-4}D;}WI*6-hgANN_GjowTx<86=ow5&NgOsW~{xqIiQ;@DTRzgKaxVoN|WBeRXSNo{pO#k0?|h- zIcF8IGc1w#{lcC3=V|NiW+}#8lR4;T&!!Pq!iaYo6{ulL(RK-MJ!=e1nLrLH0^XBr$y?u&9sGQkgLz@?K){59;z~!}>&Jh+kB*FmwLL0_bEJZX-%j}k_D;&oudp$y^%n|zt zl|3e&Omr%8XC^!3SZ_0HOy&Y|eSA}gm1b`3z0_9svWBrq;+%C?e#PRV2XP0hZ6OWHXF z?iwm!dK9`5yL^MQgVw=Y?GJPg3a;`kYEk=E%H@=-dh0szGmpW4r+z9z4B&oAP9s0yEec~C4vGio5;jF@`R*- zl#T`)P{)&Mh2L_TLEv_L3MD|6sQ7bY*+=@XdKTK^x!!*xqmWlw#-m-KogGmI2_WY9 z74I){Twc2zD;FZkkXdx%reSKkUMm{VP zm9zFM;SNSLNdl;&X~_>YVp*Ph*8cQQlmX$q3&q1qB$n8?YC5=IKXV7K?~00cQ<38U2!^)DH852riWDc0}1~9ZV zm7#0?a}OPJ*=m;n#LF)*ul_yKp#Yxr+&N6Q*vVYG=e1E97yNJI{Okvrt=)QM2e=xL zaCGo$&zxBOZ5kE3Wz2tSP^6*lwiTff1oa-B8(ei#$I*Fj4;Kx{Ta5;%?yPN;R9*R= zcB`m+7i%xwLo3vjh|$4vdWhl7U$u}f`At&p`!jmCn$fNGL$|ecH zrrD$WpUOc*?mg~i*Ua);DII>p=lAjvQW+zdi(S=j zdaK@@t6dZ1zkgle*UG7`@_Naf?`T0q zy5*I<^^||Rn>Hxf#PGem60O8y`Tzh9@woRZ_);d^g<P>$ULgIc9 zosorBUNZW(G4ae9bUKCQB_?`7#+@jb*JSd-7S}lF;+Ao zBf`m<|Jn_@O;LJ6bmqs;2;xe~<=3Mkt1lDS$O3MN1DreyE4=#ri<5p<8Zp&A?S1kF z^lq`ACIMf$o+@E>Jd@Hel_qLflLK4UJyr&44ODN#3#icaA?>|Y(#)gpGOCu^6ilLU zT`@IsAkbu_)pjX`SGv{Lvgk2+`apyn0}2QQbKt~eF^EJNL$$tlee`aawy#H8)XGcx zpWM0c5g*-^KuuaG?x;XgmcTxDJM3halg*~fhGtvhbTSPcy+u@AUKT9$bp6dY+2g)_ zSaRb!cBQVVJIvaJz(9w*<-E70Di5g=)xT0=hsrKHm9$n)0qg4>XQy-oAzh9Ds9L3* zDF;p*d!`RyM=5&d-jr9RnOOCJ7-h3xV*kN9*b?cVmFUEleGSA&d|Nc!3=>awWhz7& zjvV`*zKV8t!%0^X%cY{wycBDqaD~TZ`L$mjPjzbo5*fma_7l^*D})!AM<*Npj>SUm zBDFr*lb(8RL*JVfv? zH7@QsaNXabsuPgutLW$z-oDm(S~V!Wj{+m?K87;==GC+HwS!v`@U@WK4|77MO60AT zEQEVzZ|ABwMLX8hfBgI?lPeJBh)?cgW)`1JrtB-giM5z12e8(arW}VsAbJM*<%lao zT}nqay5Lt!8l$Om2qBvcs{xc$hDlh6{DrZvjtYoPu{u{Y%0V+ z745gyqYbVSBIn-?4HM9X6H&C{gKxdF>`<*W%Vq~)Pm;UFKp5wt1`tw_sHkfF;1MTp z0=*}l&vy8k8CgFiWWp|FhOrsav%#ZD2H0{6Ki%%Tt;K3v4&e2w=5aK3l+H5$9 zWZEj~DyG6Ktq!|y#3KkB-Gr&SVA(`|TV)$ozt##29bTSWLPj=5V!;GHTL!o~9wEqf zKgdh@fdU~9h^{vuQ@B?9xF1L3G=H4l1ha?4LepO}gr%{5dcExEZ*A*Z2quv_r2AoW z&~hU$Hy>{+Z=FnCVnZevniIptK*YCImvx4*-rVI>s_T{>*N+@-?WOJa5j9{W56r># z&!N}$xR{mh*wyad0NtQ>)q7wDe>vI*&FD?gdtAs#FqkTBFE+9pa^ObShNd&F*~%l}oQ=__hsOqQ>Jhen-DB%A>`hb}qO@@zMf=^B znB3JYd1jl_ZgY~S-gA(1<65bn0 z2mC4x!@?WAySK%&m&_{u3r~~KDTLG8XIw9*g@L(K?eBs>NFTjZQT;5r<)s zVdMI%^5$xu7KDn03}#)Ly>&DSNSSoukOBxJWXt#@f5m6Z$?BW9!rF~m|Bv_IA_fQL zj_98|c>p1%@lyIaIg5~ea1G`L*ob~eYVe|QyE+{vsVMnW zWR zAtd482Kl1ZWh)?Gzcu2IUw(PO0%G-}&fL9ZH-8`KWe*3s>o zf+810bWL^}_gR4$d?8-zs~!WRq8%?S;fH+DM!pbmkI3n(_&XTWVH;(YTt)YQBFSh= zV+!asODCaeFi~BJsa+^U)$P}y8JJSilrjg1G!VY@6Y`L&xN-X_@8=?csZ#8)z4ypI zRUJaf2t&r-v(KuISi{ouN-T`07umOp)DnohaT#$wcjA>&xBepA>bz58w~?f&icHvL z8kxs{Hd2b#k@phDzS&34gX@`>Mg|t&`&oui{=X3LoVe2XvBNH)$=|<>WK%_6*A;p| zlY43SoqcJOuVIdGgkD1xz9@&-df<*uL|$(dc0i9V6?(LB)1wZK6+LFAdnbwTygiEW z0hkdq)DcCv-m+Lhg^CnY68~m=eL~^S%rQvJu}@566Q~-_uKJT^yA){H@CUS%*M8VS z9;+h+VOPDVW};r2zob;vOf%!{)Ag1YEhz{%h^gO~ySj%feeho~w(RTVYzYOn9(qhPL zUIQf@)<~028!`rnKed66;YhMp6 z#!vXSa2!jhiix3`gy+&}3)MFISyj-W(H*DXI}!5WIX0kVkTA(%BB>#X#Zs??D=pjO z>kng~-9kvli!6T7DiF#V&}8GCTKP>o_<(B#O8h@#b+o#zgx586k~*Xg*~VCb58}U! zZvD%(i^sdzN$U6pl_p*N)CcEB%=I|(47!EJi+TG?+_xhsQTV!Z660GN9>>4*kbnTz zED|r7{E4f@PR2g3TEeG_8y9dfx>%`YwjY-VkeMLA@$rWEkYkP6JG~4Hg#s3j6c6R- z(-_OGaue;|Jl{9n_#KrY+O#1N&V;OAgl+CY4gY=3Nh%BEU#iu4zsvo`#-M7J9KP=|~L!UzRV$m^y5W>`R$cfh5(VNT^Q z^xa&SiEsw+7MXHc_@F))){xy#?JOa_z6DgtMUWtiOVh@b2ao;Q4+I7Ma;@Sr2iqjR zIasJQYja_*JBwtx?MA#nnRxBM?Po&JX33gR@PdGK(_eD!eVmR}+hw|TB< zM&g{+^aJr9vF#w4bxOrz6|sIWMd_EzgC^HAVF+4Sf(0Vxt639sGpvor!+~QO!R;cH zz~yS`WYK!K$}!yM_suwt4?rx|_^gFK5dQNd_g5O2TyQ?Vbs~WvRr+j^7B^n$M7Uyb z?V184&PxMC8!$w8E0iJwt>SNKi6uy;eUT;sIrCS70-59!lA+YIxrA5K>K<~1$G&mI z2}l#Kp?h-)e#i3<#HTj?aG6reQouE6M`jA9-n(Ip@={we2fhH5D1~WNoQ+D0=N_;B zAoe?(qLa|0t>?|Rn`QV@w|LJ#E7&v0nCbR4=9l-rC@sNPuuFal zGxaYwJC#-b9eWmL=2DTe}!KC6@8@1JlNLKqqk-Lsa{qKis9KEggkovkr&Bye@2V8I|Q)_u)@Mj(+ zNy?MENgU)~R22d#CHf9%y!=NoehJ6M3S6h9RO|vw;X-gm_Oc@5jS}O=Y4dXn%x8yl zvU#!9LCBMexHFD${YGH&%)r9!w|TFTpA{(orsg-8>+|kwT*V+{8WTZhaQkubG1{^Z z&?GR1_tfZVzS+Ew&iiMCDyTlGpJ{GP;IL?xrRuv3gssPFE!zh7d-a64sn}WTACZF2 zZNH+hGTBop<{2s1`SYuf|qfIjFZ8Ms%X2Q#;Mt5JA-N;pjZFQus}3Q zy%Wi-e+2CVcsoFR8VNpLVoEJ?GKgGd$Hx_6#cH z7h`C@6mBj5YU0QB%ag^8jjXCDwyYfIOdQGxU24fJ-`hoEXu!P|??3j=&aB0wZeaC0 zv@pn)rk6jWrTW<6e8ptfEz|nj+9uS$REXl0u`)@S&Kige>g=(lBEG2L09>&un?KTNcPs`h1M#ZIc z=ShCA9tVtqzWA`xzV6v(CVCGeV@4>v1iGhrCrqYd0<*sJ*v-D)@?ewkmu~5pXQYNG1xjC$z6M~>=FPq7EoZ*`#M?nr+0^L*py#jwS*`S;HvpOg9?dpXd zLx9Fj$Rpo?8G6fH`B%CbXO`5v_e6h^68m;(nCnJ)-+4A*^qKRmQX&9nUMm0%qxbPWn;? zA#&sWaB@#LdP*PSwuslSLL|6Z$#l{&QF@vOrD0VOy}B+F}N(m8W#88)T{f zVD%3N-2zyH5|$1DM%o1~XMy7KOtO*ePyKqVc=9+CUjChkVYgH7% zIEbAC!E@fm<=fp#Cs)2AUv^5;Pwkq}Z_DF!`_yKLNPfv~NP4H`yHm^SHd__a+Ku2OjujM{78w$?1Js0410s*!>}{5LH|;-s&ogND znZj`jXDNzi2!am0+4_RiL?1{ojcj?R>fNq4jv@s?G(pi6%@Z_%5d;}vZ8YFDUz0Z@ z7EP z|5(JarQ%U1*J64pW9P*!ppCZty90S|(PX zVXJuvtnB2tVAO>rXq@LsNg+6S*C@e_4OmolB+-sc{2IKZ-;l7fG6AFxWVuQAV?%Y2Y5Y)e0Wd8Wmh$TsgmgrJx2R-_e75 z5zcoFo$VOr)?2CuGJPaL`HY|XDxN-0xiC-hT0bpoay&on&YMhtO>AO_L3u!Y zYzSfygBU>2KSmfT6;y%JZ~^8Q5+P|B3}ux{qO1@rO4@0vB<+Yn3}O(27(^F}&zARC zK7Yy4@z!JDeEYX4*!flxF`OqQ3L`Kv`|r05cbtA9erP$h?THE+gLOK}qn&iVPq|9w z8tm2Ja*Ts!I$AsmuL(N9RmE+=Bg+_iLF3;eE}_9$=-xLf@ZCFI~NG zd2l=6`XenLv`epCxE+z!W6sdU3t1%q|QYa?QutLoqh40`fqP;UJQ+ut@yW3+K`@S~$zYo6$-#x<5$CPnHj)}xV?`Orl zzzoY@Z^bXm-9@7L4dA((vRzgY9iZge-5%E@n@f=jU2gk$B}Z|{c1JGEi2PO0zcRRT zlVPT1w3Iz!NObQVbTL5Ad~?Zh+FjnjW$w z{5^dZHw{qmU{-1Msfs*tiq8|_RX5l2V3KV9uxun~Ey?J4_~_c={Jw=%agT%hQ26(r$LnOP2yfX%0yEOaAt@zg??UMMB5YPa)(1$(y@$s zxK%32CRe|L>PZaqy;sXwzqw91l?(Hx#weMkIHSb8==~pO3XdLD6ohMW0tE*sa=G3T zacDv@x#eZB4#qhp%Jgm^l;2gxbqQG6bOb(^6)Fq8fZ0wfr*Z)}%2?iwP(D9O z?F95Ct&-q@wSo@EVA%t63BbKowBl}es688_RPF_1aD1%j%YlW6{WeKw9wUxEs$&Z_ z0C9-8Rfk{DgPaHvre%she-HTnt=b0vBgF7S4*rYspd9#p3QEEI;5`baQ>Nh0Q9{)B zbT$ldep#o(uz+&Fxr2i0eNOwoG2;742>t@Vbc9WPg3bZ}!sHiq=5b1V=y@tFfawI6 zscTLCB9X@pq0eb2lcLj())CU%E#cB#V{rO}s^=N)y|0aBqYruj@NhGPH_R2%+^Q8J zJ%#(R8hQ`(3)LSOZQj$?Z#UHXjq{(U>@AcsQ&Y_>bALiX76QOWp zi>Y+wq7?WZw+vt1=tx!8%o`HD@JFj-;v+eQ;qD|*Cx~yVxiTrbyQdL#uC&}Ub#?EBdqaB0!5LhnBv}$ff`&cg%xHibyH%Fd zqin3IqIH&)H(JH=Y+P2ab;IYf5Me^~TTH zB1PC-irbSTNMr4e+)lDk;89%N>>l_28p6s8E1MbHUEw{e)m0v7rm{h+FugMw4KG_B z%iQqYO)cAKPb?;Qp$0lIz3hpw&*kXoKAmy*=o>~Zuln8K1Q01ZT~jic{Dps7w0~$C zrsM7m-aha>>P+{M7`lI1P@Lqx+GTNu<|cKD=6Z?IE<@Xl)A9xW-~CW4bUjbtzI14O z==YDC{8JeI*2z~^c+-U5cl{myZ?K#kn0o1+Gk~m*3byZ3HoLp8Q&w)vbBaP<&wZUA zY`Q7A3P4r@O3NzHG8(p9jwYtL&%A;5gvK)xBjO~_@xweBz;m>H&0L!&`XWXz3npJLrb+!qQgLlpCV$Img)hbh+!oUe~8 zbIx`C>Cc3ELC~XU`{yY}kZ4S#WL5rb!7{!i*JRgpD?ZZZZ%lRnd|;KkEZe^Le93Jx z;FsnG+s+xn=QkxsbGMq8H>j|{=RGCmru&DbEc!aJZ3;9}lnob|*-CKXY9mK(HiJgy9ISQi$g6473 zk=%BD*C*S=gH5}whM%u$`4kA35+SZ2`iu7$3;E~XXB)p%Kkyh|q7NLa$Rr`t3PuX> zIsO#V4ecq*O-BYcnf(?xTxTMrk%ewWwy_c;5E91lGA}G+r*C+(+0b;92=R+3Yb