From a4131dfdb98bc3d99ce5ca486c998a92227c2374 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Sun, 18 Oct 2026 20:06:20 -0700 Subject: [PATCH 01/38] Read and write dynamic-tier properties through shapes and inline caches Stage S4 of the two-tier design: dynamic-tier code reads, writes and calls properties through per-site inline caches keyed on hidden-class shapes. - Runtime: scr_js_object.c (dynTier) attaches shapes to the existing checked objects: a shape is a node of the thread's transition tree over key sequences, computed on an object's first cache miss, and objects past 64 keys stay unshaped (dictionary mode). Attributes and the prototype are tested live on every hit. The dyn-tier build of scr_json.c (SCR_JS_TIER) resets a shape on every key-set change; typed programs keep linking the ordinary build. - Sites: ir/js-ic.ts recognizes the lowering's property helpers by structure; calls to them from dynamic-tier functions, and method dispatch outside module initializers, become sites with a zeroed thread-local ScrJsIc. backend/llvm/js-ic.ts tests entry 0 inline (own property or one prototype link), probes out of line, and keeps the helper as the noinline cold slow path. Array length and dense number-key element reads are inline. - Numbers: jsval reads of number properties are immediates, and a jsval number stored over an unshared number cell updates it in place; the jsval pass feeds cache stores jsvals and stops boxing discarded compound-assignment results. `%` and bitwise operators get integer paths. Static functions are untouched: typed programs compile to byte-identical LLVM and executables, and every static function of programs with dynamic code is byte-identical. Co-Authored-By: Claude Opus 5.5 --- packages/compiler/src/backend/llvm/emitter.ts | 20 + .../compiler/src/backend/llvm/expr-calls.ts | 7 + .../compiler/src/backend/llvm/expr-dynamic.ts | 15 +- packages/compiler/src/backend/llvm/js-ic.ts | 779 ++++++++++++++++++ packages/compiler/src/backend/llvm/jsval.ts | 160 +++- .../compiler/src/backend/native/executable.ts | 10 +- .../compiler/src/backend/native/library.ts | 3 +- packages/compiler/src/ir/js-ic.test.ts | 178 ++++ packages/compiler/src/ir/js-ic.ts | 236 ++++++ packages/compiler/src/ir/jsval.ts | 46 +- packages/compiler/src/ir/runtime-features.ts | 13 +- packages/compiler/test/js-ic-emission.test.ts | 92 +++ .../tests/corpus/6148-shape-caches.js.json | 6 + .../corpus/6149-jsval-integer-paths.js.json | 6 + .../runtime-pack-matrix.mjs | 7 + .../runtime-pack-common/library-matrix.mjs | 1 + .../runtime-pack-matrix.mjs | 7 + packages/runtime/src/scr_dyn.h | 23 + packages/runtime/src/scr_js.h | 54 ++ packages/runtime/src/scr_js_object.c | 333 ++++++++ packages/runtime/src/scr_json.c | 32 + tests/corpus/6148-shape-caches.js | 206 +++++ tests/corpus/6149-jsval-integer-paths.js | 45 + 23 files changed, 2256 insertions(+), 23 deletions(-) create mode 100644 packages/compiler/src/backend/llvm/js-ic.ts create mode 100644 packages/compiler/src/ir/js-ic.test.ts create mode 100644 packages/compiler/src/ir/js-ic.ts create mode 100644 packages/compiler/test/js-ic-emission.test.ts create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6148-shape-caches.js.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6149-jsval-integer-paths.js.json create mode 100644 packages/runtime/src/scr_js_object.c create mode 100644 tests/corpus/6148-shape-caches.js create mode 100644 tests/corpus/6149-jsval-integer-paths.js diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 2ca417b7d..33ed91581 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -1,4 +1,6 @@ import { jsTruthy } from "./jsval.js"; +import { JsIcSites } from "../../ir/js-ic.js"; +import { JS_IC_WORDS } from "./js-ic.js"; import { type ArraySearchKind, emitArraySearchHelpers } from "./array-search.js"; import { emitStringInputs } from "./string-lifetimes.js"; import { emitLlvmLayouts } from "./layouts.js"; @@ -910,6 +912,12 @@ export class LlEmitter { /** Constant-key reads on checked-dynamic objects, each with its own * zero-initialized ScrDynReadIc (scr_runtime.h). */ private dynReadIcSites = 0; + /** Inline-cache sites of dynamic-tier functions (js-ic.ts): one + * zero-initialized ScrJsIc per site (scr_js.h). */ + private jsIcSiteCount = 0; + private jsIcSitesMemo: JsIcSites | undefined; + /** The tier of the function being emitted. */ + currentTier: IrFunction["tier"] = undefined; /** May-throw analysis (computeMayThrow): pending * checks are emitted only after calls that can actually raise. */ readonly mayThrow: Set; @@ -2507,6 +2515,8 @@ export class LlEmitter { out.push( `@sc_dyn_read_ic_${site} = internal ${tl}global { ptr, ${this.sizeType}, ${this.sizeType}, ptr } zeroinitializer`, ); + for (let site = 0; site < this.jsIcSiteCount; site++) + out.push(`@sc_js_ic_${site} = internal ${tl}global [${JS_IC_WORDS} x i64] zeroinitializer`); } for (const table of this.constantNumericTables.values()) { const n = table.values.length; @@ -6763,6 +6773,7 @@ export class LlEmitter { const B = new BlockBuilder(); this.B = B; this.currentFunctionName = fn.name; + this.currentTier = fn.tier; // Fiber switches restore the active exception cell before control // returns to a synchronous frame (emitPendingCheck). B.excCellInvariant = fn.async !== true && fn.generator === undefined; @@ -11077,6 +11088,15 @@ export class LlEmitter { } /** A fresh read-site cache for one constant-key dyn read. */ + /** The helper classification of this module's inline-cache sites. */ + get jsIcSites(): JsIcSites { + return (this.jsIcSitesMemo ??= new JsIcSites(this.mod)); + } + + jsIcSite(): string { + return `@sc_js_ic_${this.jsIcSiteCount++}`; + } + dynReadIcSite(): string { return `@sc_dyn_read_ic_${this.dynReadIcSites++}`; } diff --git a/packages/compiler/src/backend/llvm/expr-calls.ts b/packages/compiler/src/backend/llvm/expr-calls.ts index b1d810a3f..5c5fa5083 100644 --- a/packages/compiler/src/backend/llvm/expr-calls.ts +++ b/packages/compiler/src/backend/llvm/expr-calls.ts @@ -51,6 +51,7 @@ import { inputSuffixWrites, } from "./borrowed-inputs.js"; import type { Writes } from "./reference-writes.js"; +import { emitJsIcCall, jsIcSiteOf } from "./js-ic.js"; import { emitClassMembership, emitDowncastGuard } from "./narrow-guards.js"; export function emitCallExpr( @@ -86,6 +87,12 @@ export function emitCallExpr( case "call": { const callee = host.fnByName.get(e.callee); if (!callee) throw new InternalCompilerError(`llvm emitter bug: unknown callee ${e.callee}`); + if (!plainCall && !awaitEntry && !resultSlot && !stackResult) { + // A property helper called from dynamic-tier code is an inline + // cache site (two-tier design S4). + const site = jsIcSiteOf(host, e); + if (site) return emitJsIcCall(host, e, callee, site); + } if (callee.fastRead === "index" && !plainCall && !awaitEntry && !resultSlot) { const indexed = emitNumericIndexRead(host, e, callee); if (indexed) return indexed; diff --git a/packages/compiler/src/backend/llvm/expr-dynamic.ts b/packages/compiler/src/backend/llvm/expr-dynamic.ts index 76df9795d..60bfc9bc4 100644 --- a/packages/compiler/src/backend/llvm/expr-dynamic.ts +++ b/packages/compiler/src/backend/llvm/expr-dynamic.ts @@ -28,6 +28,7 @@ import { elemAccess, vAdapters } from "./shapes.js"; import { LlvmUnsupportedError } from "./unsupported.js"; import type { LlvmEmitterContext, ExprOf, LlValue } from "./expr-context.js"; import { emitUnionWiden } from "./expr-records.js"; +import { emitJsIcMethod, jsIcMethods } from "./js-ic.js"; export function emitDynamicExpr( host: LlvmEmitterContext, @@ -206,10 +207,16 @@ export function emitDynamicExpr( // pending exception. const recv = emitBorrowedInput(host, e.recv); host.declare(`declare ptr @scr_dyn_prepare_method(ptr, ptr)`); - const prepared = B.tmp(); - B.line( - `${prepared} = call ptr @scr_dyn_prepare_method(ptr ${recv.name}, ptr ${host.cstr(e.method)})`, - ); + let prepared: string; + if (jsIcMethods(host)) { + // Dynamic-tier code reads the method through the site's cache. + prepared = emitJsIcMethod(host, e, recv.name); + } else { + prepared = B.tmp(); + B.line( + `${prepared} = call ptr @scr_dyn_prepare_method(ptr ${recv.name}, ptr ${host.cstr(e.method)})`, + ); + } host.own({ name: prepared, type: { kind: "dyn" } }); host.emitPendingCheck(); const args = e.args.map((a) => emitBorrowedInput(host, a)); diff --git a/packages/compiler/src/backend/llvm/js-ic.ts b/packages/compiler/src/backend/llvm/js-ic.ts new file mode 100644 index 000000000..5e6d32d22 --- /dev/null +++ b/packages/compiler/src/backend/llvm/js-ic.ts @@ -0,0 +1,779 @@ +/* Inline-cache lowering of the dynamic tier (two-tier design stage S4; + * runtime scr_js_object.c, sites ir/js-ic.ts). + * + * A property site in a dynamic-tier function owns one zeroed thread-local + * ScrJsIc record (`@sc_js_ic_N`). The code at the site is fixed: + * + * load ordinary object whose shape is entry 0's own-property shape: + * read entry `slot` inline unless it is an accessor; otherwise + * scr_js_load_probe (polymorphic, prototype hits, misses) + * store ordinary object whose shape is entry 0's replace shape, outside + * an intact parse region, writable data entry: replace inline; + * otherwise scr_js_store_probe (adds, polymorphic, misses) + * keyed dense array element at an integral number key: inline; + * otherwise scr_js_keyed_probe (string keys of ordinary objects) + * method dynInvoke's method read: scr_js_load_probe + * + * A probe that cannot answer from plain data leaves everything untouched, + * and the site calls its helper (or the generic runtime entry), which + * defines the operation's semantics; nothing is ever patched. + * + * Layout facts (64-bit only; 32-bit targets keep the helper calls): + * ScrDyn kind +8; v.obj {len +16, entries +32, region +72, + * shape +80}; v.arr {len +16, items +32, presence +56}; + * v.num +16 + * ScrDynEntry 48 bytes: value +16, accessor bit 0 of +40, writable +41 + * ScrJsIc entries of 40 bytes {shape, target, holder, holder_shape, + * slot}; 168 bytes in all */ +import { BOOL, DYN, F64, STRING, type IrExpr, type IrFunction } from "../../ir/ir.js"; +import { jsIcMethodSites, type JsIcSite } from "../../ir/js-ic.js"; +import { mangleBorrowedFunction } from "../mangle.js"; +import { emitBorrowedInput } from "./borrowed-inputs.js"; +import { DYN_KIND } from "./dyn.js"; +import type { ExprOf, LlValue, LlvmEmitterContext } from "./expr-context.js"; +import { JS_BITS } from "./jsval-encoding.js"; +import { jsImmediates } from "./jsval.js"; + +type Host = LlvmEmitterContext; + +/** Words of one ScrJsIc record. */ +export const JS_IC_WORDS = 21; + +const ENTRY_STRIDE = 48; + +/** The cache site a call is, when it is emitted in a dynamic-tier function + * of a 64-bit target. */ +export function jsIcSiteOf(host: Host, e: ExprOf<"call">): JsIcSite | null { + if (!jsImmediates(host) || host.currentTier !== "dynamic") return null; + if (host.int32Abis.has(e.callee)) return null; + return host.jsIcSites.site(e.callee); +} + +/** Whether method dispatch in the current function reads through a cache. */ +export function jsIcMethods(host: Host): boolean { + return ( + jsImmediates(host) && + host.currentTier === "dynamic" && + jsIcMethodSites(host.currentFunctionName) + ); +} + +function gep(host: Host, base: string, offset: number): string { + const p = host.B.tmp(); + host.B.line(`${p} = getelementptr inbounds i8, ptr ${base}, i64 ${offset}`); + return p; +} + +function load(host: Host, type: string, base: string, offset: number): string { + const v = host.B.tmp(); + host.B.line(`${v} = load ${type}, ptr ${offset === 0 ? base : gep(host, base, offset)}`); + return v; +} + +/** Branches to `yes` when the dyn cell `d` has kind `kind`. */ +function kindIs(host: Host, d: string, kind: number, yes: string, no: string): void { + const k = load(host, "i32", d, 8); + const is = host.B.tmp(); + host.B.line(`${is} = icmp eq i32 ${k}, ${kind}`); + host.B.condBr(is, yes, no); +} + +/* How many cache entries a site tests inline, and whether those tests + * cover prototype hits. Measured (two-tier S4): testing all four entries + * inline made polymorphic reads 7-10% faster (the design prototype's W2/W3) + * and monomorphic method code no faster, for 65-225 more bytes per site; + * the prototype hit made method-heavy code 3-6% faster for 15-90 bytes. + * So a site tests entry 0, own or prototype, and its probe does the rest. + * (0 makes every access probe: with a runtime built with SCR_JS_VERIFY, + * that checks every cached shape against its object.) */ +const INLINE_WAYS: number = 1; +const INLINE_PROTO = true; + +/** The address of entry `slot` of object `d`'s entries. */ +function entryAt(host: Host, d: string, slot: string): string { + const B = host.B; + const entries = load(host, "ptr", d, 32); + const offset = B.tmp(); + const entry = B.tmp(); + B.line(`${offset} = mul i64 ${slot}, ${ENTRY_STRIDE}`); + B.line(`${entry} = getelementptr inbounds i8, ptr ${entries}, i64 ${offset}`); + return entry; +} + +/** Branches to `miss` when the entry at `entry` is an accessor. */ +function dataEntry(host: Host, entry: string, miss: string): void { + const B = host.B; + const flags = load(host, "i8", entry, 40); + const accessor = B.tmp(); + const isAccessor = B.tmp(); + B.line(`${accessor} = and i8 ${flags}, 1`); + B.line(`${isAccessor} = icmp ne i8 ${accessor}, 0`); + const data = B.newLabel("ic.data"); + B.condBr(isAccessor, miss, data); + B.startBlock(data); +} + +/** The inline part of a load site, entered for an ordinary object `d`: + * tests the first `ways` cache entries (own properties, and prototype hits + * when `proto`); each hit converts the (borrowed) value with `result` and + * branches to `join`, recording its phi input. Anything else branches to + * `miss`. */ +function inlineLoad( + host: Host, + d: string, + ic: string, + ways: number, + proto: boolean, + result: (cell: string) => string, + incoming: string[], + join: string, + miss: string, +): void { + const B = host.B; + if (ways === 0) { + B.br(miss); + return; + } + const shape = load(host, "ptr", d, 80); + const present = B.tmp(); + B.line(`${present} = icmp ne ptr ${shape}, null`); + let next = B.newLabel("ic.w"); + B.condBr(present, next, miss); + for (let way = 0; way < ways; way++) { + const base = way * 40; + B.startBlock(next); + next = way + 1 < ways ? B.newLabel("ic.w") : miss; + const cached = load(host, "ptr", ic, base); + const same = B.tmp(); + B.line(`${same} = icmp eq ptr ${shape}, ${cached}`); + const matched = B.newLabel("ic.m"); + B.condBr(same, matched, next); + B.startBlock(matched); + const target = load(host, "ptr", ic, base + 8); + const own = B.tmp(); + B.line(`${own} = icmp eq ptr ${target}, null`); + const ownLabel = B.newLabel("ic.own"); + const protoLabel = proto ? B.newLabel("ic.proto") : miss; + B.condBr(own, ownLabel, protoLabel); + const hit = (holder: string): void => { + const slot = load(host, "i64", ic, base + 32); + const entry = entryAt(host, holder, slot); + dataEntry(host, entry, miss); + const value = load(host, "ptr", entry, 16); + const out = result(value); + incoming.push(`[ ${out}, %${B.currentLabel()} ]`); + B.br(join); + }; + B.startBlock(ownLabel); + hit(d); + if (!proto) continue; + B.startBlock(protoLabel); + const holder = load(host, "ptr", ic, base + 16); + const prototype = load(host, "ptr", d, 96); + const linked = B.tmp(); + B.line(`${linked} = icmp eq ptr ${prototype}, ${holder}`); + const isLinked = B.newLabel("ic.pl"); + B.condBr(linked, isLinked, miss); + B.startBlock(isLinked); + const holderObject = B.newLabel("ic.po"); + kindIs(host, holder, DYN_KIND.OBJ, holderObject, miss); + B.startBlock(holderObject); + const holderShape = load(host, "ptr", holder, 80); + const current = B.tmp(); + B.line(`${current} = icmp eq ptr ${holderShape}, ${target}`); + const holderHit = B.newLabel("ic.ph"); + B.condBr(current, holderHit, miss); + B.startBlock(holderHit); + hit(holder); + } +} + +/** The inline part of a store site that replaces an own data property, + * entered for an ordinary object `d` outside an intact parse region: + * tests the first `ways` cache entries; a hit calls `store` with the + * address of the entry's value word, which must end its block by + * branching to the site's join. */ +function inlineReplace( + host: Host, + d: string, + ic: string, + ways: number, + store: (slot: string) => void, + miss: string, +): void { + const B = host.B; + if (ways === 0) { + B.br(miss); + return; + } + const shape = load(host, "ptr", d, 80); + const present = B.tmp(); + B.line(`${present} = icmp ne ptr ${shape}, null`); + let next = B.newLabel("ic.w"); + B.condBr(present, next, miss); + for (let way = 0; way < ways; way++) { + const base = way * 40; + B.startBlock(next); + next = way + 1 < ways ? B.newLabel("ic.w") : miss; + const cached = load(host, "ptr", ic, base); + const target = load(host, "ptr", ic, base + 8); + const same = B.tmp(); + const replace = B.tmp(); + const ok = B.tmp(); + B.line(`${same} = icmp eq ptr ${shape}, ${cached}`); + B.line(`${replace} = icmp eq ptr ${target}, ${shape}`); + B.line(`${ok} = and i1 ${same}, ${replace}`); + const matched = B.newLabel("ic.m"); + B.condBr(ok, matched, next); + B.startBlock(matched); + const slot = load(host, "i64", ic, base + 32); + const entry = entryAt(host, d, slot); + const flags = load(host, "i8", entry, 40); + const writable = load(host, "i8", entry, 41); + const accessor = B.tmp(); + const data = B.tmp(); + const canWrite = B.tmp(); + const writes = B.tmp(); + B.line(`${accessor} = and i8 ${flags}, 1`); + B.line(`${data} = icmp eq i8 ${accessor}, 0`); + B.line(`${canWrite} = icmp ne i8 ${writable}, 0`); + B.line(`${writes} = and i1 ${data}, ${canWrite}`); + const hit = B.newLabel("ic.st"); + B.condBr(writes, hit, miss); + B.startBlock(hit); + store(gep(host, entry, 16)); + } +} + +/** A property value (a dyn cell, `owned` or borrowed from its entry) as + * the site's result type, owned. A jsval result reads a number cell as an + * immediate, so arithmetic on it takes no reference and a later store can + * update an unshared number cell in place; any other cell is carried under + * the cell tag. */ +function asResult(host: Host, cell: string, e: IrExpr, owned: boolean | string): string { + const B = host.B; + /** Retains the cell unless the caller owns it. */ + const hold = (): string => { + if (owned === true) return cell; + if (owned === false) return host.retainValue(cell, DYN); + const borrowed = B.newLabel("icr.b"); + const done = B.newLabel("icr.h"); + const from = B.currentLabel(); + B.condBr(owned, done, borrowed); + B.startBlock(borrowed); + const retained = host.retainValue(cell, DYN); + const retainedEnd = B.currentLabel(); + B.br(done); + B.startBlock(done); + const held = B.tmp(); + B.line(`${held} = phi ptr [ ${cell}, %${from} ], [ ${retained}, %${retainedEnd} ]`); + return held; + }; + if (e.type.kind !== "jsval") return hold(); + const number = B.newLabel("icr.num"); + const other = B.newLabel("icr.cell"); + const join = B.newLabel("icr.j"); + kindIs(host, cell, DYN_KIND.NUM, number, other); + B.startBlock(number); + const d = load(host, "double", cell, 16); + const raw = B.tmp(); + const nan = B.tmp(); + const bits = B.tmp(); + B.line(`${raw} = bitcast double ${d} to i64`); + B.line(`${nan} = fcmp uno double ${d}, ${d}`); + B.line(`${bits} = select i1 ${nan}, i64 ${JS_BITS.canonicalNan}, i64 ${raw}`); + const immediate = B.tmp(); + B.line(`${immediate} = inttoptr i64 ${bits} to ptr`); + if (owned === true) { + host.releaseValue(cell, DYN); + } else if (owned !== false) { + const release = B.newLabel("icr.r"); + const released = B.newLabel("icr.rd"); + B.condBr(owned, release, released); + B.startBlock(release); + host.releaseValue(cell, DYN); + B.br(released); + B.startBlock(released); + } + const numberEnd = B.currentLabel(); + B.br(join); + B.startBlock(other); + const held = hold(); + const cellBits = B.tmp(); + const word = B.tmp(); + const tagged = B.tmp(); + B.line(`${cellBits} = ptrtoint ptr ${held} to i64`); + B.line(`${word} = or i64 ${cellBits}, ${JS_BITS.cellTag}`); + B.line(`${tagged} = inttoptr i64 ${word} to ptr`); + const otherEnd = B.currentLabel(); + B.br(join); + B.startBlock(join); + const out = B.tmp(); + B.line(`${out} = phi ptr [ ${immediate}, %${numberEnd} ], [ ${tagged}, %${otherEnd} ]`); + return out; +} + +/** The helper call of a site's slow path, with the site's evaluated + * (borrowed) arguments. Ordinary parameters take a reference; borrowed + * ones lend it. Returns the owned result, or "" for a void helper. */ +function helperCall(host: Host, e: ExprOf<"call">, callee: IrFunction, args: LlValue[]): string { + const B = host.B; + const borrowed = host.callLifetimes.borrowed.get(e.callee); + const argList = args + .map((a, i) => { + const type = callee.params[i]!.type; + const value = borrowed?.has(i) ? a.name : host.retainValue(a.name, a.type); + return `${host.llType(type)} ${value}`; + }) + .join(", "); + const target = `@${borrowed ? mangleBorrowedFunction(e.callee) : host.callTarget(e.callee)}`; + // The helper is the site's cold path: keep its body out of line. + if (e.type.kind === "void") { + B.line(`call void ${target}(${argList}) noinline cold`); + if (host.mayThrow.has(e.callee)) host.emitPendingCheck(); + return ""; + } + let t = B.tmp(); + B.line(`${t} = call ${host.llType(e.type)} ${target}(${argList}) noinline cold`); + if (host.mayThrow.has(e.callee)) host.emitPendingCheck(); + if (borrowed && host.borrowedReturns.has(e.callee)) t = host.retainValue(t, e.type); + return t; +} + +function keyString(host: Host, key: string, e: IrExpr): string { + return host.emitExpr({ kind: "strLit", value: key, type: STRING, loc: e.loc }).name; +} + +/** Emits a call to a classified helper as a cache site. */ +export function emitJsIcCall( + host: Host, + e: ExprOf<"call">, + callee: IrFunction, + site: JsIcSite, +): LlValue { + switch (site.kind) { + case "get": + return emitGet(host, e, callee, site.key); + case "set": + return emitSet(host, e, callee, site.key); + case "keyed": + return emitKeyed(host, e, callee); + } +} + +function emitGet(host: Host, e: ExprOf<"call">, callee: IrFunction, key: string): LlValue { + const B = host.B; + const receiver = emitBorrowedInput(host, e.args[0]!); + const name = keyString(host, key, e); + const ic = host.jsIcSite(); + const object = B.newLabel("icg.obj"); + const probe = B.newLabel("icg.probe"); + const slow = B.newLabel("icg.slow"); + const join = B.newLabel("icg.j"); + const incoming: string[] = []; + if (key === "length") { + // An array's length is its element count (the generic read answers + // it before any property table). + const array = B.newLabel("icg.arr"); + const other = B.newLabel("icg.other"); + kindIs(host, receiver.name, DYN_KIND.ARR, array, other); + B.startBlock(array); + const len = load(host, "i64", receiver.name, 16); + const count = B.tmp(); + B.line(`${count} = uitofp i64 ${len} to double`); + incoming.push(`[ ${numberResult(host, count, e)}, %${B.currentLabel()} ]`); + B.br(join); + B.startBlock(other); + } + // Inline hits (borrowed values) and probe hits (owned ones) share one + // conversion to the result type. + const hit = B.newLabel("icg.hit"); + const hits: string[] = []; + kindIs(host, receiver.name, DYN_KIND.OBJ, object, probe); + B.startBlock(object); + inlineLoad(host, receiver.name, ic, INLINE_WAYS, INLINE_PROTO, (cell) => cell, hits, hit, probe); + B.startBlock(probe); + host.declare(`declare ptr @scr_js_load_probe(ptr, ptr, ptr)`); + const probed = B.tmp(); + B.line(`${probed} = call ptr @scr_js_load_probe(ptr ${receiver.name}, ptr ${name}, ptr ${ic})`); + const found = B.tmp(); + B.line(`${found} = icmp ne ptr ${probed}, null`); + const probeEnd = B.currentLabel(); + B.condBr(found, hit, slow); + B.startBlock(hit); + const cell = B.tmp(); + const owned = B.tmp(); + const ownedIn = hits.map((h) => h.replace(/^\[ [^,]+,/, "[ false,")); + B.line(`${cell} = phi ptr ${[...hits, `[ ${probed}, %${probeEnd} ]`].join(", ")}`); + B.line(`${owned} = phi i1 ${[...ownedIn, `[ true, %${probeEnd} ]`].join(", ")}`); + { + const out = asResult(host, cell, e, owned); + incoming.push(`[ ${out}, %${B.currentLabel()} ]`); + } + B.br(join); + B.startBlock(slow); + const result = helperCall(host, e, callee, [receiver]); + incoming.push(`[ ${result}, %${B.currentLabel()} ]`); + B.br(join); + B.startBlock(join); + const out = B.tmp(); + B.line(`${out} = phi ptr ${incoming.join(", ")}`); + return host.own({ name: out, type: e.type }); +} + +/** A number as the site's result type: an immediate for a jsval result, + * a fresh cell (as the generic read makes) for a dyn one. */ +function numberResult(host: Host, d: string, e: IrExpr): string { + const B = host.B; + if (e.type.kind === "jsval") { + // Counts are never NaN: the bits are the word. + const bits = B.tmp(); + const word = B.tmp(); + B.line(`${bits} = bitcast double ${d} to i64`); + B.line(`${word} = inttoptr i64 ${bits} to ptr`); + return word; + } + host.declare(`declare ptr @scr_dyn_new_num(double)`); + const cell = B.tmp(); + B.line(`${cell} = call ptr @scr_dyn_new_num(double ${d})`); + return cell; +} + +function emitSet(host: Host, e: ExprOf<"call">, callee: IrFunction, key: string): LlValue { + const B = host.B; + const receiver = emitBorrowedInput(host, e.args[0]!); + // A value the jsval pass left as `js.toDyn(x)` is stored from the jsval + // itself: a number replaces the value of an unshared number cell in + // place instead of boxing a new one. + const stored = e.args[1]!; + const word = + stored.kind === "libCall" && stored.fn === "js.toDyn" + ? host.emitExpr(stored.args[0]!).name + : null; + const value = word === null ? emitBorrowedInput(host, stored) : null; + const name = keyString(host, key, e); + const ic = host.jsIcSite(); + const object = B.newLabel("ics.obj"); + const probe = B.newLabel("ics.probe"); + const slow = B.newLabel("ics.slow"); + const join = B.newLabel("ics.j"); + // The cell the probe and the helper store when the value is a jsval; + // the statement's frame releases it. + let boxedSlot: string | null = null; + if (word !== null) { + boxedSlot = B.slot(); + B.entryAllocas.push(`${boxedSlot} = alloca ptr`); + B.line(`store ptr null, ptr ${boxedSlot}`); + host.ownSlot(boxedSlot, DYN); + } + kindIs(host, receiver.name, DYN_KIND.OBJ, object, probe); + B.startBlock(object); + const region = load(host, "ptr", receiver.name, 72); + const outside = B.tmp(); + B.line(`${outside} = icmp eq ptr ${region}, null`); + const unshared = B.newLabel("ics.r"); + B.condBr(outside, unshared, probe); + B.startBlock(unshared); + host.declare(`declare ptr @scr_js_to_dyn_borrow(ptr)`); + inlineReplace( + host, + receiver.name, + ic, + INLINE_WAYS, + (slot) => { + const old = B.tmp(); + B.line(`${old} = load ptr, ptr ${slot}`); + if (word === null) { + B.line(`store ptr ${host.retainValue(value!.name, DYN)}, ptr ${slot}`); + host.releaseValue(old, DYN); + B.br(join); + return; + } + const bits = B.tmp(); + const isDouble = B.tmp(); + B.line(`${bits} = ptrtoint ptr ${word} to i64`); + B.line(`${isDouble} = icmp ult i64 ${bits}, ${JS_BITS.limit}`); + const number = B.newLabel("ics.dbl"); + const box = B.newLabel("ics.box"); + B.condBr(isDouble, number, box); + B.startBlock(number); + const oldKind = load(host, "i32", old, 8); + const oldRc = load(host, "i64", old, 0); + const isNumber = B.tmp(); + const unshared = B.tmp(); + const inPlace = B.tmp(); + B.line(`${isNumber} = icmp eq i32 ${oldKind}, ${DYN_KIND.NUM}`); + B.line(`${unshared} = icmp eq i64 ${oldRc}, 1`); + B.line(`${inPlace} = and i1 ${isNumber}, ${unshared}`); + const update = B.newLabel("ics.inplace"); + B.condBr(inPlace, update, box); + B.startBlock(update); + const d = B.tmp(); + B.line(`${d} = bitcast i64 ${bits} to double`); + B.line(`store double ${d}, ptr ${gep(host, old, 16)}`); + B.br(join); + B.startBlock(box); + const cell = B.tmp(); + B.line(`${cell} = call ptr @scr_js_to_dyn_borrow(ptr ${word})`); + B.line(`store ptr ${cell}, ptr ${slot}`); + host.releaseValue(old, DYN); + B.br(join); + }, + probe, + ); + B.startBlock(probe); + let cell: string; + if (word !== null) { + cell = B.tmp(); + B.line(`${cell} = call ptr @scr_js_to_dyn_borrow(ptr ${word})`); + B.line(`store ptr ${cell}, ptr ${boxedSlot}`); + } else { + cell = value!.name; + } + host.declare(`declare zeroext i1 @scr_js_store_probe(ptr, ptr, ptr, ptr)`); + const done = B.tmp(); + B.line( + `${done} = call zeroext i1 @scr_js_store_probe(ptr ${receiver.name}, ptr ${name}, ptr ${cell}, ptr ${ic})`, + ); + B.condBr(done, join, slow); + B.startBlock(slow); + helperCall(host, e, callee, [receiver, { name: cell, type: DYN }]); + B.br(join); + B.startBlock(join); + return { name: "", type: e.type }; +} + +/** A keyed site's key when it is a static number boxed for the helper + * (`dynFrom(n)`, or `dynFrom(u)` of a local union with a number arm): the + * site indexes with the double and boxes the key only on a miss. */ +interface NumberKey { + /** i1: whether `value` holds the key (always for a static number). */ + present: string; + value: string; + /** Emits the key's cell (+1), exactly as the helper argument boxes it. */ + box: () => string; +} + +function numberKey(host: Host, key: IrExpr): NumberKey | null { + if (key.kind !== "dynFrom" || key.liveRef !== undefined || key.fnName !== undefined) return null; + const source = key.value; + if (source.type.kind === "f64") { + const value = host.emitExpr(source).name; + // Request the converter where the boxing it replaces did: converters + // are numbered in request order. + const conv = host.dyn.toDynHelper(source.type); + return { + present: "true", + value, + box: () => { + const cell = host.B.tmp(); + host.B.line(`${cell} = call ptr @${conv}(double ${value})`); + return cell; + }, + }; + } + if (source.type.kind !== "union" || source.kind !== "varRef") return null; + const unionId = source.type.unionId; + const tag = host.unionsById.get(unionId)?.arms.findIndex((arm) => arm.kind === "f64") ?? -1; + if (tag < 0) return null; + const B = host.B; + const present = host.emitExpr({ + kind: "unionIsTag", + unionId, + tag, + negated: false, + value: source, + type: BOOL, + loc: key.loc, + }).name; + const slot = B.slot(); + B.entryAllocas.push(`${slot} = alloca double`); + B.line(`store double 0.0, ptr ${slot}`); + const number = B.newLabel("ick.k"); + const done = B.newLabel("ick.kd"); + B.condBr(present, number, done); + B.startBlock(number); + // Temps of a conditional block are released in it. + host.frames.push([]); + const d = host.emitExpr({ + kind: "unionNarrow", + unionId, + tag, + value: source, + type: F64, + loc: key.loc, + }); + B.line(`store double ${d.name}, ptr ${slot}`); + host.releaseFrame(host.frames.pop()!); + B.br(done); + B.startBlock(done); + const value = B.tmp(); + B.line(`${value} = load double, ptr ${slot}`); + return { + present, + value, + box: () => { + // The local is a plain read: boxing it again evaluates nothing else. + host.frames.push([]); + const boxed = host.emitExpr(key); + host.moveTemp(boxed); + host.releaseFrame(host.frames.pop()!); + return boxed.name; + }, + }; +} + +function emitKeyed(host: Host, e: ExprOf<"call">, callee: IrFunction): LlValue { + const B = host.B; + const receiver = emitBorrowedInput(host, e.args[0]!); + const keyExpr = e.args[1]!; + const numeric = numberKey(host, keyExpr); + const key = numeric === null ? emitBorrowedInput(host, keyExpr) : null; + const ic = host.jsIcSite(); + const array = B.newLabel("ick.arr"); + const number = B.newLabel("ick.num"); + const probe = B.newLabel("ick.probe"); + const slow = B.newLabel("ick.slow"); + const join = B.newLabel("ick.j"); + const incoming: string[] = []; + // The boxed key of the miss paths when the site indexes with a double; + // the statement's frame releases it. + let keySlot: string | null = null; + if (numeric !== null) { + keySlot = B.slot(); + B.entryAllocas.push(`${keySlot} = alloca ptr`); + B.line(`store ptr null, ptr ${keySlot}`); + host.ownSlot(keySlot, DYN); + } + kindIs(host, receiver.name, DYN_KIND.ARR, array, probe); + B.startBlock(array); + let index: string; + if (numeric === null) { + kindIs(host, key!.name, DYN_KIND.NUM, number, probe); + B.startBlock(number); + index = load(host, "double", key!.name, 16); + } else { + B.condBr(numeric.present, number, probe); + B.startBlock(number); + index = numeric.value; + } + const len = load(host, "i64", receiver.name, 16); + const i = B.tmp(); + const back = B.tmp(); + const integral = B.tmp(); + const inRange = B.tmp(); + const nonNegative = B.tmp(); + const ok1 = B.tmp(); + const ok2 = B.tmp(); + // fptoui of a negative or out-of-range double is poison: test the + // double's range before trusting the integer. + B.line(`${nonNegative} = fcmp oge double ${index}, 0.0`); + const lenD = B.tmp(); + B.line(`${lenD} = uitofp i64 ${len} to double`); + B.line(`${inRange} = fcmp olt double ${index}, ${lenD}`); + B.line(`${ok1} = and i1 ${nonNegative}, ${inRange}`); + const ranged = B.newLabel("ick.rng"); + B.condBr(ok1, ranged, probe); + B.startBlock(ranged); + B.line(`${i} = fptoui double ${index} to i64`); + B.line(`${back} = uitofp i64 ${i} to double`); + B.line(`${integral} = fcmp oeq double ${back}, ${index}`); + const presence = load(host, "ptr", receiver.name, 56); + const dense = B.tmp(); + B.line(`${dense} = icmp eq ptr ${presence}, null`); + B.line(`${ok2} = and i1 ${integral}, ${dense}`); + const hit = B.newLabel("ick.hit"); + B.condBr(ok2, hit, probe); + B.startBlock(hit); + const items = load(host, "ptr", receiver.name, 32); + const itemPtr = B.tmp(); + B.line(`${itemPtr} = getelementptr inbounds ptr, ptr ${items}, i64 ${i}`); + const item = B.tmp(); + B.line(`${item} = load ptr, ptr ${itemPtr}`); + const itemEnd = B.currentLabel(); + const shared = B.newLabel("ick.shared"); + B.br(shared); + B.startBlock(probe); + let keyCell: string; + if (numeric === null) { + keyCell = key!.name; + } else { + keyCell = numeric.box(); + B.line(`store ptr ${keyCell}, ptr ${keySlot}`); + } + host.declare(`declare ptr @scr_js_keyed_probe(ptr, ptr, ptr)`); + const probed = B.tmp(); + B.line( + `${probed} = call ptr @scr_js_keyed_probe(ptr ${receiver.name}, ptr ${keyCell}, ptr ${ic})`, + ); + const found = B.tmp(); + B.line(`${found} = icmp ne ptr ${probed}, null`); + const probeEnd = B.currentLabel(); + B.condBr(found, shared, slow); + B.startBlock(shared); + { + const cell = B.tmp(); + const owned = B.tmp(); + B.line(`${cell} = phi ptr [ ${item}, %${itemEnd} ], [ ${probed}, %${probeEnd} ]`); + B.line(`${owned} = phi i1 [ false, %${itemEnd} ], [ true, %${probeEnd} ]`); + const out = asResult(host, cell, e, owned); + incoming.push(`[ ${out}, %${B.currentLabel()} ]`); + } + B.br(join); + B.startBlock(slow); + const result = helperCall(host, e, callee, [receiver, { name: keyCell, type: DYN }]); + incoming.push(`[ ${result}, %${B.currentLabel()} ]`); + B.br(join); + B.startBlock(join); + const out = B.tmp(); + B.line(`${out} = phi ptr ${incoming.join(", ")}`); + return host.own({ name: out, type: e.type }); +} + +/** dynInvoke's method read on an ordinary object through the site's + * cache, else scr_dyn_prepare_method. Returns the owned method (or NULL, + * which scr_dyn_invoke_prepared answers with the full dispatch). */ +export function emitJsIcMethod(host: Host, e: ExprOf<"dynInvoke">, receiver: string): string { + const B = host.B; + const method = e.method; + const ic = host.jsIcSite(); + const name = keyString(host, method, e); + host.declare(`declare ptr @scr_js_load_probe(ptr, ptr, ptr)`); + host.declare(`declare ptr @scr_dyn_prepare_method(ptr, ptr)`); + const object = B.newLabel("icm.obj"); + const probe = B.newLabel("icm.probe"); + const slow = B.newLabel("icm.slow"); + const join = B.newLabel("icm.j"); + const incoming: string[] = []; + kindIs(host, receiver, DYN_KIND.OBJ, object, probe); + B.startBlock(object); + inlineLoad( + host, + receiver, + ic, + INLINE_WAYS, + INLINE_PROTO, + (cell) => host.retainValue(cell, DYN), + incoming, + join, + probe, + ); + B.startBlock(probe); + const probed = B.tmp(); + B.line(`${probed} = call ptr @scr_js_load_probe(ptr ${receiver}, ptr ${name}, ptr ${ic})`); + const found = B.tmp(); + B.line(`${found} = icmp ne ptr ${probed}, null`); + incoming.push(`[ ${probed}, %${B.currentLabel()} ]`); + B.condBr(found, join, slow); + B.startBlock(slow); + const prepared = B.tmp(); + B.line( + `${prepared} = call ptr @scr_dyn_prepare_method(ptr ${receiver}, ptr ${host.cstr(method)}) cold`, + ); + incoming.push(`[ ${prepared}, %${B.currentLabel()} ]`); + B.br(join); + B.startBlock(join); + const out = B.tmp(); + B.line(`${out} = phi ptr ${incoming.join(", ")}`); + return out; +} diff --git a/packages/compiler/src/backend/llvm/jsval.ts b/packages/compiler/src/backend/llvm/jsval.ts index 8c955f6ee..de96c428e 100644 --- a/packages/compiler/src/backend/llvm/jsval.ts +++ b/packages/compiler/src/backend/llvm/jsval.ts @@ -205,6 +205,105 @@ function call(host: Host, ret: string, fn: string, args: [string, string][]): st return r; } +/** `x % y` on two doubles: an integer remainder when both are integers of + * magnitude below 2^31 and y is nonzero (the result keeps the dividend's + * sign, so a zero result of a negative dividend, including -0, is -0), + * else frem (libm's fmod). */ +function remainder(host: Host, x: string, y: string): string { + const B = host.B; + host.declare(`declare double @llvm.fabs.f64(double)`); + const ax = B.tmp(); + const ay = B.tmp(); + const smallX = B.tmp(); + const smallY = B.tmp(); + const small = B.tmp(); + B.line(`${ax} = call double @llvm.fabs.f64(double ${x})`); + B.line(`${ay} = call double @llvm.fabs.f64(double ${y})`); + B.line(`${smallX} = fcmp olt double ${ax}, 2147483648.0`); + B.line(`${smallY} = fcmp olt double ${ay}, 2147483648.0`); + B.line(`${small} = and i1 ${smallX}, ${smallY}`); + const ints = B.newLabel("js.rem.i"); + const exact = B.newLabel("js.rem.x"); + const float = B.newLabel("js.rem.f"); + const join = B.newLabel("js.rem.j"); + B.condBr(small, ints, float); + B.startBlock(ints); + const xi = B.tmp(); + const yi = B.tmp(); + const xb = B.tmp(); + const yb = B.tmp(); + const xInt = B.tmp(); + const yInt = B.tmp(); + const nonzero = B.tmp(); + const both = B.tmp(); + const ok = B.tmp(); + B.line(`${xi} = fptosi double ${x} to i64`); + B.line(`${yi} = fptosi double ${y} to i64`); + B.line(`${xb} = sitofp i64 ${xi} to double`); + B.line(`${yb} = sitofp i64 ${yi} to double`); + B.line(`${xInt} = fcmp oeq double ${xb}, ${x}`); + B.line(`${yInt} = fcmp oeq double ${yb}, ${y}`); + B.line(`${nonzero} = icmp ne i64 ${yi}, 0`); + B.line(`${both} = and i1 ${xInt}, ${yInt}`); + B.line(`${ok} = and i1 ${both}, ${nonzero}`); + B.condBr(ok, exact, float); + B.startBlock(exact); + const r = B.tmp(); + const rd = B.tmp(); + const xBits = B.tmp(); + const negative = B.tmp(); + const zero = B.tmp(); + const negZero = B.tmp(); + const intResult = B.tmp(); + B.line(`${r} = srem i64 ${xi}, ${yi}`); + B.line(`${rd} = sitofp i64 ${r} to double`); + B.line(`${xBits} = bitcast double ${x} to i64`); + B.line(`${negative} = icmp slt i64 ${xBits}, 0`); + B.line(`${zero} = icmp eq i64 ${r}, 0`); + B.line(`${negZero} = and i1 ${negative}, ${zero}`); + B.line(`${intResult} = select i1 ${negZero}, double -0.0, double ${rd}`); + const exactEnd = B.currentLabel(); + B.br(join); + B.startBlock(float); + const fr = B.tmp(); + B.line(`${fr} = frem double ${x}, ${y}`); + const floatEnd = B.currentLabel(); + B.br(join); + B.startBlock(join); + const out = B.tmp(); + B.line(`${out} = phi double [ ${intResult}, %${exactEnd} ], [ ${fr}, %${floatEnd} ]`); + return out; +} + +const BITWISE_OPS: Readonly> = { + "&": "and", + "|": "or", + "^": "xor", + "<<": "shl", + ">>": "ashr", + ">>>": "lshr", +}; + +/** A double's ToInt32 when its magnitude is below 2^63: truncation toward + * zero, then the low 32 bits; branches to `slow` otherwise (NaN and the + * infinities included). */ +function int32Operand(host: Host, d: string, slow: string): string { + const B = host.B; + host.declare(`declare double @llvm.fabs.f64(double)`); + const magnitude = B.tmp(); + const small = B.tmp(); + B.line(`${magnitude} = call double @llvm.fabs.f64(double ${d})`); + B.line(`${small} = fcmp olt double ${magnitude}, 9223372036854775808.0`); + const ok = B.newLabel("js.int"); + B.condBr(small, ok, slow); + B.startBlock(ok); + const wide = B.tmp(); + const i = B.tmp(); + B.line(`${wide} = fptosi double ${d} to i64`); + B.line(`${i} = trunc i64 ${wide} to i32`); + return i; +} + const ARITH_OPS: Readonly> = { "-": "fsub", "*": "fmul", @@ -334,8 +433,8 @@ export function emitJsLibCall(host: Host, e: LibCallExpr): LlValue { (slow) => { const x = a.number(slow); const y = b.number(slow); - const r = B.tmp(); - B.line(`${r} = ${inst} double ${x}, ${y}`); + const r = op === "%" ? remainder(host, x, y) : B.tmp(); + if (op !== "%") B.line(`${r} = ${inst} double ${x}, ${y}`); return boxDouble(host, r); }, runtime, @@ -410,17 +509,54 @@ export function emitJsLibCall(host: Host, e: LibCallExpr): LlValue { return { name: out, type: e.type }; } case "js.bitwise": { - const a = host.emitExpr(e.args[0]!).name; - const b = host.emitExpr(e.args[1]!).name; + const op = literalOp(e.args[2]!); + const inst = op !== null ? BITWISE_OPS[op] : undefined; + if (inst === undefined) { + const a = host.emitExpr(e.args[0]!).name; + const b = host.emitExpr(e.args[1]!).name; + const opArg = host.emitExpr(e.args[2]!).name; + const r = call(host, "ptr", "scr_js_bitwise", [ + ["ptr", a], + ["ptr", b], + ["ptr", opArg], + ]); + const out = jsval(r); + host.emitPendingCheck(); + return out; + } + // Two numbers of magnitude below 2^63: ToInt32 is truncation and + // wrapping, so the operation runs on i32; anything else (NaN, the + // infinities, larger magnitudes) takes the runtime. + const a = operand(host, e.args[0]!); + const b = operand(host, e.args[1]!); const opArg = host.emitExpr(e.args[2]!).name; - const r = call(host, "ptr", "scr_js_bitwise", [ - ["ptr", a], - ["ptr", b], - ["ptr", opArg], - ]); - const out = jsval(r); - host.emitPendingCheck(); - return out; + const out = withSlowPath( + host, + "ptr", + (slow) => { + const x = int32Operand(host, a.number(slow), slow); + const y = int32Operand(host, b.number(slow), slow); + const r = B.tmp(); + const d = B.tmp(); + if (op === "<<" || op === ">>" || op === ">>>") { + const count = B.tmp(); + B.line(`${count} = and i32 ${y}, 31`); + B.line(`${r} = ${inst} i32 ${x}, ${count}`); + } else { + B.line(`${r} = ${inst} i32 ${x}, ${y}`); + } + B.line(`${d} = ${op === ">>>" ? "uitofp" : "sitofp"} i32 ${r} to double`); + return boxDouble(host, d); + }, + () => + call(host, "ptr", "scr_js_bitwise", [ + ["ptr", a.word()], + ["ptr", b.word()], + ["ptr", opArg], + ]), + true, + ); + return jsval(out); } default: throw new InternalCompilerError(`llvm emitter bug: unknown jsval builtin ${e.fn}`); diff --git a/packages/compiler/src/backend/native/executable.ts b/packages/compiler/src/backend/native/executable.ts index 547f231e8..96f75acaf 100644 --- a/packages/compiler/src/backend/native/executable.ts +++ b/packages/compiler/src/backend/native/executable.ts @@ -445,6 +445,9 @@ export async function compileCInternal( : [optimization === "dev" ? "-O0" : "-O2"]), ...executableSectionFlags.compile, ...(opts.textDecoderLegacy ? ["-DSCR_TEXT_DECODER_LEGACY"] : []), + // Checked objects carry the dynamic tier's shapes (scr_json.c's + // SCR_JS_TIER build); only programs with dynamic-tier code select it. + ...(opts.dynTier === true ? ["-DSCR_JS_TIER"] : []), "-fno-math-errno", // The runtime object model uses type-punned C: a hierarchy // upcast is a raw pointer cast, so one object's header (rc, vt) and @@ -502,7 +505,9 @@ export async function compileCInternal( : []), ...(opts.dynInvoke || opts.workers || nativeFetch ? [rt(join(rtDir, "scr_dyn_invoke.c"))] : []), ...(opts.dc ? [rt(join(rtDir, "scr_dc.c"))] : []), - ...(opts.dynTier === true ? [rt(join(rtDir, "scr_js_value.c"))] : []), + ...(opts.dynTier === true + ? [rt(join(rtDir, "scr_js_value.c")), rt(join(rtDir, "scr_js_object.c"))] + : []), ...(opts.eventTarget || nativeFetch ? [rt(join(rtDir, "scr_event_target.c"))] : []), ...(opts.dynAsync || opts.dynInvoke || opts.workers || opts.dc || opts.fileHandle || nativeFetch ? [rt(join(rtDir, "scr_async_dyn.c"))] @@ -644,6 +649,9 @@ export async function compileCInternal( : [optimization === "dev" ? "-O0" : "-O2"]), ...executableSectionFlags.compile, ...(opts.textDecoderLegacy ? ["-DSCR_TEXT_DECODER_LEGACY"] : []), + // Checked objects carry the dynamic tier's shapes (scr_json.c's + // SCR_JS_TIER build); only programs with dynamic-tier code select it. + ...(opts.dynTier === true ? ["-DSCR_JS_TIER"] : []), "-fno-math-errno", "-fno-strict-aliasing", // the emitted object model type-puns — see buildArgs "-Wno-deprecated-declarations", diff --git a/packages/compiler/src/backend/native/library.ts b/packages/compiler/src/backend/native/library.ts index 23d081bfe..3907fd5d8 100644 --- a/packages/compiler/src/backend/native/library.ts +++ b/packages/compiler/src/backend/native/library.ts @@ -112,7 +112,7 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise ...(opts.assert && opts.bigint ? ["scr_bigint_assert.c"] : []), ...(opts.emitter ? ["scr_events_emitter.c", "scr_dyn_handle.c"] : []), ...(opts.dynInvoke ? ["scr_dyn_invoke.c"] : []), - ...(opts.dynTier === true ? ["scr_js_value.c"] : []), + ...(opts.dynTier === true ? ["scr_js_value.c", "scr_js_object.c"] : []), ...(opts.zlib ? ["scr_zlib.c"] : []), ...(opts.copying ? ["scr_copying.c"] : []), ]; @@ -129,6 +129,7 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise "-DSCR_LIB", ...(opts.threadInstances ? ["-DSCR_THREAD_INSTANCES"] : []), ...(opts.textDecoderLegacy ? ["-DSCR_TEXT_DECODER_LEGACY"] : []), + ...(opts.dynTier === true ? ["-DSCR_JS_TIER"] : []), "-I", rtDir, ...(regex ? ["-I", vendorEngineDir()] : []), diff --git a/packages/compiler/src/ir/js-ic.test.ts b/packages/compiler/src/ir/js-ic.test.ts new file mode 100644 index 000000000..1b566fd00 --- /dev/null +++ b/packages/compiler/src/ir/js-ic.test.ts @@ -0,0 +1,178 @@ +import { expect, test } from "vitest"; +import { BOOL, DYN, STRING, VOID, type IrExpr, type IrFunction, type IrStmt } from "./ir.js"; +import { JsIcSites, hasJsIcSite } from "./js-ic.js"; + +const loc = { file: "js-ic.ts", start: 0, end: 0 }; +const ref = (id: string, type = DYN): IrExpr => ({ kind: "varRef", localId: id, type, loc }); +const str = (value: string): IrExpr => ({ kind: "strLit", value, type: STRING, loc }); +const call = (callee: string, ...args: IrExpr[]): IrExpr => ({ + kind: "call", + callee, + args, + type: DYN, + loc, +}); +const ret = (value: IrExpr | null): IrStmt => ({ kind: "return", value, loc }); + +function helper(name: string, params: string[], body: IrStmt[], locals: string[] = []): IrFunction { + return { + name, + params: params.map((id) => ({ localId: id, name: id, type: DYN })), + returnType: DYN, + locals: [...params, ...locals].map((id) => ({ id, name: id, type: DYN, mutable: false })), + body, + loc, + }; +} + +/** `if (dyn.typedRefIs(p, "C")) return undefined` — a capsule branch. */ +function capsuleBranch(param: string, fn = "dyn.typedRefIs"): IrStmt { + return { + kind: "if", + cond: { + kind: "libCall", + fn: fn as "dyn.typedRefIs", + args: [ref(param), str("C")], + type: BOOL, + loc, + }, + then: [ret(ref(param))], + else_: null, + loc, + }; +} + +const identityBag = helper( + "%dyn.class.readPropertiesOrEmpty", + ["v"], + [capsuleBranch("v"), ret(ref("v"))], +); + +function getHelper(name: string, key: string, prefix: IrStmt[] = [], bag?: IrExpr): IrFunction { + const bagRef = ref("bag.0"); + return helper( + name, + ["p.0"], + [ + ...prefix, + { kind: "varDecl", localId: "bag.0", init: bag ?? call(identityBag.name, ref("p.0")), loc }, + ret({ + kind: "libCall", + fn: "js.fromDyn", + args: [ + { + kind: "ternary", + cond: { kind: "dynScalarEq", left: bagRef, right: ref("p.0"), type: BOOL, loc }, + then: { kind: "dynKeyGet", key: str(key), value: bagRef, type: DYN, loc }, + else_: { + kind: "libCall", + fn: "dyn.bagGet", + args: [bagRef, str(key), ref("p.0")], + type: DYN, + loc, + }, + type: DYN, + loc, + }, + ], + type: { kind: "jsval" }, + loc, + }), + ], + ["bag.0"], + ); +} + +function setHelper(name: string, key: string, prefix: IrStmt[] = []): IrFunction { + return { + ...helper( + name, + ["p.0", "p.1"], + [ + ...prefix, + { + kind: "exprStmt", + expr: { + kind: "libCall", + fn: "dyn.keySet", + args: [call("%dyn.class.properties", ref("p.0")), str(key), ref("p.1")], + type: VOID, + loc, + }, + loc, + }, + ret(null), + ], + ), + returnType: VOID, + }; +} + +const properties = helper("%dyn.class.properties", ["v"], [capsuleBranch("v"), ret(ref("v"))]); + +function sitesOf(...functions: IrFunction[]): JsIcSites { + return new JsIcSites({ functions: [identityBag, properties, ...functions] }); +} + +test("property helpers whose class branches only test capsules are cache sites", () => { + const sites = sitesOf( + getHelper("%dyn.class.property.0", "x", [ + capsuleBranch("p.0"), + capsuleBranch("p.0", "dyn.classIs"), + ]), + setHelper("%dyn.class.property.1", "y", [capsuleBranch("p.0", "dyn.typedRefKeyIs")]), + ); + expect(sites.site("%dyn.class.property.0")).toEqual({ kind: "get", key: "x" }); + expect(sites.site("%dyn.class.property.1")).toEqual({ kind: "set", key: "y" }); +}); + +test("any other prefix or tail keeps the helper call", () => { + const errorBranch: IrStmt = { + kind: "if", + cond: { kind: "dynTest", test: "error", value: ref("p.0"), type: BOOL, loc }, + then: [ret(ref("p.0"))], + else_: null, + loc, + }; + // A branch an ordinary object can take (toString of an error object). + const withError = getHelper("%dyn.class.property.0", "toString", [errorBranch]); + // A bag that is not the receiver for ordinary objects. + const otherBag = helper("%dyn.class.other", ["v"], [ret({ kind: "dynObjLit", type: DYN, loc })]); + const foreignBag = getHelper("%dyn.class.property.1", "x", [], call(otherBag.name, ref("p.0"))); + // A function outside the generated helper family. + const userFn = { ...getHelper("%dyn.class.property.2", "x"), name: "user" }; + const sites = sitesOf(withError, otherBag, foreignBag, userFn); + expect(sites.site("%dyn.class.property.0")).toBeNull(); + expect(sites.site("%dyn.class.property.1")).toBeNull(); + expect(sites.site("user")).toBeNull(); + expect(sites.site("%dyn.class.missing")).toBeNull(); +}); + +test("a function holds a site when it calls a classified helper or dispatches a method", () => { + const getter = getHelper("%dyn.class.property.0", "x"); + const sites = sitesOf(getter); + const caller = helper("f", ["o"], [ret(call(getter.name, ref("o")))]); + const invoker = helper( + "g", + ["o"], + [ + ret({ + kind: "dynInvoke", + recv: ref("o"), + method: "m", + calleeName: "o.m", + args: [], + type: DYN, + loc, + }), + ], + ); + const plain = helper("h", ["o"], [ret(ref("o"))]); + expect(hasJsIcSite(caller, sites)).toBe(true); + expect(hasJsIcSite(invoker, sites)).toBe(true); + expect(hasJsIcSite(plain, sites)).toBe(false); + // Module initializers keep plain method dispatch (top-level handle + // wiring), while their property sites still cache. + expect(hasJsIcSite({ ...invoker, name: "%init.0" }, sites)).toBe(false); + expect(hasJsIcSite({ ...caller, name: "%init.0" }, sites)).toBe(true); +}); diff --git a/packages/compiler/src/ir/js-ic.ts b/packages/compiler/src/ir/js-ic.ts new file mode 100644 index 000000000..316b3c958 --- /dev/null +++ b/packages/compiler/src/ir/js-ic.ts @@ -0,0 +1,236 @@ +/* Inline-cache sites of the dynamic tier (two-tier design stage S4). + * + * Lowering builds every dynamic property access on a constant key as a + * call to a generated helper (class-dynamic-dispatch.ts): a read + * `%dyn.class.property.N(receiver)`, a write + * `%dyn.class.property.N(receiver, value)` and a keyed read + * `%dyn.class.computed.N(receiver, key)`. A helper starts with one branch + * per compiled class whose capsules may reach it and ends with the generic + * checked-dynamic operation on the receiver itself. The capsule branches + * test capsule and class-object kinds only, so for an ordinary checked + * object or array receiver a helper is exactly its generic tail: [[Get]] + * (`dynKeyGet`) or [[Set]] (`dyn.keySet`) of the key. + * + * A call to such a helper from a dynamic-tier function is an inline-cache + * site: the backend (backend/llvm/js-ic.ts) answers plain data properties + * of shaped objects through the site's cache and calls the helper for + * everything else, so the helper stays the single definition of the + * operation's semantics. This module recognizes the helpers by structure, + * never by name alone: a helper whose body is not exactly capsule branches + * plus the generic tail is not a site. `dynInvoke` is a site too: its + * method read on an ordinary object is a [[Get]] (outside module + * initializers, see jsIcMethodSites). + * + * Static functions never contain sites (invariant I1): the backend only + * consults this module for calls in dynamic-tier functions. */ +import type { IrExpr, IrFunction, IrModule, IrStmt } from "./ir.js"; +import { everyStmtList } from "./traverse.js"; + +export type JsIcSite = + | { kind: "get"; key: string } + | { kind: "set"; key: string } + | { kind: "keyed" }; + +const CAPSULE_TESTS = new Set(["dyn.typedRefIs", "dyn.typedRefKeyIs", "dyn.classIs"]); + +/** Classifies the helpers of a module once. */ +export class JsIcSites { + private readonly fns: ReadonlyMap; + private readonly sites = new Map(); + private readonly identities = new Map(); + + constructor(mod: Pick) { + this.fns = new Map(mod.functions.map((fn) => [fn.name, fn])); + } + + /** The cache site a direct call to `callee` is, or null. */ + site(callee: string): JsIcSite | null { + let site = this.sites.get(callee); + if (site === undefined) { + const fn = this.fns.get(callee); + site = fn ? this.classify(fn) : null; + this.sites.set(callee, site); + } + return site; + } + + private classify(fn: IrFunction): JsIcSite | null { + if (!fn.name.startsWith("%dyn.class.")) return null; + if (fn.async !== undefined || fn.generator !== undefined || fn.captures !== undefined) + return null; + if (fn.params.some((p) => p.type.kind !== "dyn")) return null; + const receiver = fn.params[0]?.localId; + if (receiver === undefined) return null; + let i = 0; + while (i < fn.body.length && this.capsuleBranch(fn.body[i]!, receiver)) i++; + const tail = fn.body.slice(i); + if (fn.params.length === 1) return this.getTail(tail, receiver); + if (fn.params.length !== 2) return null; + const second = fn.params[1]!.localId; + return this.setTail(tail, receiver, second) ?? this.keyedTail(tail, receiver, second); + } + + /** `varDecl bag = BAG(receiver); return [js.fromDyn](bag === receiver ? + * dynKeyGet(bag, K) : dyn.bagGet(bag, K, receiver))`. */ + private getTail(tail: IrStmt[], receiver: string): JsIcSite | null { + const [decl, ret] = tail; + if (tail.length !== 2 || decl?.kind !== "varDecl" || ret?.kind !== "return") return null; + if (decl.init === null || !this.bagOf(decl.init, receiver)) return null; + const bag = decl.localId; + const value = unwrapFromDyn(ret.value); + if (value?.kind !== "ternary") return null; + const { cond, then } = value; + if ( + cond.kind !== "dynScalarEq" || + cond.negated === true || + !isVar(cond.left, bag) || + !isVar(cond.right, receiver) || + then.kind !== "dynKeyGet" || + !isVar(then.value, bag) || + then.key.kind !== "strLit" + ) + return null; + return { kind: "get", key: then.key.value }; + } + + /** `dyn.keySet(BAG(receiver), K, value); return`. */ + private setTail(tail: IrStmt[], receiver: string, value: string): JsIcSite | null { + const [stmt, ret] = tail; + if (tail.length !== 2 || stmt?.kind !== "exprStmt" || ret?.kind !== "return") return null; + if (ret.value !== null) return null; + const call = stmt.expr; + if (call.kind !== "libCall" || call.fn !== "dyn.keySet" || call.args.length !== 3) return null; + const [bag, key, stored] = call.args; + if (!bag || !this.bagOf(bag, receiver) || key?.kind !== "strLit" || !isVar(stored, value)) + return null; + return { kind: "set", key: key.value }; + } + + /** `if (typeof key === "symbol") return dynKeyGet(BAG(receiver), key); + * if (receiver == null) return dynKeyGet(receiver, key); + * const s = String(key); return dynKeyGet(BAG(receiver), s)`, each read + * possibly under js.fromDyn. */ + private keyedTail(tail: IrStmt[], receiver: string, key: string): JsIcSite | null { + const [symbol, nullish, decl, ret] = tail; + if (tail.length !== 4) return null; + if (!this.guard(symbol, "symbol", key, (read) => this.bagOf(read, receiver), key)) return null; + if (!this.guard(nullish, "nullish", receiver, (read) => isVar(read, receiver), key)) + return null; + if (decl?.kind !== "varDecl" || decl.init?.kind !== "libCall") return null; + if (decl.init.fn !== "dyn.toStringCoerce" || !isVar(decl.init.args[0], key)) return null; + if (ret?.kind !== "return") return null; + const read = unwrapFromDyn(ret.value); + if (read?.kind !== "dynKeyGet" || !isVar(read.key, decl.localId)) return null; + if (!this.bagOf(read.value, receiver)) return null; + return { kind: "keyed" }; + } + + private guard( + s: IrStmt | undefined, + test: "symbol" | "nullish", + tested: string, + object: (e: IrExpr) => boolean, + key: string, + ): boolean { + if (s?.kind !== "if" || s.else_ !== null || s.then.length !== 1) return false; + const { cond } = s; + if (cond.kind !== "dynTest" || cond.test !== test || cond.negated === true) return false; + if (!isVar(cond.value, tested)) return false; + const ret = s.then[0]!; + if (ret.kind !== "return") return false; + const read = unwrapFromDyn(ret.value); + return read?.kind === "dynKeyGet" && isVar(read.key, key) && object(read.value); + } + + /** A branch that can only be taken for a capsule or class-object + * receiver: `if () { ... }` with no else. */ + private capsuleBranch(s: IrStmt, receiver: string): boolean { + return s.kind === "if" && s.else_ === null && this.capsuleTest(s.cond, receiver); + } + + private capsuleTest(e: IrExpr, receiver: string): boolean { + if (e.kind === "libCall") + return CAPSULE_TESTS.has(e.fn) && e.args[0] !== undefined && this.bagOf(e.args[0], receiver); + if (e.kind === "ternary") + return ( + this.capsuleTest(e.cond, receiver) && e.else_.kind === "boolLit" && e.else_.value === false + ); + return false; + } + + /** Whether `e` is the receiver itself for every ordinary object or array: + * the receiver, or a chain of property-bag helpers applied to it. */ + private bagOf(e: IrExpr, receiver: string): boolean { + if (isVar(e, receiver)) return true; + return ( + e.kind === "call" && + e.args.length === 1 && + this.bagOf(e.args[0]!, receiver) && + this.identityHelper(e.callee) + ); + } + + /** A property-bag helper that returns its argument unless a capsule + * branch is taken. */ + private identityHelper(name: string): boolean { + const known = this.identities.get(name); + if (known !== undefined) return known; + this.identities.set(name, false); // a recursive chain is not an identity + const fn = this.fns.get(name); + let identity = false; + if ( + fn && + name.startsWith("%dyn.class.") && + fn.params.length === 1 && + fn.params[0]!.type.kind === "dyn" && + fn.async === undefined && + fn.generator === undefined && + fn.captures === undefined + ) { + const param = fn.params[0]!.localId; + const last = fn.body[fn.body.length - 1]; + identity = + fn.body.slice(0, -1).every((s) => this.capsuleBranch(s, param)) && + last?.kind === "return" && + last.value !== null && + this.bagOf(last.value, param); + } + this.identities.set(name, identity); + return identity; + } +} + +function isVar(e: IrExpr | undefined | null, localId: string): boolean { + return e?.kind === "varRef" && e.localId === localId; +} + +function unwrapFromDyn(e: IrExpr | null | undefined): IrExpr | null { + if (!e) return null; + if (e.kind === "libCall" && e.fn === "js.fromDyn" && e.args.length === 1) return e.args[0]!; + return e; +} + +/** Whether a function holds a cache site: a call to a classified helper or + * a method dispatch. */ +export function hasJsIcSite(fn: IrFunction, sites: JsIcSites): boolean { + const methods = jsIcMethodSites(fn.name); + return !everyStmtList(fn.body, { + expr: (e) => + !( + (methods && e.kind === "dynInvoke") || + (e.kind === "call" && sites.site(e.callee) !== null) + ), + stmt: () => true, + }); +} + +/** Whether method dispatch in a function goes through caches. Module + * initializers (`%init.N`) run once, and the methods they call at top level + * are overwhelmingly registrations on native handles (`worker.on(...)`, + * `server.listen(...)`), which a cache never answers: they keep the plain + * dispatch, so a typed program whose only dynamic code is such top-level + * wiring links no dynamic-tier runtime. Property sites still cache there + * (top-level loops over objects). */ +export function jsIcMethodSites(fnName: string): boolean { + return !fnName.startsWith("%init."); +} diff --git a/packages/compiler/src/ir/jsval.ts b/packages/compiler/src/ir/jsval.ts index 1939531f1..722225218 100644 --- a/packages/compiler/src/ir/jsval.ts +++ b/packages/compiler/src/ir/jsval.ts @@ -42,6 +42,7 @@ * suspensions keep jsvals like any pointer) but keep the dyn result ABI. */ import { exprFunctionReference, moduleFunctionReferences } from "./function-references.js"; import { DYN, JSVAL, type IrExpr, type IrFunction, type IrModule, type IrStmt } from "./ir.js"; +import { JsIcSites } from "./js-ic.js"; import { everyExprChild, everyStmtChild, @@ -104,8 +105,12 @@ export function jsvalEligible(fn: IrFunction): boolean { * Returns the number of functions that now use jsval. */ export function promoteJsvals(mod: IrModule): number { const results = jsvalResults(mod); + const sites = new JsIcSites(mod); + const storesJs = (e: IrExpr): boolean => + e.kind === "call" && sites.site(e.callee)?.kind === "set"; let count = 0; - for (const fn of mod.functions) if (jsvalEligible(fn) && promoteFunction(fn, results)) count++; + for (const fn of mod.functions) + if (jsvalEligible(fn) && promoteFunction(fn, results, storesJs)) count++; for (const fn of mod.functions) if (results.has(fn.name)) fn.returnType = JSVAL; return count; } @@ -165,6 +170,7 @@ function promotableLocals( fn: IrFunction, producer: (e: IrExpr) => boolean, jsResult: boolean, + storesJs: (e: IrExpr) => boolean, ): Set { const candidates = new Map(); for (const local of fn.locals) { @@ -218,6 +224,14 @@ function promotableLocals( return; } break; + case "call": + // An inline-cache store takes its value as a jsval (S4). + if (storesJs(e)) { + expr(e.args[0]!, false, weight); + expr(e.args[1]!, true, weight); + return; + } + break; default: break; } @@ -247,6 +261,17 @@ function promotableLocals( }; const stmt = (s: IrStmt, weight: number): void => { switch (s.kind) { + case "exprStmt": + // A discarded sequence result (a compound assignment statement) + // is never boxed. + if (discardedSequence(s.expr)) { + for (const inner of s.expr.stmts) stmt(inner, weight); + expr(s.expr.result, true, weight); + return; + } + blockStmtReferences(s); + expr(s.expr, false, weight); + return; case "return": if (s.value !== null) expr(s.value, jsResult && s.value.type.kind === "dyn", weight); return; @@ -300,15 +325,25 @@ function promotableLocals( return promoted; } +/** A statement-level sequence whose result nothing reads. */ +function discardedSequence(e: IrExpr): e is Extract { + return e.kind === "seqExpr" && e.generatorDelegate === undefined && e.diverges === undefined; +} + /* ── the rewrite ─────────────────────────────────────────────────────── */ -function promoteFunction(fn: IrFunction, results: ReadonlySet): boolean { +function promoteFunction( + fn: IrFunction, + results: ReadonlySet, + storesJs: (e: IrExpr) => boolean, +): boolean { const resultCall = (e: IrExpr): boolean => e.kind === "call" && results.has(e.callee); const operatorCall = (e: IrExpr): boolean => isProducerCall(e); const promoted = promotableLocals( fn, (e) => operatorCall(e) || resultCall(e), results.has(fn.name), + storesJs, ); const ownResult = results.has(fn.name); let changed = false; @@ -371,6 +406,13 @@ function promoteFunction(fn: IrFunction, results: ReadonlySet): boolean if (s.kind === "return" && ownResult && s.value !== null) return { ...s, value: toJs(s.value) }; // A discarded result stays a jsval: nothing boxes it. if (s.kind === "exprStmt" && producesJs(s.expr)) return { ...s, expr: toJs(s.expr) }; + if (s.kind === "exprStmt" && discardedSequence(s.expr) && producesJs(s.expr.result)) { + const result = toJs(s.expr.result); + return { + ...s, + expr: { ...s.expr, stmts: s.expr.stmts.map(rewriteStmt), result, type: result.type }, + }; + } if (s.kind === "varDecl" && promoted.has(s.localId) && s.init !== null) return { ...s, init: toJs(s.init) }; if (s.kind === "assign" && promoted.has(s.localId)) return { ...s, value: toJs(s.value) }; diff --git a/packages/compiler/src/ir/runtime-features.ts b/packages/compiler/src/ir/runtime-features.ts index 8ec2d2de8..562f1a21d 100644 --- a/packages/compiler/src/ir/runtime-features.ts +++ b/packages/compiler/src/ir/runtime-features.ts @@ -8,6 +8,7 @@ import { RUNTIME_STREAM_CLASSES, } from "./ir.js"; import { everyModuleNode } from "./traverse.js"; +import { JsIcSites, hasJsIcSite } from "./js-ic.js"; export interface RuntimeFeatures { regex: boolean; @@ -548,6 +549,12 @@ function scanRuntimeFeatures(mod: IrModule, stopAt?: keyof RuntimeFeatures): Run return keepGoing(); }; if (keepGoing()) everyModuleNode(mod, { expr, stmt: keepGoing, type }); + // Inline-cache sites of dynamic-tier functions use the tier's shapes + // (scr_js_object.c, with the SCR_JS_TIER build of scr_json.c). + if (!features.dynTier && mod.functions.some((fn) => fn.tier === "dynamic")) { + const sites = new JsIcSites(mod); + features.dynTier = mod.functions.some((fn) => fn.tier === "dynamic" && hasJsIcSite(fn, sites)); + } return features; } @@ -870,9 +877,9 @@ export function moduleUsesTlsCa(mod: IrModule): boolean { return scanRuntimeFeatures(mod, "tlsCa").tlsCa; } -/** True when the module has dynamic-tier jsval operations: the link switch - * for the scr_js_* runtime units (two-tier design I3). Programs without - * them never link a scr_js_ symbol. */ +/** True when the module has dynamic-tier jsval operations or inline-cache + * sites: the link switch for the scr_js_* runtime units (two-tier design + * I3). Programs without them never link a scr_js_ symbol. */ export function moduleUsesDynTier(mod: IrModule): boolean { return scanRuntimeFeatures(mod, "dynTier").dynTier; } diff --git a/packages/compiler/test/js-ic-emission.test.ts b/packages/compiler/test/js-ic-emission.test.ts new file mode 100644 index 000000000..47d641bff --- /dev/null +++ b/packages/compiler/test/js-ic-emission.test.ts @@ -0,0 +1,92 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { compile, deserializeModule, validateModule } from "../src/index.js"; +import { emitLlvmModule } from "../src/backend/llvm/emitter.js"; +import { moduleUsesDynTier } from "../src/ir/runtime-features.js"; + +/** The LLVM body of a function by source name. */ +function body(llvm: string, name: string): string { + const match = llvm.match( + new RegExp( + `define internal [^\\n]+ @sc_(?:b)?f_${name}(?:\\.i32)?\\([^\\n]*\\) #0 \\{([\\s\\S]*?)\\n\\}`, + ), + ); + expect(match, `${name} LLVM function`).not.toBeNull(); + return match![1]!; +} + +test("dynamic-tier property sites read and write through inline caches", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-js-ic-")); + try { + const entry = join(dir, "main.js"); + const outPath = join(dir, "main.ir.json"); + await writeFile( + entry, + ` +function dyn(v) { return v; } +function read(o) { return o.x; } +function write(o, v) { o.x = v; } +function bump(o) { o.n += 1; } +function at(a, i) { return a[i]; } +function call(o) { return o.m(); } +var o = dyn({ x: 1, n: 0, m: function () { return 2; } }); +write(o, 3); +bump(o); +console.log(read(o), at(dyn([5, 6]), 1), call(o), o.n); +`, + ); + const result = await compile(entry, { outDir: dir, outPath, outputKind: "ir" }); + if (!result.ok) + throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); + const mod = deserializeModule(await readFile(outPath, "utf8")); + expect(validateModule(mod)).toEqual([]); + expect(moduleUsesDynTier(mod)).toBe(true); + const llvm = emitLlvmModule(mod); + // Each site owns a zeroed thread-local cache record. + expect(llvm).toMatch( + /@sc_js_ic_0 = internal (thread_local )?global \[21 x i64\] zeroinitializer/, + ); + expect(body(llvm, "read")).toContain("@scr_js_load_probe("); + expect(body(llvm, "write")).toContain("@scr_js_store_probe("); + expect(body(llvm, "at")).toContain("@scr_js_keyed_probe("); + expect(body(llvm, "call")).toContain("@scr_js_load_probe("); + // A compound assignment stores its number result from the jsval: an + // unshared number cell is updated in place. + expect(body(llvm, "bump")).toMatch(/store double %t\d+, ptr %t\d+/); + // The helper stays the out-of-line slow path of every site. + expect(body(llvm, "read")).toMatch( + /@sc_bf__x25_dyn_class_property_\d+\(ptr %t\d+\) noinline cold/, + ); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("typed programs have no cache sites and keep the ordinary runtime", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-js-ic-typed-")); + try { + const entry = join(dir, "main.ts"); + const outPath = join(dir, "main.ir.json"); + await writeFile( + entry, + ` +interface P { x: number; y: number } +function len2(p: P): number { return p.x * p.x + p.y * p.y; } +const parsed: unknown = JSON.parse('{"x":3,"y":4}'); +console.log(len2({ x: 3, y: 4 }), typeof parsed); +`, + ); + const result = await compile(entry, { outDir: dir, outPath, outputKind: "ir" }); + if (!result.ok) + throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); + const mod = deserializeModule(await readFile(outPath, "utf8")); + expect(moduleUsesDynTier(mod)).toBe(false); + const llvm = emitLlvmModule(mod); + expect(llvm).not.toContain("@sc_js_ic_"); + expect(llvm).not.toContain("@scr_js_"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6148-shape-caches.js.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6148-shape-caches.js.json new file mode 100644 index 000000000..03f2ee98c --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6148-shape-caches.js.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6148-shape-caches.js" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6149-jsval-integer-paths.js.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6149-jsval-integer-paths.js.json new file mode 100644 index 000000000..fe4f105c4 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6149-jsval-integer-paths.js.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6149-jsval-integer-paths.js" + ], + "diags": [] +} diff --git a/packages/runtime-darwin-arm64/runtime-pack-matrix.mjs b/packages/runtime-darwin-arm64/runtime-pack-matrix.mjs index 7a491afb2..7161f87a5 100644 --- a/packages/runtime-darwin-arm64/runtime-pack-matrix.mjs +++ b/packages/runtime-darwin-arm64/runtime-pack-matrix.mjs @@ -70,6 +70,7 @@ const optional = [ // The dynamic tier's jsval values (two-tier design I3): linked only into // programs whose IR has dynamic-tier operations. ["scr_js_value.c", "dynTier"], + ["scr_js_object.c", "dynTier"], ["scr_event_target.c", any("eventTarget", "nativeFetch")], ["scr_async_dyn.c", any("dynAsync", "dynInvoke", "dc", "fileHandle", "nativeFetch")], // The performance timeline emits its buffer warning through scr_async_dyn.c. @@ -120,6 +121,12 @@ const optional = [ function variantsFor(source) { const variants = [{ id: "default", when: {}, defines: [] }]; + // Programs with dynamic-tier code keep the tier's hidden classes current + // in checked objects (two-tier design S4); every other program links the + // ordinary object, byte for byte. + if (source === "scr_json.c") { + variants.push({ id: "dyn-tier", when: { dynTier: true }, defines: ["SCR_JS_TIER"] }); + } if (source === "scr_bytes.c") { variants.push({ id: "text-decoder-legacy", diff --git a/packages/runtime-pack-common/library-matrix.mjs b/packages/runtime-pack-common/library-matrix.mjs index dd3071339..94a5c1831 100644 --- a/packages/runtime-pack-common/library-matrix.mjs +++ b/packages/runtime-pack-common/library-matrix.mjs @@ -16,6 +16,7 @@ export function withLibraryRuntimeFlavors(matrix) { "scr_copying.c", "scr_dyn_invoke.c", "scr_js_value.c", + "scr_js_object.c", ]); const libraryUnits = [ ...matrix.runtime_units.filter( diff --git a/packages/runtime-pack-common/runtime-pack-matrix.mjs b/packages/runtime-pack-common/runtime-pack-matrix.mjs index d1fe7690e..bb19474b6 100644 --- a/packages/runtime-pack-common/runtime-pack-matrix.mjs +++ b/packages/runtime-pack-common/runtime-pack-matrix.mjs @@ -60,6 +60,7 @@ const OPTIONAL = [ // The dynamic tier's jsval values (two-tier design I3): linked only into // programs whose IR has dynamic-tier operations. ["scr_js_value.c", "dynTier"], + ["scr_js_object.c", "dynTier"], ["scr_event_target.c", any("eventTarget", "nativeFetch")], ["scr_async_dyn.c", any("dynAsync", "dynInvoke", "dc", "fileHandle", "nativeFetch")], // The performance timeline emits its buffer warning through scr_async_dyn.c. @@ -110,6 +111,12 @@ const OPTIONAL = [ function variantsFor(source) { const variants = [{ id: "default", when: {}, defines: [] }]; + // Programs with dynamic-tier code keep the tier's hidden classes current + // in checked objects (two-tier design S4); every other program links the + // ordinary object, byte for byte. + if (source === "scr_json.c") { + variants.push({ id: "dyn-tier", when: { dynTier: true }, defines: ["SCR_JS_TIER"] }); + } if (source === "scr_bytes.c") { variants.push({ id: "text-decoder-legacy", diff --git a/packages/runtime/src/scr_dyn.h b/packages/runtime/src/scr_dyn.h index ca21a0685..5305b96fa 100644 --- a/packages/runtime/src/scr_dyn.h +++ b/packages/runtime/src/scr_dyn.h @@ -332,6 +332,13 @@ struct ScrDyn { * clear it before release, so freelist reuse never sees it. */ const char *class_name; struct ScrDynRegion *region; /* see ScrDynRegion */ + /* The dynamic tier's hidden class (scr_js_object.c): the transition + * tree node naming this object's key sequence, or NULL while it has + * none. Only programs with dynamic-tier code read or maintain it: + * their runtime is built with SCR_JS_TIER, where every allocation and + * every change to the key set resets it. Other programs never look + * at it, so it may hold anything there. */ + struct ScrJsShape *shape; } obj; /* owned */ /* SCR_DYN_FUNC: the boxed closure (owned) + its call descriptor. `sig` * and `name` are static compiler-emitted literals (never freed); name @@ -401,6 +408,9 @@ _Static_assert(offsetof(ScrDyn, v.typed_ref.type_key) == 16 + 3 * sizeof(void *) _Static_assert(sizeof(((ScrDyn *)0)->v.arr) <= sizeof(((ScrDyn *)0)->v.typed_ref) && sizeof(((ScrDyn *)0)->v.obj) <= sizeof(((ScrDyn *)0)->v.typed_ref), "dyn region links must not grow the node"); +/* Dynamic-tier inline caches read the shape word in place (64-bit). */ +_Static_assert(sizeof(void *) != 8 || offsetof(ScrDyn, v.obj.shape) == 80, "v.obj.shape offset"); +_Static_assert(sizeof(void *) != 8 || offsetof(ScrDyn, v.obj.region) == 72, "v.obj.region offset"); /* ── intact parse regions ───────────────────────────────────────────── * Every heap dyn node carries a cycle header, so an ordinary release that @@ -1358,4 +1368,17 @@ void scr_dyn_trace_v(void *d, ScrTraceVisit visit, void *ctx); long scr_dyn_live_count(void); #endif +/* Checked-object primitives the dynamic tier's shapes and caches build on + * (scr_js_object.c). They exist only in the SCR_JS_TIER build of + * scr_json.c, which programs with dynamic-tier code link. */ +/* The builtin Object.prototype node, or NULL before its first use. */ +const ScrDyn *scr_dyn_tier_object_prototype(void); +/* Shared key bytes for `key` (+1, NUL-terminated), and their release. */ +char *scr_dyn_tier_key_copy(const char *key, size_t key_len); +void scr_dyn_tier_key_release(char *key, size_t key_len); +/* Appends an own data property (writable, enumerable, configurable) that + * `obj` does not have, sharing `key`'s bytes and taking ownership of + * `value`, exactly as an ordinary insertion would; leaves the shape reset. */ +void scr_dyn_tier_append(ScrDyn *obj, char *key, size_t key_len, ScrDyn *value); + #endif /* SCR_DYN_H */ diff --git a/packages/runtime/src/scr_js.h b/packages/runtime/src/scr_js.h index 5af849757..b5b15fb9b 100644 --- a/packages/runtime/src/scr_js.h +++ b/packages/runtime/src/scr_js.h @@ -180,4 +180,58 @@ ScrJsVal scr_js_increment(ScrJsVal value, bool increment); double scr_js_to_number(ScrJsVal value); bool scr_js_truthy(ScrJsVal value); +/* ── shapes and inline caches (scr_js_object.c) ───────────────────────── + * A checked object's hidden class (ScrDyn.v.obj.shape) names its key + * sequence: a node of the thread's transition tree, whose depth is the + * entry count and whose last key sits at entry index depth - 1. Shapes are + * per thread and live until the thread's runtime state is torn down. The + * tier computes an object's shape on a cache miss; the SCR_JS_TIER build + * of scr_json.c resets it on every change to the key set. Attributes stay + * in the entries and the prototype in the node: cache hits test both live, + * so a shape only ever proves which key sits at which entry index and + * which keys an object lacks. + * + * Each property site of compiled dynamic-tier code owns one zeroed, + * thread-local ScrJsIc. The code at the site is fixed: it tests entry 0 + * inline and calls a probe for everything else. Probes never run user + * code: they answer from plain data properties or return a miss (NULL / + * false) with nothing done, and the site then runs its generic path. + * + * load entry: shape = receiver shape; target = NULL for an own property + * (slot indexes the receiver's entries), else the shape of + * `holder`, the receiver's prototype, whose entry `slot` holds it. + * store entry: shape = receiver shape before the store; target == shape + * replaces entry `slot`; any other target adds the key as entry + * `slot` and moves the receiver to `target`. An add requires the + * receiver's prototype to be `holder` (NULL or an object whose + * shape is holder_shape and that lacks the key). + * keyed entry: slot only; a hit compares the key bytes stored there. */ +typedef struct ScrJsShape ScrJsShape; + +typedef struct ScrJsIcEntry { + const ScrJsShape *shape; + const ScrJsShape *target; + const ScrDyn *holder; + const ScrJsShape *holder_shape; + size_t slot; +} ScrJsIcEntry; + +#define SCR_JS_IC_WAYS 4 +typedef struct ScrJsIc { + ScrJsIcEntry entry[SCR_JS_IC_WAYS]; + uint32_t count; /* filled entries */ + uint32_t next; /* the entry a full cache replaces next */ +} ScrJsIc; + +/* `receiver.key` for a plain data property: +1 value, or NULL. */ +ScrDyn *scr_js_load_probe(ScrDyn *receiver, const ScrStr *key, ScrJsIc *ic); +/* `receiver.key = value` (value borrowed) when it replaces a writable own + * data property or adds one the prototype chain lacks: true when done. */ +bool scr_js_store_probe(ScrDyn *receiver, const ScrStr *key, ScrDyn *value, ScrJsIc *ic); +/* `receiver[key]` for an array element or an own data property named by a + * string key: +1 value, or NULL. */ +ScrDyn *scr_js_keyed_probe(ScrDyn *receiver, const ScrDyn *key, ScrJsIc *ic); +/* `receiver[index]` for a dense array element: +1 value, or NULL. */ +ScrDyn *scr_js_index_probe(ScrDyn *receiver, double index); + #endif diff --git a/packages/runtime/src/scr_js_object.c b/packages/runtime/src/scr_js_object.c new file mode 100644 index 000000000..e53fd0b53 --- /dev/null +++ b/packages/runtime/src/scr_js_object.c @@ -0,0 +1,333 @@ +/* scriptc runtime — the dynamic tier's hidden classes and inline caches + * (scr_js.h; two-tier design stage S4). + * + * Shapes are attached to the existing checked objects (ScrDyn OBJ nodes) + * rather than replacing their entry storage: static code and the shared + * runtime read those entries in place, so a second object representation + * would have to be understood or converted at every boundary. A shape is a + * node of the thread's transition tree over key sequences; an object's + * shape proves which key sits at each entry index and which keys the + * object lacks. Everything else a cache hit depends on is tested live: the + * node kind, the accessor and writable bits of the entry it uses, and the + * prototype pointer. + * + * Linked only into programs with dynamic-tier code (the dynTier runtime + * feature), together with the SCR_JS_TIER build of scr_json.c, which + * resets an object's shape on every change to its key set. */ +#include "scr_js.h" +#include "scr_key.h" + +#include + +/* The compiler's inline cache tests read these in place (64-bit targets; + * backend/llvm/js-ic.ts). */ +#if SCR_JS_IMMEDIATES +_Static_assert(offsetof(ScrDyn, kind) == 8, "kind offset"); +_Static_assert(offsetof(ScrDyn, v.obj.len) == 16 && offsetof(ScrDyn, v.obj.entries) == 32, + "object entry offsets"); +_Static_assert(offsetof(ScrDyn, v.arr.len) == 16 && offsetof(ScrDyn, v.arr.items) == 32 && + offsetof(ScrDyn, v.arr.presence) == 56, "array element offsets"); +_Static_assert(offsetof(ScrDyn, prototype) == 96, "prototype offset"); +_Static_assert(sizeof(ScrDynEntry) == 48 && offsetof(ScrDynEntry, value) == 16 && + offsetof(ScrDynEntry, writable) == 41, "entry layout"); +_Static_assert(sizeof(ScrJsIcEntry) == 40 && sizeof(ScrJsIc) == 168, "cache layout"); +#endif + +/* Objects with more entries stay unshaped (dictionary mode): their reads + * take the generic path and its hash index. */ +#define SCR_JS_SHAPE_MAX 64 + +struct ScrJsShape { + const ScrJsShape *parent; + char *key; /* one reference to shared key bytes (scr_json.c) */ + size_t key_len; + uint64_t hash; + uint32_t count; /* entries: the depth in the tree */ + ScrJsShape *all_next; +}; + +/* The root is the empty key sequence. Transitions live in one open + * addressing table per thread keyed by (parent, key). */ +static SCR_TL ScrJsShape scr_js_root; +static SCR_TL ScrJsShape *scr_js_all_shapes; +static SCR_TL ScrJsShape **scr_js_transitions; +static SCR_TL size_t scr_js_transition_mask; /* capacity - 1; 0 when absent */ +static SCR_TL size_t scr_js_transition_count; +static SCR_TL bool scr_js_shapes_registered; + +static void scr_js_shapes_reset(void) { + while (scr_js_all_shapes) { + ScrJsShape *shape = scr_js_all_shapes; + scr_js_all_shapes = shape->all_next; + scr_dyn_tier_key_release(shape->key, shape->key_len); + free(shape); + } + free(scr_js_transitions); + scr_js_transitions = NULL; + scr_js_transition_mask = 0; + scr_js_transition_count = 0; + scr_js_shapes_registered = false; +} + +static size_t scr_js_transition_slot(const ScrJsShape *parent, uint64_t hash) { + return (size_t)scr_key_mix(hash ^ (uint64_t)(uintptr_t)parent); +} + +static void scr_js_transition_insert(ScrJsShape *child) { + size_t i = scr_js_transition_slot(child->parent, child->hash) & scr_js_transition_mask; + while (scr_js_transitions[i]) i = (i + 1) & scr_js_transition_mask; + scr_js_transitions[i] = child; +} + +static void scr_js_transitions_grow(void) { + size_t old_cap = scr_js_transition_mask ? scr_js_transition_mask + 1 : 0; + size_t cap = old_cap ? old_cap * 2 : 256; + ScrJsShape **old = scr_js_transitions; + scr_js_transitions = calloc(cap, sizeof *scr_js_transitions); + if (!scr_js_transitions) scr_trap("scriptc: out of memory\n"); + scr_js_transition_mask = cap - 1; + for (size_t i = 0; i < old_cap; i++) + if (old[i]) scr_js_transition_insert(old[i]); + free(old); +} + +/* The child of `parent` for appending `key` (borrowed bytes), created on + * first use. */ +static const ScrJsShape *scr_js_transition(const ScrJsShape *parent, const char *key, size_t key_len) { + uint64_t hash = scr_key_hash(key, key_len); + if (scr_js_transition_mask) { + size_t i = scr_js_transition_slot(parent, hash) & scr_js_transition_mask; + for (ScrJsShape *child; (child = scr_js_transitions[i]); i = (i + 1) & scr_js_transition_mask) { + if (child->parent == parent && child->hash == hash && child->key_len == key_len && + scr_key_equal(child->key, key, key_len)) + return child; + } + } + if ((scr_js_transition_count + 1) * 2 > (scr_js_transition_mask ? scr_js_transition_mask + 1 : 0)) + scr_js_transitions_grow(); + ScrJsShape *child = malloc(sizeof *child); + if (!child) scr_trap("scriptc: out of memory\n"); + *child = (ScrJsShape){parent, scr_dyn_tier_key_copy(key, key_len), key_len, hash, parent->count + 1, + scr_js_all_shapes}; + scr_js_all_shapes = child; + if (!scr_js_shapes_registered) { + scr_js_shapes_registered = true; + scr_atexit(scr_js_shapes_reset); + } + scr_js_transition_insert(child); + scr_js_transition_count++; + return child; +} + +/* An object's shape, computed from its entries when it has none; NULL for + * objects that stay unshaped (other kinds, the builtin Object.prototype, + * dictionary-sized objects). */ +#ifdef SCR_JS_VERIFY +/* Debugging aid (build the runtime with -DSCR_JS_VERIFY and compile with + * SCRIPTC_JS_IC_WAYS=0 so every access probes): traps when an object's + * shape no longer names its key sequence, i.e. when some mutation of the + * key set missed its SCR_JS_SHAPE_RESET. */ +static void scr_js_verify(const ScrDyn *object, const ScrJsShape *shape) { + size_t len = object->v.obj.len; + if (shape->count != len) scr_trap("scriptc: stale shape (entry count)\n"); + for (const ScrJsShape *s = shape; s->parent; s = s->parent) { + const ScrDynEntry *entry = &object->v.obj.entries[s->count - 1]; + if (entry->key_len != s->key_len || !scr_key_equal(entry->key, s->key, s->key_len)) + scr_trap("scriptc: stale shape (key)\n"); + } +} +#endif + +static const ScrJsShape *scr_js_shape_of(ScrDyn *object) { + if (object->kind != SCR_DYN_OBJ) return NULL; + const ScrJsShape *shape = object->v.obj.shape; +#ifdef SCR_JS_VERIFY + if (shape) scr_js_verify(object, shape); +#endif + if (shape) return shape; + size_t len = object->v.obj.len; + if (len > SCR_JS_SHAPE_MAX || object == scr_dyn_tier_object_prototype()) return NULL; + shape = &scr_js_root; + for (size_t i = 0; i < len; i++) { + ScrDynEntry *entry = &object->v.obj.entries[i]; + shape = scr_js_transition(shape, entry->key, entry->key_len); + } + object->v.obj.shape = (ScrJsShape *)shape; + return shape; +} + +/* The entry index of `key` in an object of shape `shape`, or SIZE_MAX. */ +static size_t scr_js_shape_find(const ScrJsShape *shape, const char *key, size_t key_len) { + for (; shape->parent; shape = shape->parent) + if (shape->key_len == key_len && scr_key_equal(shape->key, key, key_len)) return shape->count - 1; + return SIZE_MAX; +} + +/* Fills a cache entry: `stale` (a matching entry whose hit failed) when + * there is one, else a free or replaceable one. */ +static void scr_js_ic_add(ScrJsIc *ic, uint32_t stale, ScrJsIcEntry entry) { + uint32_t index; + if (stale < ic->count) { + index = stale; + } else if (ic->count < SCR_JS_IC_WAYS) { + index = ic->count++; + } else { + /* A full cache keeps entry 0, which the site tests inline, and cycles + * through the others. */ + index = 1 + ic->next++ % (SCR_JS_IC_WAYS - 1); + } + ic->entry[index] = entry; +} + +/* ── loads ────────────────────────────────────────────────────────────── */ + +/* The cached answer of an entry whose receiver shape matched, or NULL. */ +static ScrDyn *scr_js_load_hit(ScrDyn *receiver, const ScrJsIcEntry *entry) { + const ScrDyn *holder = receiver; + if (entry->target) { + holder = entry->holder; + if (receiver->prototype != holder || holder->kind != SCR_DYN_OBJ || + holder->v.obj.shape != entry->target) + return NULL; + } + const ScrDynEntry *found = &holder->v.obj.entries[entry->slot]; + return found->accessor ? NULL : scr_dyn_retain(found->value); +} + +ScrDyn *scr_js_load_probe(ScrDyn *receiver, const ScrStr *key, ScrJsIc *ic) { + const ScrJsShape *shape = scr_js_shape_of(receiver); + if (!shape) return NULL; + uint32_t stale = UINT32_MAX; + for (uint32_t i = 0; i < ic->count; i++) { + if (ic->entry[i].shape != shape) continue; + ScrDyn *value = scr_js_load_hit(receiver, &ic->entry[i]); + if (value) return value; + stale = i; + break; + } + size_t slot = scr_js_shape_find(shape, key->data, key->len); + if (slot != SIZE_MAX) { + const ScrDynEntry *found = &receiver->v.obj.entries[slot]; + if (found->accessor) return NULL; + scr_js_ic_add(ic, stale, (ScrJsIcEntry){shape, NULL, NULL, NULL, slot}); + return scr_dyn_retain(found->value); + } + /* One prototype link: an ordinary object other than the builtin + * Object.prototype (whose reads the generic path fences). */ + ScrDyn *holder = receiver->prototype; + if (!holder) return NULL; + const ScrJsShape *holder_shape = scr_js_shape_of(holder); + if (!holder_shape) return NULL; + slot = scr_js_shape_find(holder_shape, key->data, key->len); + if (slot == SIZE_MAX) return NULL; + const ScrDynEntry *found = &holder->v.obj.entries[slot]; + if (found->accessor) return NULL; + scr_js_ic_add(ic, stale, (ScrJsIcEntry){shape, holder_shape, holder, NULL, slot}); + return scr_dyn_retain(found->value); +} + +/* ── stores ───────────────────────────────────────────────────────────── */ + +/* The generic store keeps a native Error's fields in sync with these keys + * of its checked view; caches leave them to it. */ +static bool scr_js_error_synced_key(const ScrStr *key) { + switch (key->len) { + case 4: return memcmp(key->data, "name", 4) == 0; + case 5: return memcmp(key->data, "cause", 5) == 0 || memcmp(key->data, "stack", 5) == 0; + case 7: return memcmp(key->data, "message", 7) == 0; + default: return false; + } +} + +static bool scr_js_replace(ScrDyn *receiver, size_t slot, ScrDyn *value) { + ScrDynEntry *found = &receiver->v.obj.entries[slot]; + if (found->accessor || !found->writable) return false; + scr_dyn_region_store(receiver, value); + ScrDyn *old = found->value; + found->value = scr_dyn_retain(value); + scr_dyn_release(old); + return true; +} + +/* Whether an add may skip the generic path's prototype-chain walk: the + * chain is empty, or one ordinary object that lacks the key and ends it. */ +static bool scr_js_add_chain(const ScrDyn *prototype, const ScrJsShape *prototype_shape, + const ScrStr *key) { + if (!prototype) return true; + return prototype_shape && !prototype->prototype && + scr_js_shape_find(prototype_shape, key->data, key->len) == SIZE_MAX; +} + +static bool scr_js_append_shaped(ScrDyn *receiver, const ScrJsShape *target, ScrDyn *value) { + if (receiver->non_extensible) return false; + scr_dyn_tier_append(receiver, target->key, target->key_len, scr_dyn_retain(value)); + receiver->v.obj.shape = (ScrJsShape *)target; + return true; +} + +bool scr_js_store_probe(ScrDyn *receiver, const ScrStr *key, ScrDyn *value, ScrJsIc *ic) { + const ScrJsShape *shape = scr_js_shape_of(receiver); + if (!shape || scr_js_error_synced_key(key)) return false; + uint32_t stale = UINT32_MAX; + for (uint32_t i = 0; i < ic->count; i++) { + const ScrJsIcEntry *entry = &ic->entry[i]; + if (entry->shape != shape) continue; + if (entry->target == shape) { + if (scr_js_replace(receiver, entry->slot, value)) return true; + } else if (receiver->prototype == entry->holder && + (!entry->holder || (entry->holder->kind == SCR_DYN_OBJ && + entry->holder->v.obj.shape == entry->holder_shape && + !entry->holder->prototype))) { + if (scr_js_append_shaped(receiver, entry->target, value)) return true; + } + stale = i; + break; + } + size_t slot = scr_js_shape_find(shape, key->data, key->len); + if (slot != SIZE_MAX) { + const ScrDynEntry *found = &receiver->v.obj.entries[slot]; + if (found->accessor || !found->writable) return false; + scr_js_ic_add(ic, stale, (ScrJsIcEntry){shape, shape, NULL, NULL, slot}); + return scr_js_replace(receiver, slot, value); + } + ScrDyn *prototype = receiver->prototype; + const ScrJsShape *prototype_shape = prototype ? scr_js_shape_of(prototype) : NULL; + if (!scr_js_add_chain(prototype, prototype_shape, key) || receiver->non_extensible || + shape->count >= SCR_JS_SHAPE_MAX) + return false; + const ScrJsShape *target = scr_js_transition(shape, key->data, key->len); + scr_js_ic_add(ic, stale, (ScrJsIcEntry){shape, target, prototype, prototype_shape, target->count - 1}); + return scr_js_append_shaped(receiver, target, value); +} + +/* ── keyed loads ──────────────────────────────────────────────────────── */ + +ScrDyn *scr_js_index_probe(ScrDyn *receiver, double index) { + if (receiver->kind != SCR_DYN_ARR || !(index >= 0) || index >= (double)receiver->v.arr.len) + return NULL; + size_t i = (size_t)index; + if ((double)i != index || !scr_dyn_arr_has_index(receiver, i)) return NULL; + return scr_dyn_retain(receiver->v.arr.items[i]); +} + +ScrDyn *scr_js_keyed_probe(ScrDyn *receiver, const ScrDyn *key, ScrJsIc *ic) { + if (key->kind == SCR_DYN_NUM) return scr_js_index_probe(receiver, key->v.num); + if (key->kind != SCR_DYN_STR || receiver->kind != SCR_DYN_OBJ) return NULL; + const ScrStr *name = key->v.str; + size_t len = receiver->v.obj.len; + for (uint32_t i = 0; i < ic->count; i++) { + size_t slot = ic->entry[i].slot; + if (slot >= len) continue; + const ScrDynEntry *found = &receiver->v.obj.entries[slot]; + if (found->key_len == name->len && scr_key_equal(found->key, name->data, name->len)) + return found->accessor ? NULL : scr_dyn_retain(found->value); + } + const ScrJsShape *shape = scr_js_shape_of(receiver); + if (!shape) return NULL; + size_t slot = scr_js_shape_find(shape, name->data, name->len); + if (slot == SIZE_MAX) return NULL; + const ScrDynEntry *found = &receiver->v.obj.entries[slot]; + if (found->accessor) return NULL; + scr_js_ic_add(ic, UINT32_MAX, (ScrJsIcEntry){NULL, NULL, NULL, NULL, slot}); + return scr_dyn_retain(found->value); +} diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index 0567d0b15..a3c353162 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -24,6 +24,17 @@ #include "scr_runtime.h" #include "scr_key.h" +/* Programs with dynamic-tier code link the SCR_JS_TIER build of this unit: + * its object nodes carry the tier's hidden class (ScrDyn.v.obj.shape), and + * every allocation and change to an object's key set resets it so the + * tier recomputes it on its next cache miss. Other programs link the + * ordinary build, where this expands to nothing. */ +#ifdef SCR_JS_TIER +#define SCR_JS_SHAPE_RESET(d) ((d)->v.obj.shape = NULL) +#else +#define SCR_JS_SHAPE_RESET(d) ((void)0) +#endif + static SCR_TL ScrDyn *scr_builtin_object_prototype; /* Constructor capsules expose identity and name/length, but do not model @@ -933,6 +944,7 @@ static ScrDyn *scr_dyn_alloc(ScrDynKind kind) { } else if (kind == SCR_DYN_OBJ) { d->v.obj.len = 0; /* cap/entries preserved */ d->v.obj.region = NULL; + SCR_JS_SHAPE_RESET(d); if (d->v.obj.cap <= SCR_DYN_KEY_CACHE_SLOTS) { for (size_t i = 0; i < d->v.obj.cap; i++) { ScrDynEntry *entry = &d->v.obj.entries[i]; @@ -957,6 +969,9 @@ static ScrDyn *scr_dyn_alloc(ScrDynKind kind) { fresh->prototype = NULL; fresh->symbol_properties = NULL; fresh->symbol_keys = NULL; +#ifdef SCR_JS_TIER + if (kind == SCR_DYN_OBJ) SCR_JS_SHAPE_RESET(fresh); +#endif #ifdef SCR_RC_AUDIT scr_live_dyns++; #endif @@ -1597,6 +1612,7 @@ static void scr_dyn_obj_put(ScrDyn *obj, const char *key, size_t key_len, ScrDyn obj->v.obj.entries = entries; obj->v.obj.cap = cap; } + SCR_JS_SHAPE_RESET(obj); ScrDynEntry *e = &obj->v.obj.entries[obj->v.obj.len++]; if (!e->key || e->key_len != key_len || !scr_key_equal(e->key, key, key_len)) { char *owned = scr_dyn_key_copy(key, key_len); @@ -8033,6 +8049,7 @@ static void scr_dyn_obj_append_shared(ScrDyn *obj, char *key, size_t key_len, Sc obj->v.obj.entries = entries; obj->v.obj.cap = cap; } + SCR_JS_SHAPE_RESET(obj); ScrDynEntry *e = &obj->v.obj.entries[obj->v.obj.len++]; if (e->key != key) { ((ScrDynKey *)(key - offsetof(ScrDynKey, bytes)))->rc++; @@ -8816,6 +8833,7 @@ static void scr_json_delete_member(ScrDyn *object, const ScrStr *key) { ScrDynEntry removed = *entry; memmove(entry, entry + 1, (object->v.obj.len - i - 1) * sizeof *entry); object->v.obj.len--; + SCR_JS_SHAPE_RESET(object); free(object->v.obj.index); object->v.obj.index = NULL; /* The moved tail aliases a live entry; spare slots own only their own @@ -13086,3 +13104,17 @@ ScrDyn *scr_caught_to_dyn(const ScrCaught *c) { return scr_dyn_new_obj(); } } + +#ifdef SCR_JS_TIER +/* ── dynamic-tier primitives (scr_dyn.h, SCR_JS_TIER) ─────────────────── */ + +const ScrDyn *scr_dyn_tier_object_prototype(void) { return scr_builtin_object_prototype; } + +char *scr_dyn_tier_key_copy(const char *key, size_t key_len) { return scr_dyn_key_copy(key, key_len); } + +void scr_dyn_tier_key_release(char *key, size_t key_len) { scr_dyn_key_release(key, key_len); } + +void scr_dyn_tier_append(ScrDyn *obj, char *key, size_t key_len, ScrDyn *value) { + scr_dyn_obj_append_shared(obj, key, key_len, value); +} +#endif diff --git a/tests/corpus/6148-shape-caches.js b/tests/corpus/6148-shape-caches.js new file mode 100644 index 000000000..08f65c218 --- /dev/null +++ b/tests/corpus/6148-shape-caches.js @@ -0,0 +1,206 @@ +// Dynamic-tier inline caches over shaped objects: each property site below +// runs many times over objects of several shapes, so its cache fills, +// rotates and must notice every change a cached answer depends on: new and +// deleted keys, prototype changes, accessors, read-only and frozen +// properties, non-extensible objects, arrays and parsed records. +function dyn(value) { + return value; +} +function getX(o) { + return o.x; +} +function setX(o, v) { + o.x = v; +} +function setXStrict(o, v) { + "use strict"; + o.x = v; +} +function getAt(o, k) { + return o[k]; +} +function callM(o) { + return o.m(); +} + +// One read site, many shapes (polymorphic and past the cache size). +var shapes = dyn([ + { x: 1 }, + { y: 2, x: 3 }, + { x: 4, y: 5, z: 6 }, + { w: 0, x: 7 }, + { a: 1, b: 2, x: 8 }, + { x: 9, q: true }, + { nox: 1 }, +]); +var total = 0; +for (var round = 0; round < 3; round++) + for (var i = 0; i < shapes.length; i++) { + var v = getX(shapes[i]); + total += v === undefined ? 100 : v; + } +console.log("poly", total); + +// Writes: replace, then add, then a key that appears later. +var built = dyn([]); +for (var i = 0; i < 6; i++) { + var o = dyn(i % 2 ? { x: 0 } : {}); + setX(o, i * 10); + built.push(o); +} +console.log("set", JSON.stringify(built)); +var grow = dyn({ a: 1 }); +for (var i = 0; i < 4; i++) { + setX(grow, i); + grow["k" + i] = i; +} +console.log("grow", Object.keys(grow).join(","), getX(grow)); + +// Deleting and re-adding a key moves it to the end of the key order. +var del = dyn({ x: 1, y: 2 }); +for (var i = 0; i < 3; i++) getX(del); +delete del.x; +console.log("deleted", getX(del), "x" in del, Object.keys(del).join(",")); +setX(del, 5); +console.log("readded", getX(del), Object.keys(del).join(",")); + +// Integer-like keys enumerate first, ascending, whatever the insertion order. +var ints = dyn({}); +ints.b = 1; +ints[2] = "two"; +ints.a = 2; +ints[1] = "one"; +ints["10"] = "ten"; +for (var i = 0; i < 3; i++) getAt(ints, "a"); +console.log("order", Object.keys(ints).join(","), getAt(ints, 2), getAt(ints, "1")); + +// Prototype methods: cached through the prototype, then shadowed, replaced +// and added on the prototype after instances exist. +function Point(x) { + this.x = x; +} +Point.prototype.m = function () { + return "proto:" + this.x; +}; +var pts = dyn([new Point(1), new Point(2), new Point(3)]); +var out = dyn([]); +for (var i = 0; i < pts.length; i++) out.push(callM(pts[i])); +pts[1].m = function () { + return "own:" + this.x; +}; +for (var i = 0; i < pts.length; i++) out.push(callM(pts[i])); +Point.prototype.m = function () { + return "replaced:" + this.x; +}; +for (var i = 0; i < pts.length; i++) out.push(callM(pts[i])); +console.log("methods", out.join(" ")); +Point.prototype.later = 42; +console.log("later", pts[0].later, pts[2].later, "later" in pts[1]); +delete dyn(Point.prototype).later; +console.log("gone", pts[0].later); + +// Getters and setters are never answered by a cache. +var counter = 0; +var acc = dyn({ + get x() { + counter++; + return "getter" + counter; + }, +}); +var mixed = dyn([{ x: "plain" }, acc, { x: "plain2" }, acc]); +var got = dyn([]); +for (var i = 0; i < mixed.length; i++) got.push(getX(mixed[i])); +console.log("getters", got.join(" "), counter); +var sets = dyn([]); +var withSetter = dyn({ + /** @param {any} v */ + set x(v) { + sets.push(v); + }, +}); +for (var i = 0; i < 3; i++) setX(withSetter, i); +console.log("setters", sets.join(",")); +var inherits = Object.create(withSetter); +setX(inherits, "via-proto"); +console.log("proto-setter", sets.join(","), Object.keys(inherits).length); + +// Read-only, frozen and non-extensible objects (strict writes throw). +var ro = dyn({ x: 1 }); +Object.defineProperty(ro, "x", { writable: false }); +try { + setXStrict(ro, 2); +} catch (error) { + console.log("ro-throw", error instanceof TypeError); +} +console.log("ro", getX(ro)); +var frozen = dyn({ x: 1 }); +for (var i = 0; i < 3; i++) setX(frozen, i); +Object.freeze(frozen); +try { + setXStrict(frozen, 99); +} catch (error) { + console.log("frozen-throw", error instanceof TypeError); +} +console.log("frozen", getX(frozen), Object.isFrozen(frozen)); +var closed = dyn({ y: 1 }); +Object.preventExtensions(closed); +try { + setXStrict(closed, 3); +} catch (error) { + console.log("closed-throw", error instanceof TypeError); +} +console.log("closed", getX(closed), "x" in closed); + +// Arrays: elements, holes, length and out-of-range indexes. +var arr = dyn([10, 20, 30]); +var holey = dyn([1, , 3]); +var reads = dyn([]); +for (var i = -1; i < 5; i++) reads.push(String(getAt(arr, i)), String(getAt(holey, i))); +console.log("arrays", reads.join(","), getAt(arr, 1.5), getAt(arr, "1"), arr.length, holey.length); +arr.length = 1; +console.log("truncated", getAt(arr, 1), arr.length); + +// Parsed records share shapes; writes keep later reads exact. +var parsed = JSON.parse('[{"id":1,"name":"a"},{"id":2,"name":"b"},{"name":"c","id":3}]'); +var ids = dyn([]); +for (var i = 0; i < parsed.length; i++) ids.push(getAt(parsed[i], "id") + ":" + getAt(parsed[i], "name")); +parsed[0].name = "changed"; +parsed[1].extra = true; +delete dyn(parsed[2]).id; +for (var i = 0; i < parsed.length; i++) ids.push(getAt(parsed[i], "id") + ":" + getAt(parsed[i], "name")); +console.log("parsed", ids.join(" "), JSON.stringify(parsed)); + +// A wide object (dictionary mode) and keys that change its layout. +var wide = dyn({}); +for (var i = 0; i < 100; i++) wide["p" + i] = i; +var sum = 0; +for (var i = 0; i < 100; i++) sum += getAt(wide, "p" + i); +setX(wide, "wx"); +console.log("wide", sum, getX(wide), Object.keys(wide).length); + +// Prototype swaps and null-prototype objects. +var protoA = dyn({ x: "A" }); +var protoB = dyn({ x: "B" }); +var child = Object.create(protoA); +var seen = dyn([]); +for (var i = 0; i < 3; i++) seen.push(getX(child)); +Object.setPrototypeOf(child, protoB); +seen.push(getX(child)); +protoB.x = "B2"; +seen.push(getX(child)); +setX(child, "own"); +seen.push(getX(child), protoB.x); +var bare = Object.create(null); +setX(bare, "bare"); +seen.push(getX(bare)); +console.log("protos", seen.join(" ")); + +// Numbers written back into objects keep their identity-free values. +var acc2 = dyn({ n: 0, f: 0.5 }); +for (var i = 0; i < 1000; i++) { + acc2.n += i; + acc2.f = acc2.f * 1.5 - acc2.n / 1000; +} +var alias = acc2.n; +acc2.n = -0; +console.log("numbers", acc2.n, Object.is(acc2.n, -0), alias, acc2.f.toFixed(6)); diff --git a/tests/corpus/6149-jsval-integer-paths.js b/tests/corpus/6149-jsval-integer-paths.js new file mode 100644 index 000000000..71ac26c3a --- /dev/null +++ b/tests/corpus/6149-jsval-integer-paths.js @@ -0,0 +1,45 @@ +// Dynamic-tier `%` and bitwise operators take an integer path for small +// integral operands and keep JavaScript's exact results at every edge: +// negative zero, NaN, infinities, fractions, operands beyond 2^31 (ToInt32 +// wraps them), and shift counts outside 0..31. +function dyn(value) { + return value; +} + +function show(label, value) { + console.log(label, Object.is(value, -0) ? "-0" : String(value)); +} + +var values = dyn([0, -0, 1, -1, 7, -7, 2.5, -2.5, 3.9, -3.9, 2147483647, -2147483648, + 2147483648, -2147483649, 4294967295, 4294967296 + 5, 4.5e15 + 0.5, 9007199254740994, 1e18, + -1e18, 9223372036854775807, 1e20, -1e20, NaN, Infinity, -Infinity]); +var divisors = dyn([1, -1, 3, -3, 0, -0, 2.5, 2147483648, NaN, Infinity]); +var shifts = dyn([0, 1, 31, 32, 33, -1, -33, 2.7]); + +for (var i = 0; i < values.length; i++) { + var x = values[i]; + var rem = []; + for (var j = 0; j < divisors.length; j++) { + var r = x % divisors[j]; + rem.push(Object.is(r, -0) ? "-0" : String(r)); + } + show("rem " + x, rem.join(" ")); + var bits = [x | 0, x & 0xff, x ^ -1, x | 1, ~~x, x >>> 0]; + for (var k = 0; k < shifts.length; k++) bits.push(x << shifts[k], x >> shifts[k], x >>> shifts[k]); + show("bits " + x, bits.join(" ")); +} + +// A loop whose accumulator stays an integer, then leaves the int32 range. +var h = dyn(0); +var codes = dyn([7, 1919, 65535, 251, 3]); +for (var round = 0; round < 1000; round++) + for (var c = 0; c < codes.length; c++) h = ((h * 31) ^ codes[c]) | 0; +show("hash", h); +var acc = dyn(1); +for (var n = 0; n < 40; n++) acc = (acc * 3) % 4294967311; +show("big-mod", acc); +show("neg-rem-zero", dyn(-4) % 2); +show("neg-zero-rem", dyn(-0) % 5); +show("rem-neg-divisor", dyn(5) % -3); +show("min-int", dyn(-2147483648) % -1); +show("ushr-negative", dyn(-1) >>> 28); From 302f0f9fa99fc2e72b41aa2920f2a51fb7fe275f Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Mon, 19 Oct 2026 17:02:31 -0500 Subject: [PATCH 02/38] Compile the remaining corpus programs with the native compiler The native compiler failed on corpus programs that the Node-hosted compiler compiles. Four compiler causes: - A record that width-lifts into several members of a discriminated union had no conversion plan. As a field of a larger record, that removed the outer member it belonged to, and the outer record could then lift into the only member left: a `recordSet` statement built with a nested `recordGet` literal became a `break` statement, and a switch on `kind` read it with the `recordSet` layout. The native compiler stopped at the narrowing guard on 5585-record-results-inlined.ts and similar programs. Such a record now converts through the discriminant: each member it width-lifts into is a route, and the value's discriminant selects the member at run time, as same-layout members already do. - Three compiler paths asserted a value that can be null, and the native compiler threw "null is not representable in the target union": stableCreateRequireBindingReason7 (`cache.get(decl)!` from a cache that stores null for stable createRequire bindings; 2631-create-require, 2932-create-require-program), the Int32 contract check (`catchLocalId!` for a bindingless `catch {}`; 4970-int32-call-abi) and StringIndexBounds.binding (a memo that stores null). The caches are read without assertions and the catch test checks for null. - scriptc adopts alwaysStrict, esModuleInterop and allowSyntheticDefaultImports from a project's tsconfig, but the TS 7 SDK schema no longer declares them. A native compiler keeps only declared fields when it builds or parses a typed record, so these options were dropped, and the default-import check refused `import fs from "node:fs"` (SC1012, 2391-default-builtin-imports and the 3055 to 3057 module-setting variants). The options a project reports are parsed with these fields declared, and Ts7CompilerOptions uses the same type. - An object-literal method kept its inferred return type where an arrow function or function expression takes the return type of its contextual signature. For a method that returns an array of statement literals, a `varDecl` literal with `init: null` was converted to the member whose `init` holds only expressions, and the conversion threw (Array.fromAsync's array loop: 5854-array-from-async, 5855-array-from-async-js). A synchronous object-literal method now takes its slot from the property of the same name on the literal's contextual type and adopts its return the same way. Adds corpus 6022 and 6024. 2391, 2631, 3055, 4970, 5585 and 5854 run in the native-frontend self-host check, which now also links the dyn-async unit when the program uses it, as the production build does. --- packages/compiler/src/contracts/check.ts | 5 +- .../src/frontend/lowering/call-signatures.ts | 37 +++++- .../lowering/coercions/structural-plans.ts | 16 +++ .../frontend/lowering/coercions/structural.ts | 7 +- .../src/frontend/lowering/coercions/unions.ts | 10 +- .../compiler/src/frontend/lowering/lowerer.ts | 3 +- .../frontend/lowering/string-index-bounds.ts | 3 +- .../src/frontend/lowering/union-retag.ts | 36 +++--- .../src/frontend/lowering/width-lift.ts | 2 +- packages/compiler/src/frontend/program.ts | 3 +- .../compiler/src/frontend/ts7/program-host.ts | 15 ++- packages/compiler/src/frontend/ts7/session.ts | 30 ++++- .../6022-width-lift-discriminant.ts.json | 6 + ...-object-literal-method-slot-return.ts.json | 6 + tests/corpus/6022-width-lift-discriminant.ts | 54 +++++++++ .../6024-object-literal-method-slot-return.ts | 114 ++++++++++++++++++ .../6022-width-lift-discriminant.ts.json | 1 + ...-object-literal-method-slot-return.ts.json | 1 + tests/harness/native-frontend.ts | 8 ++ 19 files changed, 312 insertions(+), 45 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6022-width-lift-discriminant.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6024-object-literal-method-slot-return.ts.json create mode 100644 tests/corpus/6022-width-lift-discriminant.ts create mode 100644 tests/corpus/6024-object-literal-method-slot-return.ts create mode 100644 tests/harness/corpus-costs/6022-width-lift-discriminant.ts.json create mode 100644 tests/harness/corpus-costs/6024-object-literal-method-slot-return.ts.json diff --git a/packages/compiler/src/contracts/check.ts b/packages/compiler/src/contracts/check.ts index 3c25c9b1b..5934624bf 100644 --- a/packages/compiler/src/contracts/check.ts +++ b/packages/compiler/src/contracts/check.ts @@ -889,10 +889,7 @@ class Int32Contracts { const value = s.kind === "varDecl" ? s.init : s.value; if (note && value && !int32(value)) fail(note, value, value.loc, "the value stored into"); - } else if ( - s.kind === "forOf" || - (s.kind === "tryCatch" && s.catchLocalId !== undefined) - ) { + } else if (s.kind === "forOf" || (s.kind === "tryCatch" && s.catchLocalId !== null)) { const id = s.kind === "forOf" ? s.localId : s.catchLocalId!; const note = localNote(id); if (note) diff --git a/packages/compiler/src/frontend/lowering/call-signatures.ts b/packages/compiler/src/frontend/lowering/call-signatures.ts index 9d9c9668c..55f46a7b0 100644 --- a/packages/compiler/src/frontend/lowering/call-signatures.ts +++ b/packages/compiler/src/frontend/lowering/call-signatures.ts @@ -864,9 +864,18 @@ export function lambdaSignature( // Union-inferred returns adopt too (a mixed-return body inferring a // SUB-union of the slot's union — adopting is a no-op when the two + // A synchronous object-literal method adopts its slot's return the same + // way: its slot is the property of that name on the literal's + // contextual type. + const literalMethod = + ts.isMethodDeclaration(node) && + ts.isObjectLiteralExpression(node.parent) && + !node.asteriskToken && + node.modifiers?.nodes.some((m) => m.kind === ts.SyntaxKind.AsyncKeyword) !== true; if ( - (ts.isArrowFunction(node) || ts.isFunctionExpression(node)) && - (!isAsyncLike || ret.kind === "promise") + ((ts.isArrowFunction(node) || ts.isFunctionExpression(node)) && + (!isAsyncLike || ret.kind === "promise")) || + literalMethod ) { // The slot's signature: the contextual type stripped of its nullish // parts (an OPTIONAL callback field's context is the whole @@ -874,7 +883,10 @@ export function lambdaSignature( // declared in USER code: stdlib callback slots (flatMap's // `U | readonly U[]`, sort comparators) have intrinsic lowerings // that inspect the INFERRED type, so they must not widen. - const ctxType = lowerer.checker.getContextualType(node); + const ctxType = + literalMethod && ts.isMethodDeclaration(node) && node.parent + ? objectLiteralMethodSlot(lowerer, node, node.parent) + : lowerer.checker.getContextualType(node); const ctxSigs = ctxType ? lowerer.checker.getCallSignatures(lowerer.checker.getNonNullableType(ctxType)) : []; @@ -943,6 +955,7 @@ export function lambdaSignature( const inferredLiteral = literalSlot !== undefined && (literalType === null || literalType.kind === "dyn"); if ( + !literalMethod && !isAsyncLike && !node.type && !inferredLiteral && @@ -958,3 +971,21 @@ export function lambdaSignature( const funcType = funcTypeFromParamShapes(shapes, ret); return { shapes, funcType }; } + +/** The type of the property an object-literal method fills on the + * literal's contextual type; undefined when the literal has no contextual + * type or the method has a computed name. */ +function objectLiteralMethodSlot( + lowerer: Lowerer, + node: ts.MethodDeclaration, + literal: ts.Node, +): ts.Type | undefined { + if (!ts.isIdentifier(node.name) && !ts.isStringLiteral(node.name)) return undefined; + const literalType = lowerer.checker.getContextualType(literal); + if (!literalType) return undefined; + const property = lowerer.checker.getPropertyOfType( + lowerer.checker.getNonNullableType(literalType), + node.name.text, + ); + return property ? lowerer.checker.getTypeOfSymbol(property) : undefined; +} diff --git a/packages/compiler/src/frontend/lowering/coercions/structural-plans.ts b/packages/compiler/src/frontend/lowering/coercions/structural-plans.ts index 78c7cb73d..83798a89c 100644 --- a/packages/compiler/src/frontend/lowering/coercions/structural-plans.ts +++ b/packages/compiler/src/frontend/lowering/coercions/structural-plans.ts @@ -78,6 +78,22 @@ export function widthLiftPlan(lowerer: Lowerer, src: IrType, dst: IrType): Width }); if (absentOnly.length === 1) selected = absentOnly; } + // A record that width-lifts into several members of a discriminated + // union is converted by its discriminant's value at run time, when + // every candidate member is one of the routes. + if (selected.length > 1 && src.kind === "record" && def.discriminant) { + const shape = lowerer.shapes.get(src.shapeId); + const plan = + shape && + planRecordUnionWrap( + shape, + def, + (id) => lowerer.shapes.get(id), + (from, to) => lowerer.widthLiftPlan(from, to), + ); + if (plan && selected.every(({ tag }) => plan.routes.some((route) => route.tag === tag))) + return { how: "discriminantWrap", width: true }; + } if (selected.length !== 1) return null; return { how: "liftWrap", tag: selected[0]!.tag, arm: selected[0]!.arm }; } diff --git a/packages/compiler/src/frontend/lowering/coercions/structural.ts b/packages/compiler/src/frontend/lowering/coercions/structural.ts index 925f59d9b..5931b33a7 100644 --- a/packages/compiler/src/frontend/lowering/coercions/structural.ts +++ b/packages/compiler/src/frontend/lowering/coercions/structural.ts @@ -99,7 +99,12 @@ export function applyWidthLift( case "discriminantWrap": { if (dst.kind !== "union" || value.type.kind !== "record") throw new InternalCompilerError("lowerer bug: record discriminator lift shape"); - const helper = lowerer.recordUnionWrapHelper(value.type, dst.unionId, loc); + const helper = lowerer.recordUnionWrapHelper( + value.type, + dst.unionId, + loc, + lift.width === true, + ); if (!helper) throw new InternalCompilerError( "lowerer bug: planned record discriminator lift failed to intern", diff --git a/packages/compiler/src/frontend/lowering/coercions/unions.ts b/packages/compiler/src/frontend/lowering/coercions/unions.ts index 0356b714e..84c625c3f 100644 --- a/packages/compiler/src/frontend/lowering/coercions/unions.ts +++ b/packages/compiler/src/frontend/lowering/coercions/unions.ts @@ -265,13 +265,19 @@ export function recordUnionWrapHelper( source: IrType & { kind: "record" }, toId: string, loc: SrcLoc, + width = false, ): string | null { const shape = lowerer.shapes.get(source.shapeId); const to = lowerer.unions.get(toId); if (!shape || !to) return null; - const plan = planRecordUnionWrap(shape, to, (id) => lowerer.shapes.get(id)); + const plan = planRecordUnionWrap( + shape, + to, + (id) => lowerer.shapes.get(id), + width ? (from, dst) => lowerer.widthLiftPlan(from, dst) : undefined, + ); if (!plan) return null; - const key = `recordWrap:${source.shapeId}:${toId}`; + const key = `recordWrap${width ? "Width" : ""}:${source.shapeId}:${toId}`; const existing = lowerer.coercions.retags.get(key); if (existing) return existing; const name = `%record.wrap.${lowerer.coercions.retags.size}`; diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index 9e7359a34..3c9a18713 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -9143,8 +9143,9 @@ export class Lowerer { source: IrType & { kind: "record" }, toId: string, loc: SrcLoc, + width = false, ): string | null { - return recordUnionWrapHelper(this, source, toId, loc); + return recordUnionWrapHelper(this, source, toId, loc, width); } unionRetagHelper( diff --git a/packages/compiler/src/frontend/lowering/string-index-bounds.ts b/packages/compiler/src/frontend/lowering/string-index-bounds.ts index acb41514f..32b96e63a 100644 --- a/packages/compiler/src/frontend/lowering/string-index-bounds.ts +++ b/packages/compiler/src/frontend/lowering/string-index-bounds.ts @@ -243,7 +243,8 @@ export class StringIndexBounds { /** Every write of a block-scoped local or parameter, or null for any * other binding (module imports, var, class members, ...). */ private binding(symbol: ts.Symbol): Binding | null { - if (this.bindings.has(symbol)) return this.bindings.get(symbol)!; + const cached = this.bindings.get(symbol); + if (cached !== undefined) return cached; const result = this.scanBinding(symbol); this.bindings.set(symbol, result); return result; diff --git a/packages/compiler/src/frontend/lowering/union-retag.ts b/packages/compiler/src/frontend/lowering/union-retag.ts index 8d1c61d09..82ec8226c 100644 --- a/packages/compiler/src/frontend/lowering/union-retag.ts +++ b/packages/compiler/src/frontend/lowering/union-retag.ts @@ -41,11 +41,15 @@ export interface RecordUnionWrapPlan { /** Recursive variants can retain distinct ids for the same field layout. * An independently inferred record may intern to either id, so that id - * alone cannot select its semantic destination when wrapping into a union. */ + * alone cannot select its semantic destination when wrapping into a union. + * With `widthLift`, members the record only width-lifts into are routes + * too: a record's own fields do not say which of them its discriminant + * names, so the value's discriminant selects the member at run time. */ export function planRecordUnionWrap( source: IrRecordShape, to: IrUnionDef, shapeOf: (id: string) => IrRecordShape | undefined, + widthLift?: (src: IrType, dst: IrType) => WidthLift | null, ): RecordUnionWrapPlan | null { const discriminant = to.discriminant; if (!discriminant || source.tuple || source.indexValue || shapeHasAccessorSlots(source)) @@ -57,30 +61,22 @@ export function planRecordUnionWrap( const arm = to.arms[tag]!; if (arm.kind !== "record") continue; const target = shapeOf(arm.shapeId); - if ( - !target || - target.tuple || - target.indexValue || - target.fields.length !== source.fields.length - ) - continue; - let same = true; - for (let i = 0; i < source.fields.length; i++) { + if (!target || target.tuple || target.indexValue) continue; + let same = target.fields.length === source.fields.length; + for (let i = 0; same && i < source.fields.length; i++) { const a = source.fields[i]!, b = target.fields[i]!; - if (a.name !== b.name || !typeEquals(a.type, b.type)) { - same = false; - break; - } + if (a.name !== b.name || !typeEquals(a.type, b.type)) same = false; } - if (!same) continue; + let lift: WidthLift | null = same + ? { how: arm.shapeId === source.id ? "copy" : "width" } + : null; + if (lift === null && widthLift !== undefined) + lift = widthLift({ kind: "record", shapeId: source.id }, arm); + if (lift === null) continue; const entry = discriminant.cases.find((candidate) => candidate.tag === tag); if (!entry) return null; - routes.push({ - tag, - lift: { how: arm.shapeId === source.id ? "copy" : "width" }, - values: entry.values, - }); + routes.push({ tag, lift, values: entry.values }); } if (routes.length < 2 || discriminantOwners(to, shapeOf) === null) return null; return { field: discriminant.field, fieldType, routes }; diff --git a/packages/compiler/src/frontend/lowering/width-lift.ts b/packages/compiler/src/frontend/lowering/width-lift.ts index 40c533d72..6b1ad8c84 100644 --- a/packages/compiler/src/frontend/lowering/width-lift.ts +++ b/packages/compiler/src/frontend/lowering/width-lift.ts @@ -6,7 +6,7 @@ import type { IrType } from "../../ir/ir.js"; export type WidthLift = | { how: "copy" } | { how: "wrap"; tag: number } - | { how: "discriminantWrap" } + | { how: "discriminantWrap"; width?: true } | { how: "retag" } | { how: "liftWrap"; tag: number; arm: IrType } | { how: "width" } diff --git a/packages/compiler/src/frontend/program.ts b/packages/compiler/src/frontend/program.ts index bde168c38..5270f7bb7 100644 --- a/packages/compiler/src/frontend/program.ts +++ b/packages/compiler/src/frontend/program.ts @@ -2042,7 +2042,8 @@ function stableCreateRequireBindingReason7( decl: ts.VariableDeclaration, ): string | null { const cache = program.analysis.createRequireReasons; - if (cache.has(decl)) return cache.get(decl)!; + const cached = cache.get(decl); + if (cached !== undefined) return cached; const check = (): string | null => { const stmt = decl.parent?.parent; if ( diff --git a/packages/compiler/src/frontend/ts7/program-host.ts b/packages/compiler/src/frontend/ts7/program-host.ts index 2f2fdf76b..fa1eeab8f 100644 --- a/packages/compiler/src/frontend/ts7/program-host.ts +++ b/packages/compiler/src/frontend/ts7/program-host.ts @@ -21,9 +21,13 @@ import { dirname, join, resolve } from "node:path"; import { Ts7Api } from "./api.js"; import type { Ts7TimingInfo } from "./session-timing.js"; import type { Ts7FileSystem } from "./rpc-filesystem.js"; -import type { CompilerOptions, Diagnostic } from "./semantic-types.js"; +import type { Diagnostic } from "./semantic-types.js"; import type { SourceFile } from "./ast-types.js"; -import type { Ts7SessionProject as Project, Ts7SessionSnapshot as Snapshot } from "./session.js"; +import type { + Ts7ProjectCompilerOptions, + Ts7SessionProject as Project, + Ts7SessionSnapshot as Snapshot, +} from "./session.js"; import { CheckerFacade } from "./checker.js"; import { moduleDetectionKindName, @@ -44,8 +48,9 @@ import { /** The compiler options our createProgram accepts: TS7's CompilerOptions * shape (numeric enums for target/module/moduleResolution — the enums module * generates them from the pinned SDK) with the 5.9.3 "lib.es2025.d.ts" lib - * spelling also accepted. */ -export type Ts7CompilerOptions = CompilerOptions; + * spelling also accepted, plus the project options scriptc adopts that the + * TS7 schema no longer declares (Ts7ProjectCompilerOptions). */ +export type Ts7CompilerOptions = Ts7ProjectCompilerOptions; /* tsconfig JSON wants enum NAMES; the options object carries TS7's numeric * enum values. Generated name lookups preserve the SDK's reverse mappings @@ -282,7 +287,7 @@ export class Ts7Program { private readonly configPath: string, ) {} - getCompilerOptions(): CompilerOptions { + getCompilerOptions(): Ts7CompilerOptions { return this.project.program.getCompilerOptions(); } diff --git a/packages/compiler/src/frontend/ts7/session.ts b/packages/compiler/src/frontend/ts7/session.ts index 4933da9d6..bc4c09496 100644 --- a/packages/compiler/src/frontend/ts7/session.ts +++ b/packages/compiler/src/frontend/ts7/session.ts @@ -17,6 +17,24 @@ import type { Ts7SourceMetadata, } from "./session-schema.generated.js"; +/** Project options tsgo still reports for a tsconfig but the TS 7 SDK + * schema no longer declares. scriptc adopts them (ADOPTED_OPTIONS), so the + * options a project reports are parsed with them declared: a native + * compiler keeps only declared fields when it parses the response. */ +export interface Ts7ProjectCompilerOptions extends Ts7CompilerOptionsData { + alwaysStrict?: boolean; + esModuleInterop?: boolean; + allowSyntheticDefaultImports?: boolean; +} + +interface Ts7ReportedProjectData extends Ts7ProjectData { + compilerOptions: Ts7ProjectCompilerOptions; +} + +interface Ts7ReportedSnapshotData extends Ts7SnapshotData { + projects: Ts7ReportedProjectData[]; +} + export interface Ts7FileChanges { changed?: SemanticDocument[]; created?: SemanticDocument[]; @@ -103,7 +121,7 @@ export class Ts7Session { updateSnapshot(params: Ts7Update = {}): Ts7SessionSnapshot { const paths = this.initialize(); - const data = parseSemanticJson( + const data = parseSemanticJson( this.text("updateSnapshot", JSON.stringify(params)), ); const snapshot = new Ts7SessionSnapshot(data, this, paths); @@ -159,7 +177,7 @@ export class Ts7SessionSnapshot { private disposed = false; constructor( - data: Ts7SnapshotData, + data: Ts7ReportedSnapshotData, private readonly session: Ts7Session, readonly paths: Ts7Paths, ) { @@ -188,7 +206,7 @@ export class Ts7SessionSnapshot { } getDefaultProjectForFile(file: SemanticDocument): Ts7SessionProject | undefined { this.ensureActive(); - const data = this.session.request("getDefaultProjectForFile", { + const data = this.session.request("getDefaultProjectForFile", { snapshot: this.id, file, }); @@ -214,12 +232,12 @@ export class Ts7SessionSnapshot { export class Ts7SessionProject { readonly id: string; readonly configFileName: string; - readonly compilerOptions: Ts7CompilerOptionsData; + readonly compilerOptions: Ts7ProjectCompilerOptions; readonly rootFiles: string[]; readonly program: Ts7SessionProgram; readonly checker: SemanticChecker; - constructor(data: Ts7ProjectData, snapshot: Ts7SessionSnapshot, session: Ts7Session) { + constructor(data: Ts7ReportedProjectData, snapshot: Ts7SessionSnapshot, session: Ts7Session) { this.id = data.id; this.configFileName = data.configFileName; this.compilerOptions = data.compilerOptions; @@ -260,7 +278,7 @@ export class Ts7SessionProgram { ...(file === undefined ? {} : { file }), }; } - getCompilerOptions(): Ts7CompilerOptionsData { + getCompilerOptions(): Ts7ProjectCompilerOptions { this.ensureActive(); return this.project.compilerOptions; } diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6022-width-lift-discriminant.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6022-width-lift-discriminant.ts.json new file mode 100644 index 000000000..38c0cb787 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6022-width-lift-discriminant.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6022-width-lift-discriminant.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6024-object-literal-method-slot-return.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6024-object-literal-method-slot-return.ts.json new file mode 100644 index 000000000..c04cbf304 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6024-object-literal-method-slot-return.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6024-object-literal-method-slot-return.ts" + ], + "diags": [] +} diff --git a/tests/corpus/6022-width-lift-discriminant.ts b/tests/corpus/6022-width-lift-discriminant.ts new file mode 100644 index 000000000..73bb5059b --- /dev/null +++ b/tests/corpus/6022-width-lift-discriminant.ts @@ -0,0 +1,54 @@ +// A record built in a map callback whose fields width-lift into several +// members of a discriminated union (a nested literal with a narrowed field, +// and a member that only needs `kind` and `loc`): its discriminant selects +// the member. +interface Loc { + start: number; +} +type Expr = + | { kind: "get"; obj: Expr; field: string; loc: Loc } + | { kind: "ref"; name: string; loc: Loc } + | { kind: "undef"; loc: Loc }; +type Stmt = + | { kind: "set"; obj: Expr; field: string; value: Expr; loc: Loc } + | { kind: "break"; label?: string; loc: Loc } + | { kind: "return"; value: Expr | null; loc: Loc }; + +function showExpr(expr: Expr): string { + switch (expr.kind) { + case "get": + return `${showExpr(expr.obj)}.${expr.field}`; + case "ref": + return expr.name; + case "undef": + return "undefined"; + } +} + +function show(stmt: Stmt): string { + switch (stmt.kind) { + case "set": + return `${showExpr(stmt.obj)}.${stmt.field} = ${showExpr(stmt.value)}`; + case "break": + return `break${stmt.label === undefined ? "" : " " + stmt.label}`; + case "return": + return `return ${stmt.value === null ? "-" : showExpr(stmt.value)}`; + } +} + +function copies(fields: string[], source: Expr & { kind: "ref" }, dst: Expr, loc: Loc): Stmt[] { + return fields.map((field) => ({ + kind: "set", + obj: dst, + field, + value: { kind: "get", obj: source, field, loc }, + loc, + })); +} + +const loc: Loc = { start: 0 }; +const out: Expr = { kind: "ref", name: "out", loc }; +const span: Expr = { kind: "ref", name: "span", loc }; +if (span.kind === "ref") console.log(copies(["lo", "hi"], span, out, loc).map(show).join("; ")); +const stops: Stmt[] = ["a", "b"].map((label) => ({ kind: "break", label, loc })); +console.log(stops.map(show).join("; ")); diff --git a/tests/corpus/6024-object-literal-method-slot-return.ts b/tests/corpus/6024-object-literal-method-slot-return.ts new file mode 100644 index 000000000..b5fa7ddd5 --- /dev/null +++ b/tests/corpus/6024-object-literal-method-slot-return.ts @@ -0,0 +1,114 @@ +// An object-literal method returns the type its slot declares. The literal +// statements below have their own inferred type, a union of records whose +// `init` fields are `Expr` or `null` and whose tryCatch member has fields +// the declared `Stmt` does not; the method still returns `Stmt[]`. +type Expr = + | { kind: "num"; value: number } + | { kind: "str"; text: string } + | { kind: "ref"; name: string }; +type Stmt = + | { kind: "varDecl"; name: string; init: Expr | null } + | { kind: "exprStmt"; expr: Expr } + | { kind: "if"; cond: Expr; then: Stmt[]; else_: Stmt[] | null } + | { kind: "loop"; init: Stmt | null; body: Stmt[] } + | { + kind: "tryCatch"; + tryBody: Stmt[]; + catchBody: Stmt[] | null; + catchName: string | null; + finallyBody: Stmt[] | null; + }; + +interface Step { + label: string; + build(source: Expr, append: (value: Expr) => Stmt[]): Stmt[]; + first(source: Expr): Stmt | Expr; +} + +const ref = (name: string): Expr => ({ kind: "ref", name }); + +function arrayStep(label: string): Step { + return { + label, + build(source, append) { + const loop: Stmt = { + kind: "loop", + init: { kind: "varDecl", name: "cursor", init: { kind: "num", value: 0 } }, + body: [...append(ref("value"))], + }; + return [ + { kind: "varDecl", name: "array", init: source }, + { kind: "varDecl", name: "close", init: ref("flag") }, + { kind: "varDecl", name: "value", init: null }, + { + kind: "tryCatch", + tryBody: [loop], + catchBody: null, + catchName: null, + finallyBody: [{ kind: "if", cond: ref("close"), then: [], else_: null }], + }, + ]; + }, + first(source) { + if (source.kind === "num") return { kind: "varDecl", name: "n", init: null }; + return source; + }, + }; +} + +function show(expr: Expr | null): string { + if (expr === null) return "null"; + switch (expr.kind) { + case "num": + return String(expr.value); + case "str": + return JSON.stringify(expr.text); + case "ref": + return expr.name; + } +} + +function print(stmts: Stmt[], depth: number): void { + const pad = " ".repeat(depth); + for (const s of stmts) { + switch (s.kind) { + case "varDecl": + console.log(`${pad}let ${s.name} = ${show(s.init)}`); + break; + case "exprStmt": + console.log(`${pad}${show(s.expr)}`); + break; + case "if": + console.log(`${pad}if ${show(s.cond)} (else: ${s.else_ === null ? "none" : s.else_.length})`); + print(s.then, depth + 1); + break; + case "loop": + console.log(`${pad}loop`); + if (s.init) print([s.init], depth + 1); + print(s.body, depth + 1); + break; + case "tryCatch": + console.log(`${pad}try (catch: ${s.catchName ?? "none"})`); + print(s.tryBody, depth + 1); + if (s.catchBody) print(s.catchBody, depth + 1); + if (s.finallyBody) { + console.log(`${pad}finally`); + print(s.finallyBody, depth + 1); + } + break; + } + } +} + +const step = arrayStep("array"); +console.log(step.label); +const stmts = step.build({ kind: "str", text: "items" }, (value) => [ + { kind: "exprStmt", expr: value }, + { kind: "varDecl", name: "done", init: null }, +]); +print(stmts, 0); +console.log(stmts.length, JSON.stringify(stmts[2])); +for (const source of [{ kind: "num", value: 1 }, ref("x")] satisfies Expr[]) { + const head = step.first(source); + console.log(head.kind, "name" in head ? head.name : "", JSON.stringify(head)); +} diff --git a/tests/harness/corpus-costs/6022-width-lift-discriminant.ts.json b/tests/harness/corpus-costs/6022-width-lift-discriminant.ts.json new file mode 100644 index 000000000..c0f25a6f7 --- /dev/null +++ b/tests/harness/corpus-costs/6022-width-lift-discriminant.ts.json @@ -0,0 +1 @@ +{ "plain": 3800, "san": 7400 } diff --git a/tests/harness/corpus-costs/6024-object-literal-method-slot-return.ts.json b/tests/harness/corpus-costs/6024-object-literal-method-slot-return.ts.json new file mode 100644 index 000000000..80c95a135 --- /dev/null +++ b/tests/harness/corpus-costs/6024-object-literal-method-slot-return.ts.json @@ -0,0 +1 @@ +{ "plain": 3000, "san": 4400 } diff --git a/tests/harness/native-frontend.ts b/tests/harness/native-frontend.ts index 475db54aa..048a08251 100644 --- a/tests/harness/native-frontend.ts +++ b/tests/harness/native-frontend.ts @@ -20,6 +20,7 @@ import { moduleUsesBigInt, moduleUsesCopying, moduleUsesDynInvoke, + moduleUsesDynAsync, moduleUsesInspect, moduleUsesRegex, moduleUsesSymbol, @@ -65,7 +66,13 @@ export const programs = [ "nullish-long-chain.ts", "class-array-optional-return.ts", "contextual-array-union-write.ts", + "2391-default-builtin-imports/main.ts", + "3055-default-builtin-nodenext/main.ts", + "5854-array-from-async.ts", "6020-class-extends-function-constructor.js", + "5585-record-results-inlined.ts", + "2631-create-require/main.ts", + "4970-int32-call-abi.ts", ]; interface LowerRequest { @@ -224,6 +231,7 @@ export async function checkProgram(frontend: NativeFrontend, file: string): Prom copying: moduleUsesCopying(module), inspect: moduleUsesInspect(module), dynInvoke: moduleUsesDynInvoke(module), + dynAsync: moduleUsesDynAsync(module), symbol: moduleUsesSymbol(module), bigint: moduleUsesBigInt(module), zlib: moduleUsesZlib(module), From 3701f266817515949e74de5a09bfb3f38974810b Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Mon, 19 Oct 2026 18:53:23 -0700 Subject: [PATCH 03/38] Lower Headers, Request and Response members that undici-types declares as properties @types/node's fetch declarations (undici-types) spell the Headers methods and the body mixin as function-typed properties (readonly append: (...) => void), where scriptc's own declarations use methods. The receiver call lowering treated such a standard-library property as a stored callback: Headers.forEach was refused at build time and append/get/set/... compiled to a runtime 'is not a function'. Standard-library members now take the shared receiver dispatch either way. Co-Authored-By: Claude Opus 5.5 --- .../src/frontend/lowering/lower-calls.ts | 12 +++++++ .../node-types/fetch-handle-members.ts.json | 6 ++++ .../node-types/fetch-handle-members.ts | 32 +++++++++++++++++++ tests/harness/project-config.test.ts | 21 ++++++++++++ 4 files changed, 71 insertions(+) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/node-types/fetch-handle-members.ts.json create mode 100644 tests/fixtures/node-types/fetch-handle-members.ts diff --git a/packages/compiler/src/frontend/lowering/lower-calls.ts b/packages/compiler/src/frontend/lowering/lower-calls.ts index 5a4838782..464499ba9 100644 --- a/packages/compiler/src/frontend/lowering/lower-calls.ts +++ b/packages/compiler/src/frontend/lowering/lower-calls.ts @@ -3665,8 +3665,20 @@ function lowerDynReceiverMethodCall( ) ) return true; + // A standard-library member spelled as a function-typed property + // (undici-types declares Headers.append/forEach/... and the body mixin + // as `readonly append: (...) => void`; scriptc's own declarations use + // methods) is still the handle's native method: it takes the shared + // receiver dispatch below either way (which also calls a plain object's + // own member), so the lowering does not depend on which declaration + // file supplies the type. + const libMethod = + prop !== undefined && + prop.declarations.length > 0 && + prop.declarations.every((decl) => lowerer.isStdlibFile(decl.getSourceFile())); return ( prop !== undefined && + !libMethod && prop.declarations.some( (decl) => ts.isPropertyDeclaration(decl) || ts.isPropertySignature(decl), ) diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/node-types/fetch-handle-members.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/node-types/fetch-handle-members.ts.json new file mode 100644 index 000000000..80464768a --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/node-types/fetch-handle-members.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/node-types/fetch-handle-members.ts" + ], + "diags": [] +} diff --git a/tests/fixtures/node-types/fetch-handle-members.ts b/tests/fixtures/node-types/fetch-handle-members.ts new file mode 100644 index 000000000..07f8f689e --- /dev/null +++ b/tests/fixtures/node-types/fetch-handle-members.ts @@ -0,0 +1,32 @@ +// undici-types (@types/node's fetch declarations) spells the Headers methods +// and the Request/Response body mixin as function-typed properties; they +// must lower to the same native handle methods as the shipped declarations. +async function main(): Promise { + const h = new Headers([["b", "2"], ["a", "1"]]); + h.append("set-cookie", "x=1"); + h.append("set-cookie", "y=2"); + h.set("c", "3"); + h.delete("b"); + const seen: string[] = []; + h.forEach((value, name) => seen.push(`${name}=${value}`)); + console.log(seen.join(","), h.get("a"), h.has("b"), h.getSetCookie().join("|")); + const r1 = new Response("body-text", { status: 202, statusText: "Acc", headers: h }); + console.log(r1.status, r1.statusText, r1.ok, r1.headers.get("c"), r1.bodyUsed); + console.log(await r1.text(), r1.bodyUsed); + const r2 = new Response(JSON.stringify({ k: 1 })); + console.log(JSON.stringify(await r2.json())); + const r3 = new Response(new Uint8Array([1, 2, 3])); + console.log((await r3.arrayBuffer()).byteLength); + const r4 = new Response("abc"); + console.log((await r4.bytes()).length); + const r5 = Response.json({ z: true }, { status: 200 }); + console.log(r5.headers.get("content-type"), await r5.text()); + const req = new Request("http://x.test/a?b=1", { method: "POST", body: "payload", headers: { q: "1" } }); + console.log(req.method, req.url, req.headers.get("q"), await req.text()); + const req2 = new Request("http://x.test/j", { method: "PUT", body: "{\"n\":2}" }); + console.log(JSON.stringify(await req2.json())); + const ac = new AbortController(); + ac.abort(); + console.log(ac.signal.aborted); +} +main(); diff --git a/tests/harness/project-config.test.ts b/tests/harness/project-config.test.ts index 802fbd3e1..d98f12da3 100644 --- a/tests/harness/project-config.test.ts +++ b/tests/harness/project-config.test.ts @@ -52,6 +52,27 @@ test("node-types: filesystem capacity supports typed number and BigInt results u expect(comparableStderr(native.stderr)).toBe(node.stderr); }); +test("node-types: Headers, Request and Response members typed by undici-types run natively", async () => { + const outDir = outDirFor("fetch-handle-members"); + const entry = join(nodeTypesDir, "fetch-handle-members.ts"); + const result = await compile(entry, { + outPath: join( + outDir, + process.platform === "win32" ? "fetch-handle-members.exe" : "fetch-handle-members", + ), + outDir, + sanitize, + }); + expect(result.ok, !result.ok ? JSON.stringify(result.diagnostics, null, 2) : "").toBe(true); + if (!result.ok) return; + const [native, node] = await Promise.all([ + execFileAsync(result.binaryPath), + execFileAsync(process.execPath, ["--disable-warning=MODULE_TYPELESS_PACKAGE_JSON", entry]), + ]); + expect(native.stdout).toBe(node.stdout); + expect(comparableStderr(native.stderr)).toBe(node.stderr); +}); + test("node-types: http header arrays and server.address() follow @types/node's types", async () => { const outDir = outDirFor("http-headers-address"); const entry = join(nodeTypesDir, "http-headers-address.ts"); From 6e67b1da09ea5638b46f1b7b784649a9e3971883 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Mon, 19 Oct 2026 18:53:23 -0700 Subject: [PATCH 04/38] Run the poll phase once per event-loop iteration The net unit drained its poller every time the loop came back to the I/O station, including the restarts that drain microtasks. I/O that became ready during the iteration's own callbacks (a loopback connect started by one of them) was delivered before the check phase, so setImmediate callbacks queued earlier ran after it; Node delivers it in the next iteration's poll phase. The poller now drains once per iteration (a full batch keeps draining), reopening after the check phase. Back-to-back undici requests now reuse connections like Node, so the fixtures drop their setImmediate turn between requests. Co-Authored-By: Claude Opus 5.5 --- .../6251-net-connect-poll-phase-order.ts.json | 6 ++ packages/runtime/src/scr_async.c | 20 ++++++ packages/runtime/src/scr_net.c | 22 +++++- .../6251-net-connect-poll-phase-order.ts | 70 +++++++++++++++++++ tests/fixtures/npm/cases/undici5/main.ts | 1 - tests/fixtures/npm/cases/undici7/main.ts | 1 - 6 files changed, 115 insertions(+), 5 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6251-net-connect-poll-phase-order.ts.json create mode 100644 tests/corpus/6251-net-connect-poll-phase-order.ts diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6251-net-connect-poll-phase-order.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6251-net-connect-poll-phase-order.ts.json new file mode 100644 index 000000000..0d0190a32 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6251-net-connect-poll-phase-order.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6251-net-connect-poll-phase-order.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_async.c b/packages/runtime/src/scr_async.c index 88ad0f7bd..de48d6b61 100644 --- a/packages/runtime/src/scr_async.c +++ b/packages/runtime/src/scr_async.c @@ -4149,6 +4149,19 @@ static int scr_workers_pollfd(void) { * macrotask). */ bool scr_loop_has_ready(void) { return scr_ready_len > 0 || scr_nt_head != NULL; } +/* The poll phase runs once per loop iteration, like libuv's: the net unit + * drains the poller at most once between check phases. I/O that becomes + * ready while this iteration's callbacks run (a loopback connect started + * by one of them) waits for the next iteration, so immediates queued in + * the meantime fire first — Node's poll → check order. */ +static SCR_TL bool scr_loop_io_open = true; +bool scr_loop_io_take(void) { + bool open = scr_loop_io_open; + scr_loop_io_open = false; + return open; +} +void scr_loop_io_reopen(void) { scr_loop_io_open = true; } + /* The process.nextTick queue to exhaustion, ticks queued meanwhile * included. false: a tick threw (uncaught, pending). */ static bool scr_loop_run_ticks(void) { @@ -4618,6 +4631,7 @@ static bool scr_loop_run_pass(ScrPromise *top_level, bool first_checkpoint) { * every macrotask; pending ticks or a woken fiber restart the turn * first (the timer loop's break above), so the phase only runs on a * drained checkpoint. */ + bool check_ran = false; if (scr_ready_len == 0 && scr_nt_head == NULL && scr_nunhandled == 0 && scr_pending_immediates > 0) { size_t end = scr_nimmediates; while (scr_immediates_head < end) { @@ -4648,7 +4662,13 @@ static bool scr_loop_run_pass(ScrPromise *top_level, bool first_checkpoint) { scr_immediates_head = 0; scr_nimmediates = 0; } + check_ran = true; } + /* The next poll phase opens once this iteration's check phase ran (or + * had nothing to run). A check phase deferred for pending microtasks + * keeps it closed, so the next iteration reaches the check phase + * before polling again. */ + if (check_ran || scr_pending_immediates == 0) scr_loop_io_open = true; } return rejection_failed; } diff --git a/packages/runtime/src/scr_net.c b/packages/runtime/src/scr_net.c index f13906454..9684aebe9 100644 --- a/packages/runtime/src/scr_net.c +++ b/packages/runtime/src/scr_net.c @@ -4908,17 +4908,28 @@ static int scr_net_pollfd(void) { return scr_net_poller != NULL ? scrp_poller_fd(scr_net_poller) : -1; } +bool scr_loop_io_take(void); +void scr_loop_io_reopen(void); + static void scr_net_dispatch(void) { if (!scr_net_servers && !scr_net_socks && !scr_net_proto_pending()) { return; } + /* One poller drain per loop iteration (scr_loop_io_take); a full batch + * keeps draining, like libuv's poll phase. Deferred emits (the sweep) + * still run on every call. */ + bool may_drain = scr_loop_io_take(); for (;;) { scr_net_sweep(); if (scr_exc_pending()) return; if (scr_net_poller == NULL) return; ScrPollerEvent evs[64]; - int n = scrp_drain(scr_net_poller, evs, 64); + int n = 0; + if (may_drain) { + n = scrp_drain(scr_net_poller, evs, 64); + may_drain = n == 64; + } for (int i = 0; i < n; i++) { void *udata = evs[i].udata; if (!udata) continue; @@ -4992,8 +5003,13 @@ static void scr_net_dispatch(void) { } if (scr_exc_pending()) return; } - if (!scr_net_flags_pending()) return; - if (scr_loop_has_ready()) return; /* microtasks interleave first */ + if (!scr_net_flags_pending() && !may_drain) return; + if (scr_loop_has_ready()) { + /* microtasks interleave first; a full batch's rest drains on the + * next call, still within this poll phase */ + if (may_drain) scr_loop_io_reopen(); + return; + } } } diff --git a/tests/corpus/6251-net-connect-poll-phase-order.ts b/tests/corpus/6251-net-connect-poll-phase-order.ts new file mode 100644 index 000000000..cfe09292a --- /dev/null +++ b/tests/corpus/6251-net-connect-poll-phase-order.ts @@ -0,0 +1,70 @@ +// A connection started by a poll-phase callback completes in the NEXT loop +// iteration's poll phase: the callback's nextTick and promise jobs run +// first, then the check phase's setImmediate, then the new socket's +// 'connect' (libuv/Node order; loopback connects complete at once, so an +// event loop that polls again before the check phase reorders them). The +// second half is the same through an awaited promise, the shape of an HTTP +// client that dials a fresh connection after the previous response. +import * as net from "node:net"; + +const log: string[] = []; +const server = net.createServer((socket) => { + socket.on("data", (chunk: Buffer) => socket.write(chunk)); +}); + +function dial(port: number, label: string): Promise { + return new Promise((resolve) => { + const socket = net.connect({ port, host: "127.0.0.1" }); + socket.on("connect", () => { + log.push(`connect ${label}`); + resolve(socket); + }); + }); +} + +function roundTrip(socket: net.Socket, label: string): Promise { + return new Promise((resolve) => { + socket.once("data", () => { + log.push(`data ${label}`); + setImmediate(() => log.push(`immediate after data ${label}`)); + process.nextTick(() => log.push(`tick after data ${label}`)); + Promise.resolve().then(() => log.push(`promise after data ${label}`)); + resolve(); + }); + socket.write("ping"); + }); +} + +server.listen(0, "127.0.0.1", () => { + const address = server.address(); + const port = address !== null && typeof address === "object" ? address.port : 0; + const first = net.connect({ port, host: "127.0.0.1" }); + first.on("connect", () => { + log.push("connect first"); + first.write("ping"); + }); + first.once("data", () => { + log.push("data first"); + setImmediate(() => log.push("immediate from data")); + process.nextTick(() => log.push("tick from data")); + Promise.resolve().then(() => log.push("promise from data")); + const second = net.connect({ port, host: "127.0.0.1" }); + second.on("connect", async () => { + log.push("connect second"); + second.destroy(); + first.destroy(); + // Awaited: each response's continuation dials the next connection. + const a = await dial(port, "a"); + await roundTrip(a, "a"); + log.push("continuation after a"); + const b = await dial(port, "b"); + await roundTrip(b, "b"); + a.destroy(); + b.destroy(); + setImmediate(() => { + server.close(); + console.log(log.join("\n")); + }); + }); + }); +}); diff --git a/tests/fixtures/npm/cases/undici5/main.ts b/tests/fixtures/npm/cases/undici5/main.ts index a65601a09..b9c2eafc9 100644 --- a/tests/fixtures/npm/cases/undici5/main.ts +++ b/tests/fixtures/npm/cases/undici5/main.ts @@ -195,7 +195,6 @@ server.listen(0, "127.0.0.1", async () => { for (let i = 0; i < 4; i++) { const r = await request(`${origin}/r${i}`, { dispatcher: agent }); await r.body.text(); - await turn(); } console.log("connections opened:", agent.connections); console.log("stats:", agent.statsFor(origin)); diff --git a/tests/fixtures/npm/cases/undici7/main.ts b/tests/fixtures/npm/cases/undici7/main.ts index 24b608e61..054175e4d 100644 --- a/tests/fixtures/npm/cases/undici7/main.ts +++ b/tests/fixtures/npm/cases/undici7/main.ts @@ -201,7 +201,6 @@ server.listen(0, "127.0.0.1", async () => { for (let i = 0; i < 4; i++) { const r = await request(`${origin}/r${i}`, { dispatcher: agent }); await r.body.text(); - await turn(); } console.log("connections opened:", agent.connections); console.log("stats:", agent.statsFor(origin)); From 4e0e7fc1afbb114163ad1bd55c598f10db9d5a00 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Mon, 19 Oct 2026 19:55:18 -0700 Subject: [PATCH 05/38] Box URL into checked-dynamic unions beside fetch handles fetch's input under @types/node is string | URL | Request. A union with a checked-dynamic arm collapses to the checked-dynamic representation only when every sibling survives the collapse; URL did not qualify, so the type (and anything typed as typeof globalThis.fetch, such as a custom fetch cast through unknown) did not compile. URL's handle boxes by reference, keeps its accessors and setters, answers instanceof URL and checks back out to the typed value, so URL arms now join the collapse. A fetch() input typed string | URL also converts through the checked-dynamic input path instead of being narrowed to a string, which threw on the URL arm. Co-Authored-By: Claude Opus 5.5 --- .../src/frontend/lowering/lower-stdlib.ts | 22 +++++++-- packages/compiler/src/frontend/type-mapper.ts | 27 +++++++---- .../6252-fetch-input-url-unions.ts.json | 6 +++ .../node-types/fetch-input-unions.ts.json | 6 +++ tests/corpus/6252-fetch-input-url-unions.ts | 48 +++++++++++++++++++ .../fixtures/node-types/fetch-input-unions.ts | 48 +++++++++++++++++++ tests/harness/project-config.test.ts | 21 ++++++++ 7 files changed, 165 insertions(+), 13 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6252-fetch-input-url-unions.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/node-types/fetch-input-unions.ts.json create mode 100644 tests/corpus/6252-fetch-input-url-unions.ts create mode 100644 tests/fixtures/node-types/fetch-input-unions.ts diff --git a/packages/compiler/src/frontend/lowering/lower-stdlib.ts b/packages/compiler/src/frontend/lowering/lower-stdlib.ts index 2f0e1c66f..68a5145c3 100644 --- a/packages/compiler/src/frontend/lowering/lower-stdlib.ts +++ b/packages/compiler/src/frontend/lowering/lower-stdlib.ts @@ -619,6 +619,20 @@ export function lowerFetchCall(lowerer: Lowerer, call: ts.CallExpression): IrExp : lowerer.lowerExpr(argument); stmts.push({ kind: "exprStmt", expr: discarded, loc: discarded.loc }); } + // A union input holding a URL arm (`string | URL`) converts through the + // checked-dynamic input path, which reads the URL's href at the call + // like the bare-URL arm below; narrowing it to a string would throw on + // the URL arm. + const unionWithUrl = + input.type.kind === "union" && + (lowerer.unions.get(input.type.unionId)?.arms.some((arm) => arm.kind === "url") ?? false) && + lowerer.dynConvertible(input.type); + const dynInput: IrExpr | null = + input.type.kind === "dyn" + ? inputRef + : unionWithUrl + ? { kind: "dynFrom", value: inputRef, type: DYN, loc: locOf(inputNode) } + : null; const url: IrExpr = input.type.kind === "url" ? { @@ -628,13 +642,13 @@ export function lowerFetchCall(lowerer: Lowerer, call: ts.CallExpression): IrExp type: STRING, loc: locOf(inputNode), } - : input.type.kind === "dyn" - ? inputRef + : dynInput !== null + ? dynInput : lowerer.coerceInto(inputNode, inputRef, STRING); const answer: IrExpr = { kind: "libCall", - fn: input.type.kind === "dyn" ? "fetch.input" : "fetch.start", - args: [input.type.kind === "dyn" ? inputRef : url, initRef], + fn: dynInput !== null ? "fetch.input" : "fetch.start", + args: [url, initRef], type: { kind: "promise", inner: DYN }, loc, }; diff --git a/packages/compiler/src/frontend/type-mapper.ts b/packages/compiler/src/frontend/type-mapper.ts index 90b5d1e9f..24ff477f5 100644 --- a/packages/compiler/src/frontend/type-mapper.ts +++ b/packages/compiler/src/frontend/type-mapper.ts @@ -1360,8 +1360,9 @@ export function mapAsyncGeneratorAnnotation(type: ts.Type, ctx: TypeMapperCtx): if ( !symbol || !["AsyncIterator", "AsyncIterableIterator", "AsyncIterable"].includes(symbol.name) || - !symbol.declarations - .some((d) => ts.isInterfaceDeclaration(d) && ctx.isStdlibFile(d.getSourceFile())) + !symbol.declarations.some( + (d) => ts.isInterfaceDeclaration(d) && ctx.isStdlibFile(d.getSourceFile()), + ) ) return null; const args = checker.getTypeArguments(type as ts.TypeReference); @@ -2513,12 +2514,11 @@ function mapTypeInner(type: ts.Type, ctx: TypeMapperCtx): IrType | null { if ( psym && WEBCRYPTO_DYN_TYPES.has(psym.name) && - psym.declarations - .some( - (d) => - (ts.isClassDeclaration(d) || ts.isInterfaceDeclaration(d)) && - ctx.isStdlibFile(d.getSourceFile()), - ) + psym.declarations.some( + (d) => + (ts.isClassDeclaration(d) || ts.isInterfaceDeclaration(d)) && + ctx.isStdlibFile(d.getSourceFile()), + ) ) { return DYN; } @@ -4184,7 +4184,9 @@ function mapTypeInner(type: ts.Type, ctx: TypeMapperCtx): IrType | null { // semantics would DEGRADE under the collapse — class instances, // Maps/Sets, functions (closure identity, `x === String` // narrowing), promises, regexes, generators, handles — keep their - // existing union homes and fences below. A pending recursive + // existing union homes and fences below. URL is the + // exception: its handle boxes by reference and keeps every member, + // so it collapses too (dynSubsumableUnionArm). A pending recursive // placeholder cannot collapse (back-references already hold the // union id); the degenerate recursive spelling keeps its fence. if ( @@ -6285,6 +6287,13 @@ function dynSubsumableUnionArm(arm: IrType, ctx: TypeMapperCtx): boolean { case "undefinedT": case "nullT": return true; + // URL boxes BY REFERENCE (its checked-dynamic handle keeps identity, + // reads and writes its members through the native accessors, answers + // `instanceof URL`, and checks back out to the typed value), so it + // rides the collapse beside fetch's Request/Response handles: + // `string | URL | Request`, fetch's own input type under @types/node. + case "url": + return true; case "record": case "array": return canConvertToDyn( diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6252-fetch-input-url-unions.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6252-fetch-input-url-unions.ts.json new file mode 100644 index 000000000..13010068a --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6252-fetch-input-url-unions.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6252-fetch-input-url-unions.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/node-types/fetch-input-unions.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/node-types/fetch-input-unions.ts.json new file mode 100644 index 000000000..15aba2051 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/node-types/fetch-input-unions.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/node-types/fetch-input-unions.ts" + ], + "diags": [] +} diff --git a/tests/corpus/6252-fetch-input-url-unions.ts b/tests/corpus/6252-fetch-input-url-unions.ts new file mode 100644 index 000000000..3394fe5b4 --- /dev/null +++ b/tests/corpus/6252-fetch-input-url-unions.ts @@ -0,0 +1,48 @@ +// `typeof globalThis.fetch` under @types/node takes `string | URL | Request`: +// unions holding URL handles beside fetch handles compile (URL boxes by +// reference into the checked-dynamic representation), and a custom fetch +// cast to that type through `unknown` keeps working, including an extra +// init member written through a record view (the dispatcher pattern). +import * as http from "node:http"; +type FetchImpl = typeof globalThis.fetch; +interface DispatchingInit { + method?: string; + headers?: Record; + dispatcher?: { name: string }; +} +function customFetch(input: string | URL, init?: DispatchingInit): Promise { + console.log("custom fetch", typeof input === "string" ? "string" : "url", init?.dispatcher?.name ?? "none"); + return fetch(input, { method: init?.method, headers: init?.headers }); +} +const server = http.createServer((req, res) => res.end(`ok ${req.method} ${req.url}`)); +server.listen(0, "127.0.0.1", async () => { + const address = server.address(); + const port = address !== null && typeof address === "object" ? address.port : 0; + const impl: FetchImpl = customFetch as unknown as FetchImpl; + const init: RequestInit = { method: "POST" }; + (init as Record).dispatcher = { name: "pinned" }; + const res = await impl(`http://127.0.0.1:${port}/a`, init); + console.log(res.status, await res.text()); + const viaUrl = await impl(new URL(`http://127.0.0.1:${port}/b`)); + console.log(viaUrl.status, await viaUrl.text()); + const direct = await fetch(new URL(`http://127.0.0.1:${port}/c`)); + console.log(direct.status, await direct.text()); + server.close(); +}); + +function describeInput(input: string | Request | URL): string { + if (typeof input === "string") return `string ${input}`; + if (input instanceof URL) return `url ${input.href} ${input.hostname}`; + return `request ${input.method} ${input.url}`; +} +console.log(describeInput("http://a.test/x")); +const u = new URL("http://b.test:8080/p?q=1"); +console.log(describeInput(u)); +console.log(describeInput(new Request("http://c.test/r", { method: "PUT" }))); +const items: (string | URL | Request)[] = [u, "s"]; +console.log(items.length, items[0] === u, String(items[0])); +let slot: string | URL | Request = u; +if (slot instanceof URL) { slot.pathname = "/changed"; } +console.log(u.href); +const params: string | Request | URL = new URL("http://d.test/?b=2"); +console.log(typeof params === "string" ? params : params instanceof URL ? params.searchParams.get("b") : "req"); diff --git a/tests/fixtures/node-types/fetch-input-unions.ts b/tests/fixtures/node-types/fetch-input-unions.ts new file mode 100644 index 000000000..3394fe5b4 --- /dev/null +++ b/tests/fixtures/node-types/fetch-input-unions.ts @@ -0,0 +1,48 @@ +// `typeof globalThis.fetch` under @types/node takes `string | URL | Request`: +// unions holding URL handles beside fetch handles compile (URL boxes by +// reference into the checked-dynamic representation), and a custom fetch +// cast to that type through `unknown` keeps working, including an extra +// init member written through a record view (the dispatcher pattern). +import * as http from "node:http"; +type FetchImpl = typeof globalThis.fetch; +interface DispatchingInit { + method?: string; + headers?: Record; + dispatcher?: { name: string }; +} +function customFetch(input: string | URL, init?: DispatchingInit): Promise { + console.log("custom fetch", typeof input === "string" ? "string" : "url", init?.dispatcher?.name ?? "none"); + return fetch(input, { method: init?.method, headers: init?.headers }); +} +const server = http.createServer((req, res) => res.end(`ok ${req.method} ${req.url}`)); +server.listen(0, "127.0.0.1", async () => { + const address = server.address(); + const port = address !== null && typeof address === "object" ? address.port : 0; + const impl: FetchImpl = customFetch as unknown as FetchImpl; + const init: RequestInit = { method: "POST" }; + (init as Record).dispatcher = { name: "pinned" }; + const res = await impl(`http://127.0.0.1:${port}/a`, init); + console.log(res.status, await res.text()); + const viaUrl = await impl(new URL(`http://127.0.0.1:${port}/b`)); + console.log(viaUrl.status, await viaUrl.text()); + const direct = await fetch(new URL(`http://127.0.0.1:${port}/c`)); + console.log(direct.status, await direct.text()); + server.close(); +}); + +function describeInput(input: string | Request | URL): string { + if (typeof input === "string") return `string ${input}`; + if (input instanceof URL) return `url ${input.href} ${input.hostname}`; + return `request ${input.method} ${input.url}`; +} +console.log(describeInput("http://a.test/x")); +const u = new URL("http://b.test:8080/p?q=1"); +console.log(describeInput(u)); +console.log(describeInput(new Request("http://c.test/r", { method: "PUT" }))); +const items: (string | URL | Request)[] = [u, "s"]; +console.log(items.length, items[0] === u, String(items[0])); +let slot: string | URL | Request = u; +if (slot instanceof URL) { slot.pathname = "/changed"; } +console.log(u.href); +const params: string | Request | URL = new URL("http://d.test/?b=2"); +console.log(typeof params === "string" ? params : params instanceof URL ? params.searchParams.get("b") : "req"); diff --git a/tests/harness/project-config.test.ts b/tests/harness/project-config.test.ts index d98f12da3..226d3872a 100644 --- a/tests/harness/project-config.test.ts +++ b/tests/harness/project-config.test.ts @@ -73,6 +73,27 @@ test("node-types: Headers, Request and Response members typed by undici-types ru expect(comparableStderr(native.stderr)).toBe(node.stderr); }); +test("node-types: fetch's string | URL | Request input and a custom fetch cast to typeof fetch", async () => { + const outDir = outDirFor("fetch-input-unions"); + const entry = join(nodeTypesDir, "fetch-input-unions.ts"); + const result = await compile(entry, { + outPath: join( + outDir, + process.platform === "win32" ? "fetch-input-unions.exe" : "fetch-input-unions", + ), + outDir, + sanitize, + }); + expect(result.ok, !result.ok ? JSON.stringify(result.diagnostics, null, 2) : "").toBe(true); + if (!result.ok) return; + const [native, node] = await Promise.all([ + execFileAsync(result.binaryPath), + execFileAsync(process.execPath, ["--disable-warning=MODULE_TYPELESS_PACKAGE_JSON", entry]), + ]); + expect(native.stdout).toBe(node.stdout); + expect(comparableStderr(native.stderr)).toBe(node.stderr); +}); + test("node-types: http header arrays and server.address() follow @types/node's types", async () => { const outDir = outDirFor("http-headers-address"); const entry = join(nodeTypesDir, "http-headers-address.ts"); From 87895c340e4de2284af6f5c5fc0c38cf10194c6a Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Mon, 19 Oct 2026 22:08:08 -0500 Subject: [PATCH 06/38] Test the undici shims against a peer that sends malformed responses The undici5 and undici7 npm cases now also run peers.ts against Node with the vendored undici: header values with CR or LF, chunk-size lines that are valid, malformed, overflowing or 1 MiB long, and fetch bodies that are endless (read once, then cancelled), gzipped, truncated, empty, corrupt, cut off, aborted after the headers, redirected while endless, or carry six content codings. The undici5 case leaves out the Host values, which undici 5.12 writes into the request head, and the two fetch cases where undici 5.12 differs (six codings, and an endless redirect, which fails with a TypeError in 5.12). package-shims.test.ts compiles two programs in copies of the cases: one reads 64 MiB chunk-size lines and 64 MiB plain and gzipped fetch bodies from a separate Node peer and checks that RSS stays under 48 MiB (plain lane), the other checks that the 5.x profile rejects a Host value with CR or LF with undici 5.29.0's error. --- packages/compiler/shims/undici/core/client.ts | 217 +++++++++++- packages/compiler/shims/undici/core/fetch.ts | 327 +++++++++++++++--- .../compiler/shims/undici/core/request.ts | 8 + .../fixtures/npm/cases/undici5/main.ts.json | 1 + .../fixtures/npm/cases/undici7/main.ts.json | 1 + tests/fixtures/npm/cases/undici5/main.ts | 2 + tests/fixtures/npm/cases/undici5/peers.ts | 183 ++++++++++ tests/fixtures/npm/cases/undici7/main.ts | 2 + tests/fixtures/npm/cases/undici7/peers.ts | 197 +++++++++++ .../fixtures/package-shims/undici-bounded.ts | 29 ++ .../package-shims/undici-hostile-peer.mjs | 49 +++ tests/fixtures/package-shims/undici5-host.ts | 26 ++ tests/harness/package-shims.test.ts | 70 +++- 13 files changed, 1041 insertions(+), 71 deletions(-) create mode 100644 tests/fixtures/npm/cases/undici5/peers.ts create mode 100644 tests/fixtures/npm/cases/undici7/peers.ts create mode 100644 tests/fixtures/package-shims/undici-bounded.ts create mode 100644 tests/fixtures/package-shims/undici-hostile-peer.mjs create mode 100644 tests/fixtures/package-shims/undici5-host.ts diff --git a/packages/compiler/shims/undici/core/client.ts b/packages/compiler/shims/undici/core/client.ts index 24ebe54d0..2d0f5c361 100644 --- a/packages/compiler/shims/undici/core/client.ts +++ b/packages/compiler/shims/undici/core/client.ts @@ -782,6 +782,50 @@ function errorCode(err: Error): string { const CRLF = Buffer.from("\r\n"); +// Chunk-size line states (llhttp's chunk_size* and chunk_extension* nodes). +const SIZE_FIRST = 0; +const SIZE_DIGITS = 1; +const SIZE_AFTER = 2; +const SIZE_EXT = 3; +const SIZE_EXT_NAME = 4; +const SIZE_EXT_VALUE = 5; +const SIZE_EXT_QUOTED = 6; +const SIZE_EXT_PAIR = 7; +const SIZE_EXT_QUOTED_DONE = 8; +const SIZE_CR = 9; + +function hexDigit(c: number): number { + if (c >= 0x30 && c <= 0x39) return c - 0x30; + if (c >= 0x41 && c <= 0x46) return c - 0x37; + if (c >= 0x61 && c <= 0x66) return c - 0x57; + return -1; +} + +/** tchar: ALPHA, DIGIT and !#$%&'*+-.^_`|~. */ +const TOKEN_BYTES = tokenBytes(); + +function tokenBytes(): Uint8Array { + const table = new Uint8Array(256); + const marks = "!#$%&'*+-.^_`|~"; + for (let i = 0; i < marks.length; i++) table[marks.charCodeAt(i)] = 1; + for (let c = 0x30; c <= 0x39; c++) table[c] = 1; + for (let c = 0x41; c <= 0x5a; c++) { + table[c] = 1; + table[c + 0x20] = 1; + } + return table; +} + +/** qdtext: HTAB, SP and 0x21-0xff but '"' and '\\'. */ +function isQuotedByte(c: number): boolean { + return c === 0x09 || c === 0x20 || (c >= 0x21 && c !== 0x22 && c !== 0x5c); +} + +/** The byte after '\\' in a quoted-pair: HTAB, SP, VCHAR or obs-text. */ +function isQuotedPairByte(c: number): boolean { + return c === 0x09 || c === 0x20 || (c >= 0x21 && c !== 0x7f); +} + /** An incremental HTTP/1.1 response parser (the subset of llhttp undici * relies on): status line, raw header pairs, Content-Length, chunked and * read-until-close bodies, trailers, and pause/resume back-pressure. */ @@ -805,6 +849,9 @@ class ResponseParser { bytesRead: number; trailers: Buffer[]; skipBody: boolean; + sizeState: number; + sizeDigits: number; + chunkSize: number; constructor(client: Client, socket: Socket) { this.client = client; @@ -826,6 +873,9 @@ class ResponseParser { this.bytesRead = 0; this.trailers = []; this.skipBody = false; + this.sizeState = SIZE_FIRST; + this.sizeDigits = 0; + this.chunkSize = 0; } reset(): void { @@ -868,11 +918,154 @@ class ResponseParser { } } - fail(message: string, code: string): void { - this.client.__onParseError(this.socket, new HTTPParserError(message, code, "")); + /** A parse error with llhttp's reason: undici 7 puts it inside its + * protocol message and, as its error table maps names to numbers only, + * sets no code; undici 5 uses the reason as the message and the code. */ + fail(reason: string, code: string): void { + const err = + this.client.__profile.major >= 7 + ? new HTTPParserError(`Response does not match the HTTP/1.1 protocol (${reason})`) + : new HTTPParserError(reason, code, ""); + this.client.__onParseError(this.socket, err); this.state = 99; } + /** Reads the chunk-size line one byte at a time as llhttp does, so a line + * that never ends holds no memory: hex digits (past 16 significant ones + * the size overflows), then extensions, checked and dropped, then CRLF. + * The 7.x profile follows llhttp 9 (undici 7), the 5.x profile llhttp 8 + * in its lenient mode (undici 5). Returns true when the line is complete + * and chunkSize holds its size; false when it needs more bytes or failed. */ + chunkSizeLine(): boolean { + const buf = this.buf; + if (this.sizeState === SIZE_FIRST) { + // The common line, hex digits then CRLF in this buffer, in one pass; + // any other line takes the byte loop below from its start. + let size = 0; + let digits = 0; + let j = 0; + for (; j < buf.length && j < 32; j++) { + const d = hexDigit(buf[j]!); + if (d === -1) break; + if (digits > 0 || d !== 0) { + digits++; + size = size * 16 + d; + } + } + if (j > 0 && digits <= 16 && j + 1 < buf.length && buf[j] === 0x0d && buf[j + 1] === 0x0a) { + this.chunkSize = size; + this.buf = buf.subarray(j + 2); + return true; + } + } + const v7 = this.client.__profile.major >= 7; + let i = 0; + while (i < buf.length) { + const c = buf[i]!; + const state = this.sizeState; + if (state === SIZE_FIRST || state === SIZE_DIGITS) { + const d = hexDigit(c); + if (d === -1) { + if (state === SIZE_FIRST) { + this.fail("Invalid character in chunk size", "INVALID_CHUNK_SIZE"); + return false; + } + this.sizeState = SIZE_AFTER; + continue; + } + if (state === SIZE_FIRST) { + this.chunkSize = 0; + this.sizeDigits = 0; + this.sizeState = SIZE_DIGITS; + } + if (this.sizeDigits > 0 || d !== 0) { + if (this.sizeDigits === 16) { + this.fail("Chunk size overflow", "INVALID_CHUNK_SIZE"); + return false; + } + this.sizeDigits++; + this.chunkSize = this.chunkSize * 16 + d; + } + } else if (state === SIZE_AFTER) { + if (c === 0x0d) this.sizeState = SIZE_CR; + else if (c === 0x3b) this.sizeState = SIZE_EXT; + else if (c === 0x0a && v7) { + this.fail("Missing expected CR after chunk size", "CR_EXPECTED"); + return false; + } else { + this.fail("Invalid character in chunk size", "INVALID_CHUNK_SIZE"); + return false; + } + } else if (state === SIZE_EXT) { + if (c === 0x20 || c === 0x0d) { + this.fail("Invalid character in chunk extensions", "STRICT"); + return false; + } + this.sizeState = SIZE_EXT_NAME; + continue; + } else if (state === SIZE_EXT_NAME) { + if (c === 0x3d) this.sizeState = SIZE_EXT_VALUE; + else if (c === 0x3b) this.sizeState = SIZE_EXT; + else if (c === 0x0d) this.sizeState = SIZE_CR; + else if (c === 0x0a && v7) { + this.fail("Missing expected CR after chunk extension name", "CR_EXPECTED"); + return false; + } else if (TOKEN_BYTES[c] !== 1) { + this.fail("Invalid character in chunk extensions name", "STRICT"); + return false; + } + } else if (state === SIZE_EXT_VALUE) { + if (c === 0x22) this.sizeState = SIZE_EXT_QUOTED; + // llhttp 8 goes on with the chunk-size node after a value's ';'. + else if (c === 0x3b) this.sizeState = v7 ? SIZE_EXT : SIZE_AFTER; + else if (c === 0x0d) this.sizeState = SIZE_CR; + else if (c === 0x0a && v7) { + this.fail("Missing expected CR after chunk extension value", "CR_EXPECTED"); + return false; + } else if (TOKEN_BYTES[c] !== 1) { + this.fail("Invalid character in chunk extensions value", "STRICT"); + return false; + } + } else if (state === SIZE_EXT_QUOTED) { + if (c === 0x22) this.sizeState = SIZE_EXT_QUOTED_DONE; + else if (c === 0x5c && v7) this.sizeState = SIZE_EXT_PAIR; + else if (!isQuotedByte(c)) { + this.fail("Invalid character in chunk extensions quoted value", "STRICT"); + return false; + } + } else if (state === SIZE_EXT_PAIR) { + if (!isQuotedPairByte(c)) { + this.fail("Invalid quoted-pair in chunk extensions quoted value", "STRICT"); + return false; + } + this.sizeState = SIZE_EXT_QUOTED; + } else if (state === SIZE_EXT_QUOTED_DONE) { + if (c === 0x3b) this.sizeState = SIZE_EXT; + else if (c === 0x0d) this.sizeState = SIZE_CR; + else if (c === 0x0a && v7) { + this.fail("Missing expected CR after chunk extension value", "CR_EXPECTED"); + return false; + } else { + this.fail("Invalid character in chunk extensions quote value", "STRICT"); + return false; + } + } else { + // SIZE_CR: llhttp 9 needs the LF; llhttp 8's lenient mode skips + // whatever byte follows the CR. + if (c !== 0x0a && v7) { + this.fail("Expected LF after chunk size", "STRICT"); + return false; + } + this.buf = buf.subarray(i + 1); + this.sizeState = SIZE_FIRST; + return true; + } + i++; + } + this.buf = Buffer.alloc(0); + return false; + } + line(): Buffer | null { const idx = this.buf.indexOf(CRLF); if (idx === -1) return null; @@ -897,7 +1090,12 @@ class ResponseParser { const text = line.toString("latin1"); const m = /^HTTP\/1\.([01]) (\d{3})(?: (.*))?$/.exec(text); if (m === null) { - this.fail("Response does not match the HTTP/1.1 protocol", "INVALID_CONSTANT"); + const v7 = this.client.__profile.major >= 7; + if (!text.startsWith("HTTP/")) { + this.fail(v7 ? "Expected HTTP/, RTSP/ or ICE/" : "Expected HTTP/", "INVALID_CONSTANT"); + } else { + this.fail(v7 ? "Invalid status code" : "Invalid response status", "INVALID_STATUS"); + } return; } this.versionMinor = Number(m[1]); @@ -1007,18 +1205,11 @@ class ResponseParser { } } else if (this.state === 3) { // Chunk size line. - const line = this.line(); - if (line === null) return; - const sizeText = line.toString("latin1").split(";")[0]!.trim(); - if (!/^[0-9a-fA-F]+$/.test(sizeText)) { - this.fail("Invalid character in chunk size", "INVALID_CHUNK_SIZE"); - return; - } - const size = parseInt(sizeText, 16); - if (size === 0) { + if (!this.chunkSizeLine()) return; + if (this.chunkSize === 0) { this.state = 5; } else { - this.remaining = size; + this.remaining = this.chunkSize; this.state = 4; } } else if (this.state === 4) { diff --git a/packages/compiler/shims/undici/core/fetch.ts b/packages/compiler/shims/undici/core/fetch.ts index 9b287a7bd..54a0ce0d5 100644 --- a/packages/compiler/shims/undici/core/fetch.ts +++ b/packages/compiler/shims/undici/core/fetch.ts @@ -2,14 +2,16 @@ // program-installed global dispatcher) it is the runtime's own fetch, which // implements the same WHATWG algorithm. With one, the request goes through // that dispatcher (its connector, lookup and pool), following undici's -// fetch for request headers, redirects and errors; the response body is -// buffered into a native Response. +// fetch for request headers, redirects and errors. The promise resolves +// when the response headers arrive, and the Response body streams from the +// connection with undici's back-pressure (see dispatchOnce). // // Known gaps: a Response built here reports `url` as "" and `redirected` as // false (the runtime's Response has no settable URL yet), and https requests // offer `gzip, deflate` rather than undici's `br, gzip, deflate` (the runtime // has no Brotli decoder). -import { gunzipSync, inflateSync } from "node:zlib"; +import type { Transform } from "node:stream"; +import * as zlib from "node:zlib"; import type { Dispatcher } from "./dispatcher.ts"; import { getGlobalDispatcher, hasCustomGlobalDispatcher } from "./api.ts"; import type { DispatchHandlers, Profile } from "./types.ts"; @@ -115,7 +117,9 @@ interface RawResponse { status: number; statusText: string; headers: string[]; - body: Buffer; + /** null when the response has no body to expose (null-body status, HEAD, + * or a redirect fetch follows); its bytes are then read and dropped. */ + body: ReadableStream | null; } const redirectStatuses = [301, 302, 303, 307, 308]; @@ -125,7 +129,7 @@ export function undiciFetch( input: string | URL, init: UndiciRequestInit | undefined, createGlobal: () => Dispatcher, - _profile: Profile, + profile: Profile, ): Promise { const explicit = init !== undefined ? init.dispatcher : undefined; if (explicit === undefined && !hasCustomGlobalDispatcher()) { @@ -139,7 +143,7 @@ export function undiciFetch( }); } const dispatcher = explicit ?? getGlobalDispatcher(createGlobal); - return fetchThrough(dispatcher, input, init ?? {}); + return fetchThrough(dispatcher, input, init ?? {}, profile); } function normalizeMethod(method: string): string { @@ -153,6 +157,7 @@ async function fetchThrough( dispatcher: Dispatcher, input: string | URL, init: UndiciRequestInit, + profile: Profile, ): Promise { let url = typeof input === "string" ? new URL(input) : new URL(input.href); let method = normalizeMethod(init.method ?? "GET"); @@ -197,7 +202,7 @@ async function fetchThrough( let raw: RawResponse; try { - raw = await dispatchOnce(dispatcher, url, method, pairs, body, signal); + raw = await dispatchOnce(dispatcher, url, method, pairs, body, signal, redirectMode, profile); } catch (err) { if (signal !== null && signal.aborted) throw abortReason(signal); throw new TypeError("fetch failed", { cause: err }); @@ -243,15 +248,11 @@ async function fetchThrough( continue; } - const nullBody = nullBodyStatuses.includes(raw.status) || method === "HEAD"; - return new Response( - nullBody ? null : decode(raw.body, responseHeaders.get("content-encoding")), - { - status: raw.status, - statusText: raw.statusText, - headers: responseHeaders, - }, - ); + return new Response(raw.body, { + status: raw.status, + statusText: raw.statusText, + headers: responseHeaders, + }); } } @@ -324,20 +325,28 @@ function headerListOf( return list; } -function decode(body: Buffer, encoding: string | null): Buffer { - if (encoding === null || body.length === 0) return body; - let out = body; - const codings = encoding - .toLowerCase() - .split(",") - .map((c) => c.trim()) - .reverse(); +/** undici 7's limit on a response's content codings, applied in both + * profiles: undici 5 has none, and builds one decoder per listed coding. */ +const maxContentEncodings = 5; + +/** Whether every coding has a decoder; otherwise the body passes through + * as sent. */ +function decodable(codings: string[]): boolean { + if (codings.length === 0) return false; for (const coding of codings) { - if (coding === "gzip" || coding === "x-gzip") out = gunzipSync(out); - else if (coding === "deflate") out = inflateSync(out); - else return body; + if (coding !== "gzip" && coding !== "x-gzip" && coding !== "deflate") return false; } - return out; + return true; +} + +/** A decoder for one coding. undici 7 decodes with Z_SYNC_FLUSH, which + * accepts a truncated stream; undici 5 uses zlib's defaults. */ +function decoderFor(coding: string, profile: Profile): Transform { + const options = + profile.major >= 7 + ? { flush: zlib.constants.Z_SYNC_FLUSH, finishFlush: zlib.constants.Z_SYNC_FLUSH } + : {}; + return coding === "deflate" ? zlib.createInflate(options) : zlib.createGunzip(options); } function abortReason(signal: AbortSignal): Error { @@ -347,6 +356,16 @@ function abortReason(signal: AbortSignal): Error { : new DOMException("This operation was aborted", "AbortError"); } +/** Unread bytes held beside the response stream before the connection + * pauses: Node's Readable highWaterMark, which undici's fetch body uses. */ +const bodyHighWaterMark = 64 * 1024; + +/** Dispatches one request and resolves when its response headers arrive. + * The body streams as undici's fetch reads one: up to bodyHighWaterMark + * unread bytes wait beside the stream, past that the connection (or the + * last decoder) pauses until the reader pulls again; cancelling the stream + * aborts the request, and an error after the headers errors the stream + * with TypeError('terminated'). */ function dispatchOnce( dispatcher: Dispatcher, url: URL, @@ -354,50 +373,248 @@ function dispatchOnce( headers: string[], body: Buffer | null, signal: AbortSignal | null, + redirectMode: "follow" | "manual" | "error", + profile: Profile, ): Promise { return new Promise((resolve, reject) => { - let status = 0; - let statusText = ""; - let rawHeaders: string[] = []; - const chunks: Buffer[] = []; - let settled = false; + let resolved = false; + let done = false; + let dump = false; let abortFn: ((err?: Error) => void) | null = null; - const onAbort = (): void => { + + // The response body: its stream controller, the unread bytes beside it, + // and how to restart the paused producer. + let controller: ReadableStreamDefaultController | null = null; + let hasBody = false; + const queue: Buffer[] = []; + let queued = 0; + let ended = false; + let closed = false; + let onDrain: (() => void) | null = null; + // The decoders, first-applied last; their listeners hold this scope, so + // releaseDecoders drops them once the body ends, fails or is cancelled. + let decoding = false; + let decoders: Transform[] = []; + let first: Transform | null = null; + let encodedBytes = 0; + let resumeConnection: (() => void) | null = null; + + const abortRequest = (err: Error): void => { const abort = abortFn; - if (abort !== null && signal !== null) abort(abortReason(signal)); + if (abort !== null && !done) abort(err); + }; + const onAbort = (): void => { + if (signal === null) return; + const reason = abortReason(signal); + // After the headers undici 7 errors the body with the signal's + // reason; undici 5 aborts its fetch controller without one, so the + // body gets the controller's default AbortError. + errorBody( + profile.major >= 7 ? reason : new DOMException("The operation was aborted.", "AbortError"), + ); + abortRequest(reason); + }; + const stopListening = (): void => { + if (signal !== null) signal.removeEventListener("abort", onAbort); }; if (signal !== null) signal.addEventListener("abort", onAbort); - const finish = (err: Error | null): void => { - if (settled) return; - settled = true; - if (signal !== null) signal.removeEventListener("abort", onAbort); - if (err !== null) reject(err); - else resolve({ status, statusText, headers: rawHeaders, body: Buffer.concat(chunks) }); + + const releaseDecoders = (): void => { + const chain = decoders; + decoders = []; + first = null; + for (const decoder of chain) { + decoder.removeAllListeners(); + decoder.destroy(); + } }; + const closeBody = (): void => { + closed = true; + controller!.close(); + stopListening(); + }; + const errorBody = (err: unknown): void => { + if (!hasBody || closed) return; + closed = true; + queue.length = 0; + queued = 0; + controller!.error(err); + stopListening(); + releaseDecoders(); + }; + const endBody = (): void => { + if (closed) return; + ended = true; + if (queue.length === 0) closeBody(); + }; + /** Takes a chunk; false when the producer should pause until onDrain. */ + const pushBody = (chunk: Buffer): boolean => { + if (closed) return true; + if (queue.length === 0 && (controller!.desiredSize ?? 0) > 0) { + controller!.enqueue(new Uint8Array(chunk)); + return true; + } + queue.push(chunk); + queued += chunk.length; + return queued < bodyHighWaterMark; + }; + const pullBody = (): void => { + if (closed) return; + const next = queue.shift(); + if (next !== undefined) { + queued -= next.length; + controller!.enqueue(new Uint8Array(next)); + } + if (queue.length === 0 && ended) { + closeBody(); + return; + } + const drain = onDrain; + if (drain !== null && queued < bodyHighWaterMark) { + onDrain = null; + drain(); + } + }; + const cancelBody = (reason: unknown): void => { + if (closed) return; + closed = true; + queue.length = 0; + queued = 0; + stopListening(); + releaseDecoders(); + abortRequest( + reason instanceof Error + ? reason + : new DOMException("The operation was aborted.", "AbortError"), + ); + }; + const failBody = (err: Error): void => { + errorBody(new TypeError("terminated", { cause: err })); + abortRequest(err); + }; + const handler: DispatchHandlers = { onConnect: (abort: (err?: Error) => void): void => { abortFn = abort; if (signal !== null && signal.aborted) abort(abortReason(signal)); }, - onHeaders: ( - statusCode: number, - raw: Buffer[], - _resume: () => void, - text: string, - ): boolean => { + onHeaders: (statusCode: number, raw: Buffer[], resume: () => void, text: string): boolean => { if (statusCode < 200) return true; - status = statusCode; - statusText = text; - rawHeaders = []; - for (const b of raw) rawHeaders.push(b.toString("latin1")); + resolved = true; + resumeConnection = resume; + const rawHeaders: string[] = []; + let location = false; + let encoding: string | null = null; + for (let i = 0; i + 1 < raw.length; i += 2) { + const name = raw[i]!.toString("latin1"); + const value = raw[i + 1]!.toString("latin1"); + rawHeaders.push(name, value); + const lower = name.toLowerCase(); + if (lower === "location") location = true; + else if (lower === "content-encoding") + encoding = encoding === null ? value : `${encoding}, ${value}`; + } + const follows = + redirectStatuses.includes(statusCode) && location && redirectMode !== "manual"; + if (follows || nullBodyStatuses.includes(statusCode) || method === "HEAD") { + dump = true; + stopListening(); + resolve({ status: statusCode, statusText: text, headers: rawHeaders, body: null }); + return true; + } + const codings = + encoding === null + ? [] + : encoding + .toLowerCase() + .split(",") + .map((c) => c.trim()); + if (codings.length > maxContentEncodings) { + dump = true; + stopListening(); + reject( + new Error( + `too many content-encodings in response: ${codings.length}, maximum allowed is ${maxContentEncodings}`, + ), + ); + abortRequest(new DOMException("The operation was aborted.", "AbortError")); + return true; + } + hasBody = true; + const stream = new ReadableStream({ + start: (c: ReadableStreamDefaultController): void => { + controller = c; + }, + pull: (): void => pullBody(), + cancel: (reason: unknown): void => cancelBody(reason), + }); + if (decodable(codings)) { + // Decoders run last-applied coding first, piped into each other; + // the last one feeds the body. + decoding = true; + let last: Transform | null = null; + for (let i = codings.length - 1; i >= 0; i--) { + const decoder = decoderFor(codings[i]!, profile); + decoder.on("error", (err: Error) => failBody(err)); + if (last === null) first = decoder; + else last.pipe(decoder); + decoders.push(decoder); + last = decoder; + } + const output = last!; + output.on("data", (chunk: Buffer) => { + if (!pushBody(chunk)) { + output.pause(); + onDrain = (): void => { + output.resume(); + }; + } + }); + output.on("end", () => { + releaseDecoders(); + endBody(); + }); + first!.on("drain", () => resume()); + } + resolve({ status: statusCode, statusText: text, headers: rawHeaders, body: stream }); return true; }, onData: (chunk: Buffer): boolean => { - chunks.push(chunk); - return true; + if (dump || !hasBody || closed) return true; + if (decoding) { + const decoder = first; + if (decoder === null) return true; + encodedBytes += chunk.length; + return decoder.write(chunk); + } + if (pushBody(chunk)) return true; + onDrain = resumeConnection; + return false; + }, + onComplete: (_trailers: string[] | null): void => { + done = true; + if (!hasBody) { + stopListening(); + return; + } + // zlib rejects an empty stream; undici ends such a body unread. + const decoder = first; + if (decoder !== null && encodedBytes > 0) { + decoder.end(); + } else { + releaseDecoders(); + endBody(); + } + }, + onError: (err: Error): void => { + done = true; + if (!resolved) { + stopListening(); + reject(err); + return; + } + errorBody(new TypeError("terminated", { cause: err })); }, - onComplete: (_trailers: string[] | null): void => finish(null), - onError: (err: Error): void => finish(err), }; dispatcher.dispatch( { diff --git a/packages/compiler/shims/undici/core/request.ts b/packages/compiler/shims/undici/core/request.ts index a876e42f0..6e0f68022 100644 --- a/packages/compiler/shims/undici/core/request.ts +++ b/packages/compiler/shims/undici/core/request.ts @@ -227,8 +227,16 @@ export class Request { function processHeader(request: Request, key: string, val: string, profile: Profile): void { const lower = key.toLowerCase(); + // undici 7 checks the name and every value before the special cases; + // undici 5 checks the first Host value before storing it. Either way a + // CR or LF in a value never reaches the request head. + if (profile.major >= 7) { + if (!tokenRegExp.test(lower)) throw new InvalidArgumentError("invalid header key"); + if (headerCharRegex.test(val)) throw new InvalidArgumentError(`invalid ${key} header`); + } if (lower === "host") { if (request.host === null) { + if (headerCharRegex.test(val)) throw new InvalidArgumentError(`invalid ${key} header`); request.host = val; return; } diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/undici5/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/undici5/main.ts.json index 80e48f7f1..dec84be6a 100644 --- a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/undici5/main.ts.json +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/undici5/main.ts.json @@ -15,6 +15,7 @@ "/packages/compiler/shims/undici/core/fetch.ts", "/packages/compiler/shims/undici/core/errors-v5.ts", "/packages/compiler/shims/undici/v5.ts", + "/tests/fixtures/npm/cases/undici5/peers.ts", "/tests/fixtures/npm/cases/undici5/main.ts" ], "diags": [] diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/undici7/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/undici7/main.ts.json index d4b65f4ae..75f724bde 100644 --- a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/undici7/main.ts.json +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/undici7/main.ts.json @@ -15,6 +15,7 @@ "/packages/compiler/shims/undici/core/fetch.ts", "/packages/compiler/shims/undici/core/errors-v7.ts", "/packages/compiler/shims/undici/v7.ts", + "/tests/fixtures/npm/cases/undici7/peers.ts", "/tests/fixtures/npm/cases/undici7/main.ts" ], "diags": [] diff --git a/tests/fixtures/npm/cases/undici5/main.ts b/tests/fixtures/npm/cases/undici5/main.ts index a65601a09..37f653495 100644 --- a/tests/fixtures/npm/cases/undici5/main.ts +++ b/tests/fixtures/npm/cases/undici5/main.ts @@ -18,6 +18,7 @@ import { setGlobalDispatcher, } from "undici"; import type { Dispatcher } from "undici"; +import { checkPeers } from "./peers.ts"; const server = http.createServer((req, res) => { const url = req.url ?? "/"; @@ -294,5 +295,6 @@ server.listen(0, "127.0.0.1", async () => { await agent.close(); server.close(); + await checkPeers(); console.log("done"); }); diff --git a/tests/fixtures/npm/cases/undici5/peers.ts b/tests/fixtures/npm/cases/undici5/peers.ts new file mode 100644 index 000000000..300fa6b35 --- /dev/null +++ b/tests/fixtures/npm/cases/undici5/peers.ts @@ -0,0 +1,183 @@ +// Requests against a raw TCP peer that sends malformed or endless +// responses: header values with CR or LF never reach the wire, a +// chunk-size line is read as llhttp reads it (so an endless one holds no +// memory), and fetch() resolves at the headers with a body that streams, +// cancels and fails as undici's does. +import * as net from "node:net"; +import type { Socket } from "node:net"; +import { gzipSync } from "node:zlib"; +import { Agent, errors, fetch, request } from "undici"; + +function describe(err: unknown): string { + // HTTPParserError extends Error, not UndiciError. + if (err instanceof errors.HTTPParserError) return `${err.name} ${err.code ?? ""} ${err.message}`; + if (err instanceof errors.UndiciError) return `${err.name} ${err.code} ${err.message}`; + if (err instanceof DOMException) return `${err.name} ${err.message}`; + if (err instanceof Error) { + const code = (err as NodeJS.ErrnoException).code ?? ""; + const cause = err.cause !== undefined ? ` cause: ${describe(err.cause)}` : ""; + return `${err.name} ${code} ${err.message}${cause}`; + } + return String(err); +} + +const MiB = 1024 * 1024; +const text = "hello ".repeat(2000); +const packed = gzipSync(text); + +// Chunked bodies whose chunk-size lines the client reads. Each is a +// complete response, or fails at the size line. +const chunkCases: [string, string][] = [ + ["plain", "5\r\nhello\r\n0\r\n\r\n"], + ["extensions", '5;a=b;c="d e";f\r\nhello\r\n0\r\n\r\n'], + ["quoted pair", '5;a="b\\"c"\r\nhello\r\n0\r\n\r\n'], + ["value then extension", "5;a=b;c=d\r\nhello\r\n0\r\n\r\n"], + ["space in name", "5;a b\r\nhello\r\n0\r\n\r\n"], + ["empty extension", "5;\r\nhello\r\n0\r\n\r\n"], + ["bad value", "5;a=b c\r\nhello\r\n0\r\n\r\n"], + ["bad quoted value", '5;a="b\x01"\r\nhello\r\n0\r\n\r\n'], + ["after quoted value", '5;a="b"c\r\nhello\r\n0\r\n\r\n'], + ["LF after name", "5;a\nhello\r\n0\r\n\r\n"], + ["LF after value", "5;a=b\nhello\r\n0\r\n\r\n"], + ["space after size", "5 \r\nhello\r\n0\r\n\r\n"], + ["space before size", " 5\r\nhello\r\n0\r\n\r\n"], + ["empty size", "\r\nhello\r\n0\r\n\r\n"], + ["not hex", "g\r\nhello\r\n0\r\n\r\n"], + ["LF only", "5\nhello\r\n0\r\n\r\n"], + ["CR then other byte", "5\rXhello\r\n0\r\n\r\n"], + ["16 hex digits", `${"0".repeat(11)}00005\r\nhello\r\n0\r\n\r\n`], + ["17 significant digits", "10000000000000000\r\nhello\r\n0\r\n\r\n"], + ["1 MiB of leading zeros", `${"0".repeat(MiB)}5\r\nhello\r\n0\r\n\r\n`], + ["1 MiB extension", `5;a=${"b".repeat(MiB)}\r\nhello\r\n0\r\n\r\n`], + ["1 MiB of digits", `1${"0".repeat(MiB)}\r\nhello\r\n0\r\n\r\n`], + ["1 MiB of spaces", `5${" ".repeat(MiB)}\r\nhello\r\n0\r\n\r\n`], +]; + +let chunkBody = ""; +let closes = 0; + +const peer = net.createServer((s: Socket) => { + s.on("error", () => {}); + s.on("close", () => { + closes++; + }); + let head = ""; + s.on("data", (d: Buffer) => { + head += d.toString("latin1"); + if (!head.includes("\r\n\r\n")) return; + const lines = head.split("\r\n"); + const path = lines[0]!.split(" ")[1]!; + head = ""; + const send = (status: string, fields: string, body: string): void => { + s.write(`HTTP/1.1 ${status}\r\n${fields}\r\n`); + s.write(body); + }; + if (path === "/echo") { + const received = lines.slice(1).filter((l) => l !== ""); + send("200 OK", `Content-Length: ${JSON.stringify(received).length}\r\n`, JSON.stringify(received)); + } else if (path === "/chunked") { + send("200 OK", "Transfer-Encoding: chunked\r\n", chunkBody); + } else if (path === "/endless") { + s.write("HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n"); + const timer = setInterval(() => s.write("4\r\ndata\r\n"), 5); + s.on("close", () => clearInterval(timer)); + } else if (path === "/gzip") { + s.write(`HTTP/1.1 200 OK\r\nContent-Encoding: gzip\r\nContent-Length: ${packed.length}\r\n\r\n`); + for (let i = 0; i < packed.length; i += 50) s.write(packed.subarray(i, i + 50)); + } else if (path === "/gzip-truncated") { + const part = packed.subarray(0, packed.length - 12); + s.write(`HTTP/1.1 200 OK\r\nContent-Encoding: gzip\r\nContent-Length: ${part.length}\r\n\r\n`); + s.write(part); + } else if (path === "/gzip-empty") { + send("200 OK", "Content-Encoding: gzip\r\nContent-Length: 0\r\n", ""); + } else if (path === "/gzip-bad") { + send("200 OK", "Content-Encoding: gzip\r\nContent-Length: 8\r\n", "not gzip"); + } else if (path === "/cut") { + s.write("HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\n0123456789"); + setTimeout(() => s.destroy(), 20); + } else if (path === "/no-content") { + send("204 No Content", "", ""); + } else { + send("200 OK", "Content-Length: 5\r\n", "plain"); + } + }); +}); + +export async function checkPeers(): Promise { + await new Promise((resolve) => peer.listen(0, "127.0.0.1", () => resolve())); + const origin = `http://127.0.0.1:${(peer.address() as net.AddressInfo).port}`; + const dispatcher = new Agent(); + + console.log("== peers: header values"); + // undici 5.12 writes a Host value with CR or LF into the request head; + // the built-in implementation rejects it as undici 5.29 does + // (package-shims.test.ts covers that). + for (const [name, value] of [["x-test", "a\r\nX-Injected: 1"]]) { + try { + const res = await request(`${origin}/echo`, { dispatcher, headers: { [name!]: value! } }); + console.log(JSON.stringify(value), "sent:", await res.body.text()); + } catch (err) { + console.log(JSON.stringify(value), describe(err)); + } + } + + console.log("== peers: chunk-size lines"); + for (const [label, body] of chunkCases) { + chunkBody = body; + try { + const res = await request(`${origin}/chunked`, { dispatcher }); + console.log(label, res.statusCode, JSON.stringify(await res.body.text())); + } catch (err) { + console.log(label, describe(err)); + } + } + + console.log("== peers: fetch bodies"); + const tryFetch = async ( + label: string, + path: string, + read: (res: Response) => Promise, + method = "GET", + signal: AbortSignal | null = null, + ): Promise => { + try { + const res = await fetch(`${origin}${path}`, { dispatcher, method, signal }); + console.log(label, res.status, res.body === null ? "null body" : "body"); + await read(res); + } catch (err) { + console.log(label, describe(err)); + } + }; + await tryFetch("endless", "/endless", async (res) => { + const reader = res.body!.getReader(); + const first = await reader.read(); + console.log("first chunk:", first.done, new TextDecoder().decode(first.value!).slice(0, 4)); + const before = closes; + await reader.cancel(); + await new Promise((resolve) => setTimeout(resolve, 100)); + console.log("cancel closed the connection:", closes > before); + }); + await tryFetch("gzip", "/gzip", async (res) => console.log("text matches:", (await res.text()) === text)); + await tryFetch("gzip-truncated", "/gzip-truncated", async (res) => + console.log("length", (await res.text()).length), + ); + await tryFetch("gzip-empty", "/gzip-empty", async (res) => console.log(JSON.stringify(await res.text()))); + await tryFetch("gzip-bad", "/gzip-bad", async (res) => console.log(JSON.stringify(await res.text()))); + await tryFetch("cut", "/cut", async (res) => console.log(JSON.stringify(await res.text()))); + const controller = new AbortController(); + await tryFetch( + "abort after headers", + "/endless", + async (res) => { + controller.abort(); + console.log(await res.text()); + }, + "GET", + controller.signal, + ); + await tryFetch("no-content", "/no-content", async (res) => console.log(JSON.stringify(await res.text()))); + await tryFetch("head", "/plain", async (res) => console.log(JSON.stringify(await res.text())), "HEAD"); + + await dispatcher.close(); + peer.close(); +} diff --git a/tests/fixtures/npm/cases/undici7/main.ts b/tests/fixtures/npm/cases/undici7/main.ts index 24b608e61..8ded547fb 100644 --- a/tests/fixtures/npm/cases/undici7/main.ts +++ b/tests/fixtures/npm/cases/undici7/main.ts @@ -20,6 +20,7 @@ import { setGlobalDispatcher, } from "undici"; import type { Dispatcher } from "undici"; +import { checkPeers } from "./peers.ts"; const server = http.createServer((req, res) => { const url = req.url ?? "/"; @@ -357,5 +358,6 @@ server.listen(0, "127.0.0.1", async () => { await agent.close(); server.close(); + await checkPeers(); console.log("done"); }); diff --git a/tests/fixtures/npm/cases/undici7/peers.ts b/tests/fixtures/npm/cases/undici7/peers.ts new file mode 100644 index 000000000..3c2c54fbc --- /dev/null +++ b/tests/fixtures/npm/cases/undici7/peers.ts @@ -0,0 +1,197 @@ +// Requests against a raw TCP peer that sends malformed or endless +// responses: header values with CR or LF never reach the wire, a +// chunk-size line is read as llhttp reads it (so an endless one holds no +// memory), and fetch() resolves at the headers with a body that streams, +// cancels and fails as undici's does. +import * as net from "node:net"; +import type { Socket } from "node:net"; +import { gzipSync } from "node:zlib"; +import { Agent, errors, fetch, request } from "undici"; + +function describe(err: unknown): string { + // HTTPParserError extends Error, not UndiciError. + if (err instanceof errors.HTTPParserError) return `${err.name} ${err.code ?? ""} ${err.message}`; + if (err instanceof errors.UndiciError) return `${err.name} ${err.code} ${err.message}`; + if (err instanceof DOMException) return `${err.name} ${err.message}`; + if (err instanceof Error) { + const code = (err as NodeJS.ErrnoException).code ?? ""; + const cause = err.cause !== undefined ? ` cause: ${describe(err.cause)}` : ""; + return `${err.name} ${code} ${err.message}${cause}`; + } + return String(err); +} + +const MiB = 1024 * 1024; +const text = "hello ".repeat(2000); +const packed = gzipSync(text); + +// Chunked bodies whose chunk-size lines the client reads. Each is a +// complete response, or fails at the size line. +const chunkCases: [string, string][] = [ + ["plain", "5\r\nhello\r\n0\r\n\r\n"], + ["extensions", '5;a=b;c="d e";f\r\nhello\r\n0\r\n\r\n'], + ["quoted pair", '5;a="b\\"c"\r\nhello\r\n0\r\n\r\n'], + ["value then extension", "5;a=b;c=d\r\nhello\r\n0\r\n\r\n"], + ["space in name", "5;a b\r\nhello\r\n0\r\n\r\n"], + ["empty extension", "5;\r\nhello\r\n0\r\n\r\n"], + ["bad value", "5;a=b c\r\nhello\r\n0\r\n\r\n"], + ["bad quoted value", '5;a="b\x01"\r\nhello\r\n0\r\n\r\n'], + ["after quoted value", '5;a="b"c\r\nhello\r\n0\r\n\r\n'], + ["LF after name", "5;a\nhello\r\n0\r\n\r\n"], + ["LF after value", "5;a=b\nhello\r\n0\r\n\r\n"], + ["space after size", "5 \r\nhello\r\n0\r\n\r\n"], + ["space before size", " 5\r\nhello\r\n0\r\n\r\n"], + ["empty size", "\r\nhello\r\n0\r\n\r\n"], + ["not hex", "g\r\nhello\r\n0\r\n\r\n"], + ["LF only", "5\nhello\r\n0\r\n\r\n"], + ["CR then other byte", "5\rXhello\r\n0\r\n\r\n"], + ["16 hex digits", `${"0".repeat(11)}00005\r\nhello\r\n0\r\n\r\n`], + ["17 significant digits", "10000000000000000\r\nhello\r\n0\r\n\r\n"], + ["1 MiB of leading zeros", `${"0".repeat(MiB)}5\r\nhello\r\n0\r\n\r\n`], + ["1 MiB extension", `5;a=${"b".repeat(MiB)}\r\nhello\r\n0\r\n\r\n`], + ["1 MiB of digits", `1${"0".repeat(MiB)}\r\nhello\r\n0\r\n\r\n`], + ["1 MiB of spaces", `5${" ".repeat(MiB)}\r\nhello\r\n0\r\n\r\n`], +]; + +let chunkBody = ""; +let redirectSocket: Socket | null = null; +let closes = 0; + +const peer = net.createServer((s: Socket) => { + s.on("error", () => {}); + s.on("close", () => { + closes++; + }); + let head = ""; + s.on("data", (d: Buffer) => { + head += d.toString("latin1"); + if (!head.includes("\r\n\r\n")) return; + const lines = head.split("\r\n"); + const path = lines[0]!.split(" ")[1]!; + head = ""; + const send = (status: string, fields: string, body: string): void => { + s.write(`HTTP/1.1 ${status}\r\n${fields}\r\n`); + s.write(body); + }; + if (path === "/echo") { + const received = lines.slice(1).filter((l) => l !== ""); + send("200 OK", `Content-Length: ${JSON.stringify(received).length}\r\n`, JSON.stringify(received)); + } else if (path === "/chunked") { + send("200 OK", "Transfer-Encoding: chunked\r\n", chunkBody); + } else if (path === "/endless") { + s.write("HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n"); + const timer = setInterval(() => s.write("4\r\ndata\r\n"), 5); + s.on("close", () => clearInterval(timer)); + } else if (path === "/gzip") { + s.write(`HTTP/1.1 200 OK\r\nContent-Encoding: gzip\r\nContent-Length: ${packed.length}\r\n\r\n`); + for (let i = 0; i < packed.length; i += 50) s.write(packed.subarray(i, i + 50)); + } else if (path === "/gzip-truncated") { + const part = packed.subarray(0, packed.length - 12); + s.write(`HTTP/1.1 200 OK\r\nContent-Encoding: gzip\r\nContent-Length: ${part.length}\r\n\r\n`); + s.write(part); + } else if (path === "/gzip-empty") { + send("200 OK", "Content-Encoding: gzip\r\nContent-Length: 0\r\n", ""); + } else if (path === "/gzip-bad") { + send("200 OK", "Content-Encoding: gzip\r\nContent-Length: 8\r\n", "not gzip"); + } else if (path === "/cut") { + s.write("HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\n0123456789"); + setTimeout(() => s.destroy(), 20); + } else if (path === "/many-codings") { + send("200 OK", "Content-Encoding: gzip, gzip, gzip, gzip, gzip, gzip\r\nContent-Length: 2\r\n", "ok"); + } else if (path === "/redirect-endless") { + redirectSocket = s; + s.write("HTTP/1.1 302 Found\r\nLocation: /plain\r\nTransfer-Encoding: chunked\r\n\r\n"); + const timer = setInterval(() => s.write("4\r\ndata\r\n"), 5); + s.on("close", () => clearInterval(timer)); + } else if (path === "/no-content") { + send("204 No Content", "", ""); + } else { + send("200 OK", "Content-Length: 5\r\n", "plain"); + } + }); +}); + +export async function checkPeers(): Promise { + await new Promise((resolve) => peer.listen(0, "127.0.0.1", () => resolve())); + const origin = `http://127.0.0.1:${(peer.address() as net.AddressInfo).port}`; + const dispatcher = new Agent(); + + console.log("== peers: header values"); + for (const [name, value] of [ + ["host", "a\r\nX-Injected: 1"], + ["host", "a\nb"], + ["x-test", "a\r\nX-Injected: 1"], + ]) { + try { + const res = await request(`${origin}/echo`, { dispatcher, headers: { [name!]: value! } }); + console.log(JSON.stringify(value), "sent:", await res.body.text()); + } catch (err) { + console.log(JSON.stringify(value), describe(err)); + } + } + + console.log("== peers: chunk-size lines"); + for (const [label, body] of chunkCases) { + chunkBody = body; + try { + const res = await request(`${origin}/chunked`, { dispatcher }); + console.log(label, res.statusCode, JSON.stringify(await res.body.text())); + } catch (err) { + console.log(label, describe(err)); + } + } + + console.log("== peers: fetch bodies"); + const tryFetch = async ( + label: string, + path: string, + read: (res: Response) => Promise, + method = "GET", + signal: AbortSignal | null = null, + ): Promise => { + try { + const res = await fetch(`${origin}${path}`, { dispatcher, method, signal }); + console.log(label, res.status, res.body === null ? "null body" : "body"); + await read(res); + } catch (err) { + console.log(label, describe(err)); + } + }; + await tryFetch("endless", "/endless", async (res) => { + const reader = res.body!.getReader(); + const first = await reader.read(); + console.log("first chunk:", first.done, new TextDecoder().decode(first.value!).slice(0, 4)); + const before = closes; + await reader.cancel(); + await new Promise((resolve) => setTimeout(resolve, 100)); + console.log("cancel closed the connection:", closes > before); + }); + await tryFetch("gzip", "/gzip", async (res) => console.log("text matches:", (await res.text()) === text)); + await tryFetch("gzip-truncated", "/gzip-truncated", async (res) => + console.log("length", (await res.text()).length), + ); + await tryFetch("gzip-empty", "/gzip-empty", async (res) => console.log(JSON.stringify(await res.text()))); + await tryFetch("gzip-bad", "/gzip-bad", async (res) => console.log(JSON.stringify(await res.text()))); + await tryFetch("cut", "/cut", async (res) => console.log(JSON.stringify(await res.text()))); + const controller = new AbortController(); + await tryFetch( + "abort after headers", + "/endless", + async (res) => { + controller.abort(); + console.log(await res.text()); + }, + "GET", + controller.signal, + ); + await tryFetch("many-codings", "/many-codings", async (res) => console.log(await res.text())); + await tryFetch("redirect-endless", "/redirect-endless", async (res) => + console.log("followed:", await res.text()), + ); + await tryFetch("no-content", "/no-content", async (res) => console.log(JSON.stringify(await res.text()))); + await tryFetch("head", "/plain", async (res) => console.log(JSON.stringify(await res.text())), "HEAD"); + + if (redirectSocket !== null) (redirectSocket as Socket).destroy(); + await dispatcher.close(); + peer.close(); +} diff --git a/tests/fixtures/package-shims/undici-bounded.ts b/tests/fixtures/package-shims/undici-bounded.ts new file mode 100644 index 000000000..a531a6649 --- /dev/null +++ b/tests/fixtures/package-shims/undici-bounded.ts @@ -0,0 +1,29 @@ +// Reads every response of undici-hostile-peer.mjs (port in argv[2]): the +// endless chunk-size lines through request(), the large and the gzipped +// bodies through fetch() with a pause after the first chunk. Memory stays +// bounded by the parser's and the body stream's buffers. +import { Agent, fetch, request } from "undici"; + +const MiB = 1024 * 1024; +const origin = `http://127.0.0.1:${process.argv[2]}`; +const dispatcher = new Agent(); +for (const path of ["/extension", "/zeros"]) { + const res = await request(`${origin}${path}`, { dispatcher }); + console.log(path, res.statusCode, await res.body.text()); +} +for (const path of ["/big", "/bomb"]) { + const res = await fetch(`${origin}${path}`, { dispatcher }); + const reader = res.body!.getReader(); + const first = await reader.read(); + let total = first.value!.length; + await new Promise((resolve) => setTimeout(resolve, 500)); + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + total += value!.length; + } + console.log(path, res.status, total / MiB, "MiB"); +} +const rss = process.memoryUsage().rss; +console.log(rss < 48 * MiB ? "rss under 48 MiB" : `rss ${Math.round(rss / MiB)} MiB`); +await dispatcher.close(); diff --git a/tests/fixtures/package-shims/undici-hostile-peer.mjs b/tests/fixtures/package-shims/undici-hostile-peer.mjs new file mode 100644 index 000000000..ce6c5881c --- /dev/null +++ b/tests/fixtures/package-shims/undici-hostile-peer.mjs @@ -0,0 +1,49 @@ +// A peer for package-shims.test.ts: prints its port, then answers +// /extension a chunk whose size line carries 64 MiB of extension, +// /zeros a chunk size with 64 MiB of leading zeros, +// /big a 64 MiB body, as fast as the socket takes it, +// /bomb 64 MiB of zeros, gzipped, +// and exits when its stdin closes. +import * as net from "node:net"; +import * as zlib from "node:zlib"; + +const MiB = 1024 * 1024; +const size = 64 * MiB; +const bomb = zlib.gzipSync(Buffer.alloc(size)); + +function stream(s, head, piece, tail) { + s.write(head); + let sent = 0; + const pump = () => { + while (sent < size) { + sent += piece.length; + if (!s.write(piece)) return void s.once("drain", pump); + } + s.write(tail); + }; + pump(); +} + +const server = net.createServer((s) => { + s.on("error", () => {}); + let head = ""; + s.on("data", (d) => { + head += d.toString("latin1"); + if (!head.includes("\r\n\r\n")) return; + const path = head.split(" ")[1]; + head = ""; + const chunked = "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n"; + if (path === "/extension") { + stream(s, `${chunked}5;a=`, Buffer.alloc(64 * 1024, 98), "\r\nhello\r\n0\r\n\r\n"); + } else if (path === "/zeros") { + stream(s, chunked, Buffer.alloc(64 * 1024, 48), "5\r\nhello\r\n0\r\n\r\n"); + } else if (path === "/big") { + stream(s, `HTTP/1.1 200 OK\r\nContent-Length: ${size}\r\n\r\n`, Buffer.alloc(64 * 1024, 97), ""); + } else { + s.write(`HTTP/1.1 200 OK\r\nContent-Encoding: gzip\r\nContent-Length: ${bomb.length}\r\n\r\n`); + s.write(bomb); + } + }); +}); +server.listen(0, "127.0.0.1", () => console.log(server.address().port)); +process.stdin.on("end", () => process.exit(0)).resume(); diff --git a/tests/fixtures/package-shims/undici5-host.ts b/tests/fixtures/package-shims/undici5-host.ts new file mode 100644 index 000000000..92b395ff8 --- /dev/null +++ b/tests/fixtures/package-shims/undici5-host.ts @@ -0,0 +1,26 @@ +// A Host header value with CR or LF, through the undici 5.x profile: it +// throws before the request is written, as undici 5.29 does (5.12, the +// vendored fixture version, writes the value into the request head). +import * as http from "node:http"; +import { Agent, errors, request } from "undici"; + +const server = http.createServer((_req, res) => { + res.statusCode = 204; + res.end(); +}); +server.listen(0, "127.0.0.1", async () => { + const origin = `http://127.0.0.1:${(server.address() as { port: number }).port}`; + const dispatcher = new Agent(); + for (const host of ["ok.test", "a\r\nX-Injected: 1", "a\nb"]) { + try { + const res = await request(`${origin}/`, { dispatcher, headers: { host } }); + await res.body.dump(); + console.log(JSON.stringify(host), res.statusCode); + } catch (err) { + if (err instanceof errors.UndiciError) console.log(JSON.stringify(host), err.name, err.code, err.message); + else throw err; + } + } + await dispatcher.close(); + server.close(); +}); diff --git a/tests/harness/package-shims.test.ts b/tests/harness/package-shims.test.ts index 3b31c5e33..0be52d2df 100644 --- a/tests/harness/package-shims.test.ts +++ b/tests/harness/package-shims.test.ts @@ -6,10 +6,14 @@ * package, and the package's code never runs; * - a shimmed package that is not installed, or whose major version has no * shim, resolves exactly as before (here: refused like any missing or - * uncompiled package). + * uncompiled package); + * - the undici shims keep memory bounded against a peer that sends endless + * chunk-size lines and large or compressed bodies, and the 5.x profile + * rejects a Host value with CR or LF as undici 5.29 does (the vendored + * 5.12 writes it into the request). */ -import { execFile } from "node:child_process"; -import { cpSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { execFile, spawn } from "node:child_process"; +import { copyFileSync, cpSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { promisify } from "node:util"; @@ -19,6 +23,7 @@ import { compile } from "@scriptc/compiler"; const execFileAsync = promisify(execFile); const repoRoot = join(import.meta.dirname, "../.."); const casesRoot = join(repoRoot, "tests/fixtures/npm/cases"); +const shimFixtures = join(repoRoot, "tests/fixtures/package-shims"); const sanitize = process.env["SCRIPTC_SAN"] === "1"; const scratch = mkdtempSync(join(tmpdir(), "scriptc-package-shims-")); @@ -49,6 +54,37 @@ function installEncoding(dir: string): void { ); } +/** Compiles `entry` (a file in tests/fixtures/package-shims) as main.ts of + * a copy of the npm case `name`, which carries the package. */ +async function buildInCase(name: string, entry: string): Promise<{ dir: string; binary: string }> { + const dir = copyCase(name, `${name}-${entry.replace(/\..*$/, "")}`); + copyFileSync(join(shimFixtures, entry), join(dir, "main.ts")); + const out = join(dir, ".scriptc"); + const result = await compile(join(dir, "main.ts"), { + outPath: join(out, "program"), + outDir: out, + sanitize, + }); + if (!result.ok) throw new Error(result.diagnostics.map((d) => d.message).join("\n")); + return { dir, binary: result.binaryPath }; +} + +/** Runs `binary ` against undici-hostile-peer.mjs. */ +async function runAgainstPeer(binary: string, cwd: string): Promise { + const peer = spawn(process.execPath, [join(shimFixtures, "undici-hostile-peer.mjs")], { + stdio: ["pipe", "pipe", "inherit"], + }); + try { + const port = await new Promise((resolve, reject) => { + peer.stdout.once("data", (d: Buffer) => resolve(d.toString().trim())); + peer.once("exit", (code) => reject(new Error(`peer exited with ${code}`))); + }); + return await run(binary, [port], cwd); + } finally { + peer.stdin.end(); + } +} + describe("package shims", () => { test("an unloaded package is never loaded, even when installed", async () => { const original = join(casesRoot, "unloaded-optional-dependency"); @@ -112,4 +148,32 @@ describe("package shims", () => { expect(result.diagnostics.map((d) => d.message).join("\n")).not.toContain("shims/undici"); } }, 180_000); + + for (const name of ["undici7", "undici5"]) { + test(`${name}: endless chunk-size lines and large bodies stay in bounded memory`, async () => { + const { dir, binary } = await buildInCase(name, "undici-bounded.ts"); + const out = await runAgainstPeer(binary, dir); + expect(out.split("\n").slice(0, 4)).toEqual([ + "/extension 200 hello", + "/zeros 200 hello", + "/big 200 64 MiB", + "/bomb 200 64 MiB", + ]); + // ASan's shadow memory and quarantine keep freed buffers mapped. + if (!sanitize) expect(out.split("\n")[4]).toBe("rss under 48 MiB"); + }, 300_000); + } + + test("undici5: a Host value with CR or LF throws before the request is written", async () => { + const { dir, binary } = await buildInCase("undici5", "undici5-host.ts"); + // undici 5.29.0's output. + expect(await run(binary, [], dir)).toBe( + [ + '"ok.test" 204', + '"a\\r\\nX-Injected: 1" InvalidArgumentError UND_ERR_INVALID_ARG invalid host header', + '"a\\nb" InvalidArgumentError UND_ERR_INVALID_ARG invalid host header', + "", + ].join("\n"), + ); + }, 180_000); }); From bbefa3639465a689658139eaa847f97de1c0fc83 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Mon, 19 Oct 2026 23:19:46 -0500 Subject: [PATCH 07/38] Bound loop counters compared as `i + c` with a bound The range analysis narrowed a counter only when the comparison read the counter itself (`i < n`). A comparison of the counter plus or minus an integer literal (`i + 1 < xs.length`, `1 + i < n`, `i - 1 >= 0`) now narrows it as `i < n - c` does. Against a bound within 2^52, a sum that rounds has a magnitude of at least 2^53, so every such comparison answers as the exact sum would. The counter of `for (let i = 0; i + 1 < xs.length; i++)` then lands in an i32 slot like the one of `i < xs.length - 1`, and helper calls that index arrays with it read the elements without the double index checks. --- .../compiler/src/ir/integer-ranges.test.ts | 74 +++++++++++++++ packages/compiler/src/ir/integer-ranges.ts | 45 +++++++-- .../5685-loop-counter-offset-bounds.ts.json | 6 ++ .../corpus/5685-loop-counter-offset-bounds.ts | 94 +++++++++++++++++++ .../5685-loop-counter-offset-bounds.ts.json | 1 + 5 files changed, 213 insertions(+), 7 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5685-loop-counter-offset-bounds.ts.json create mode 100644 tests/corpus/5685-loop-counter-offset-bounds.ts create mode 100644 tests/harness/corpus-costs/5685-loop-counter-offset-bounds.ts.json diff --git a/packages/compiler/src/ir/integer-ranges.test.ts b/packages/compiler/src/ir/integer-ranges.test.ts index 8e1c97f66..a8c9978f3 100644 --- a/packages/compiler/src/ir/integer-ranges.test.ts +++ b/packages/compiler/src/ir/integer-ranges.test.ts @@ -867,6 +867,80 @@ test("an array length bounds while and for counters even when the body resizes t expect(analyzeIntegerRanges(g).get(cursor)).toEqual({ min: 0, max: 4294967295 }); }); +test("a counter plus or minus a literal compared with a bound narrows the counter", () => { + // for (let i = 0; i + 1 < items.length; i++) { pair(items, i, i + 1); } + const ARR = { kind: "array", elem: F64 } as const; + const items: IrExpr = { loc, kind: "varRef", localId: "items", type: ARR }; + const length: IrExpr = { + loc, + kind: "arrIntrinsic", + method: "length", + receiver: items, + args: [], + type: F64, + }; + const inside = ref("i"), + next = bin("+", ref("i"), num(1)), + step = bin("+", ref("i"), num(1)); + const call: IrStmt = { + loc, + kind: "exprStmt", + expr: { loc, kind: "call", callee: "pair", args: [items, inside, next], type: F64 }, + }; + const f = fn([ + { + loc, + kind: "for", + init: { loc, kind: "varDecl", localId: "i", init: num(0) }, + cond: condition("<", bin("+", ref("i"), num(1)), length), + update: { loc, kind: "assign", localId: "i", value: step }, + body: [call], + }, + ]); + f.locals.push( + { id: "i", name: "i", type: F64, mutable: true }, + { id: "items", name: "items", type: ARR, mutable: false }, + ); + const ranges = analyzeIntegerRanges(f); + expect(ranges.get(inside)).toEqual({ min: 0, max: 4294967293 }); + expect(ranges.get(next)).toEqual({ min: 1, max: 4294967294 }); + expect(ranges.get(step)).toEqual({ min: 1, max: 4294967294 }); + + // if (1 + x < 10) observe(x); if (x - 2 >= 0) observe(x); with x in [0, 100] + const below = ref(), + above = ref(), + wide = ref(); + const g = fn([ + assign(bin("&", ref("unknown"), num(100))), + { + loc, + kind: "if", + cond: condition("<", bin("+", num(1), ref()), num(10)), + then: [observe(below)], + else_: null, + }, + { + loc, + kind: "if", + cond: condition(">=", bin("-", ref(), num(2)), num(0)), + then: [observe(above)], + else_: null, + }, + // A bound beyond 2^52 narrows nothing through an offset. + { + loc, + kind: "if", + cond: condition("<", bin("+", ref(), num(1)), num(2 ** 53)), + then: [observe(wide)], + else_: null, + }, + ]); + const gRanges = analyzeIntegerRanges(g); + expect(gRanges.get(below)).toEqual({ min: 0, max: 8 }); + expect(gRanges.get(above)).toEqual({ min: 2, max: 100 }); + expect(gRanges.get(wide)).toEqual({ min: 0, max: 100 }); +}); + test("whole-program slot facts seed parameters, field reads and call results", () => { const field: IrExpr = { loc, diff --git a/packages/compiler/src/ir/integer-ranges.ts b/packages/compiler/src/ir/integer-ranges.ts index 5c365e2ca..410c1b6f7 100644 --- a/packages/compiler/src/ir/integer-ranges.ts +++ b/packages/compiler/src/ir/integer-ranges.ts @@ -215,6 +215,24 @@ export interface IntegerSlotFacts { elements?: (arr: IrExpr) => boolean; } +/** A local read, or a numeric local plus or minus an integer literal + * (`i + 1`, `1 + i`, `i - 2`): the local and its offset. */ +function offsetRef(e: IrExpr): { localId: string; offset: number } | null { + if (e.kind === "varRef") return { localId: e.localId, offset: 0 }; + if (e.kind !== "bin" || (e.op !== "+" && e.op !== "-") || e.type.kind !== "f64") return null; + const literal = (x: IrExpr): number | null => + x.kind === "numLit" && Number.isSafeInteger(x.value) && !Object.is(x.value, -0) + ? x.value + : null; + const right = literal(e.right); + if (e.left.kind === "varRef" && e.left.type.kind === "f64" && right !== null) + return { localId: e.left.localId, offset: e.op === "+" ? right : -right }; + const left = literal(e.left); + if (e.op === "+" && e.right.kind === "varRef" && e.right.type.kind === "f64" && left !== null) + return { localId: e.right.localId, offset: left }; + return null; +} + /** Structured local dataflow. Calls cannot write uncaptured local slots, * but argument expressions can; unknown evaluation forms invalidate all * syntactically written slots before their children are inspected. Loops @@ -508,7 +526,7 @@ export function analyzeIntegerRanges(fn: IrFunction, slots: IntegerSlotFacts = { let ref = condition.left; let bound = condition.right; let op: string = condition.op; - if (ref.kind !== "varRef" && bound.kind === "varRef") { + if (offsetRef(ref) === null && offsetRef(bound) !== null) { [ref, bound] = [bound, ref]; op = ( @@ -518,18 +536,31 @@ export function analyzeIntegerRanges(fn: IrFunction, slots: IntegerSlotFacts = { > )[op] ?? ""; } - if (ref.kind !== "varRef") return; - const boundRange = + const target = offsetRef(ref); + if (target === null) return; + const literalBound = bound.kind === "numLit" && Number.isFinite(bound.value) ? { min: bound.value, max: bound.value } : ranges.get(bound); - if (!boundRange) return; + if (!literalBound) return; // An Int32 contract local holds an int32 even where the dataflow has // no fact for it (loop headers, after unknown regions). const previous = - facts.get(ref.localId) ?? - (eligible.has(ref.localId) ? (slots.locals?.(ref.localId) ?? undefined) : undefined); + facts.get(target.localId) ?? + (eligible.has(target.localId) ? (slots.locals?.(target.localId) ?? undefined) : undefined); if (!previous) return; + // `x + c < b` bounds x as `x < b - c` does. A sum that rounds has a + // magnitude of at least 2^53, so against a bound within 2^52 every + // comparison answers as the exact sum would. + const offset = target.offset; + if ( + offset !== 0 && + (Math.abs(offset) > 2 ** 31 || + Math.abs(literalBound.min) > 2 ** 52 || + Math.abs(literalBound.max) > 2 ** 52) + ) + return; + const boundRange = { min: literalBound.min - offset, max: literalBound.max - offset }; if (!truth) op = ( @@ -552,7 +583,7 @@ export function analyzeIntegerRanges(fn: IrFunction, slots: IntegerSlotFacts = { else if (max === boundRange.max) max -= 1; } if (min <= max && Number.isInteger(min) && Number.isInteger(max)) - facts.set(ref.localId, { min, max }); + facts.set(target.localId, { min, max }); } function expr(e: IrExpr, facts: Facts): IntegerRange | null { let range: IntegerRange | null = null; diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5685-loop-counter-offset-bounds.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5685-loop-counter-offset-bounds.ts.json new file mode 100644 index 000000000..da2bfdbe2 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5685-loop-counter-offset-bounds.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5685-loop-counter-offset-bounds.ts" + ], + "diags": [] +} diff --git a/tests/corpus/5685-loop-counter-offset-bounds.ts b/tests/corpus/5685-loop-counter-offset-bounds.ts new file mode 100644 index 000000000..e43034496 --- /dev/null +++ b/tests/corpus/5685-loop-counter-offset-bounds.ts @@ -0,0 +1,94 @@ +// Loops whose condition compares the counter plus or minus a literal with a +// bound (`i + 1 < xs.length`, `1 + i < n`, `i - 1 >= 0`): fractional, NaN, +// infinite and negative bounds, counters past 2^31 and near 2^53, helper +// calls with the counter, string lengths and arrays that grow in the body. + +interface Point { + x: number; + y: number; +} +function pointAt(xs: number[], ys: number[], i: number): Point { + return { x: xs[i], y: ys[i] }; +} +function gap(p: Point, q: Point): number { + return Math.abs(p.x - q.x) + Math.abs(p.y - q.y); +} + +const xs: number[] = []; +const ys: number[] = []; +for (let i = 0; i < 50; i++) { + xs.push((i * 37) % 11); + ys.push((i * 53) % 7); +} + +let path = 0; +for (let i = 0; i + 1 < xs.length; i++) path += gap(pointAt(xs, ys, i), pointAt(xs, ys, i + 1)); +console.log("path", path); + +let pairs = 0; +for (let i = 0; 1 + i < xs.length; i++) pairs += xs[i]! * xs[i + 1]!; +console.log("pairs", pairs); + +let triples = 0; +for (let i = 0; i + 2 <= xs.length - 1; i += 2) triples += xs[i]! + xs[i + 1]! + xs[i + 2]!; +console.log("triples", triples); + +// Bounds that are not integers, or not finite. +function count(bound: number): number { + let n = 0; + for (let i = 0; i + 2 < bound; i++) { + n++; + if (n > 20) break; + } + return n; +} +console.log("bounds", count(7.5), count(2), count(2.0001), count(-3), count(Number.NaN), count(Infinity)); + +// Counters below zero, past int32 and near the end of exact integers. +let negative = 0; +for (let i = -5; i + 5 < 3; i++) negative += i; +console.log("negative", negative); +let wide = 0; +for (let i = 2 ** 31 - 3; i + 1 < 2 ** 31 + 3; i++) wide += i - 2 ** 31; +console.log("past int32", wide); +const exact: number[] = []; +for (let i = 2 ** 53 - 4; i + 1 < 2 ** 53; i++) exact.push(i - 2 ** 53); +console.log("near 2^53", exact.join(",")); +let half = 0; +for (let i = 0; i + 0.5 < 4; i++) half += i; +console.log("half offset", half); + +// Descending counters. +const down: number[] = []; +for (let i = 6; i - 1 >= 0; i--) down.push(i); +console.log("down", down.join(",")); +let back = 0; +for (let i = xs.length - 1; i - 1 >= 0; i--) back += xs[i]! - xs[i - 1]!; +console.log("back", back); + +// A while loop on a string's length. +const text = "abcdefghij"; +let pos = 0; +let codes = 0; +while (pos + 3 < text.length) { + codes += text.charCodeAt(pos) * text.charCodeAt(pos + 3); + pos++; +} +console.log("string", pos, codes); + +// An array that grows inside the loop: the bound is reread on every pass. +const grow = [1, 2, 3]; +let steps = 0; +for (let i = 0; i + 1 < grow.length; i++) { + if (grow.length < 12) grow.push(grow[i]! + grow[i + 1]!); + steps++; +} +console.log("grow", steps, grow.join(",")); + +// The counter after the loop, and an equality bound. +let last = 0; +for (last = 0; last + 1 < 9; last++) {} +console.log("last", last); +let hits = 0; +for (let i = 0; i < 10; i++) if (i + 3 === 7) hits += i; +console.log("equal", hits); diff --git a/tests/harness/corpus-costs/5685-loop-counter-offset-bounds.ts.json b/tests/harness/corpus-costs/5685-loop-counter-offset-bounds.ts.json new file mode 100644 index 000000000..1c0e5bd28 --- /dev/null +++ b/tests/harness/corpus-costs/5685-loop-counter-offset-bounds.ts.json @@ -0,0 +1 @@ +{ "plain": 2450, "san": 2700 } From 9548ab3663d2c89261c9213d5f8fa23ce2a48e6f Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Mon, 19 Oct 2026 23:38:15 -0500 Subject: [PATCH 08/38] Give the undici npm cases a 600 s timeout The undici cases compile the built-in undici implementation's module graph; a sanitized build of it on a CI runner outlasted the default 120 s per case. --- .../compiler/test/ts7/resolver-parity.test.ts | 15 +++++- .../compiler/test/undici-shim-types.test.ts | 46 +++++++++---------- tests/harness/node-types-timers.test.ts | 14 ++++-- tests/harness/npm.test.ts | 7 ++- 4 files changed, 50 insertions(+), 32 deletions(-) diff --git a/packages/compiler/test/ts7/resolver-parity.test.ts b/packages/compiler/test/ts7/resolver-parity.test.ts index b1788d4da..b3190ae9f 100644 --- a/packages/compiler/test/ts7/resolver-parity.test.ts +++ b/packages/compiler/test/ts7/resolver-parity.test.ts @@ -22,6 +22,7 @@ import { resolveTypeDirective, setProjectPathMappings, } from "../../src/frontend/resolve.js"; +import { packageShimFor } from "../../src/frontend/package-shims.js"; import { isRelativeSpecifier } from "../../src/frontend/workspace-registry.js"; import { options5 } from "./harness.js"; @@ -143,8 +144,18 @@ test("relative and bare specifiers across the whole test tree resolve identicall // for bare specifiers are exactly the package.json-mediated // resolutions inside the project — resolveProjectImport must agree. const oursProj = resolveProjectImport(file, spec); - const theirsProj = - reference && !reference.isExternalLibraryImport ? reference.resolvedFileName : null; + // The deliberate package-shim delta: a package scriptc implements + // itself (package-shims.ts) answers its shim module here, where + // 5.9.3 answers the installed package as an external import; the + // installed package's answer is still compared below. + const shimmed = packageShimFor(file, spec) !== null; + const theirsProj = shimmed + ? reference && reference.isExternalLibraryImport + ? packageShimFor(file, spec) + : null + : reference && !reference.isExternalLibraryImport + ? reference.resolvedFileName + : null; if (oursProj !== theirsProj) { failures.push(`${file} '${spec}' (project): ours=${oursProj} ts5=${theirsProj}`); } diff --git a/packages/compiler/test/undici-shim-types.test.ts b/packages/compiler/test/undici-shim-types.test.ts index c7bc4ccaf..a0b770add 100644 --- a/packages/compiler/test/undici-shim-types.test.ts +++ b/packages/compiler/test/undici-shim-types.test.ts @@ -18,9 +18,12 @@ * program the check builds includes them), since real projects compile * them with their own Node types. */ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; import { join } from "node:path"; -import ts5 from "typescript5"; import { describe, expect, test } from "vitest"; +import * as ad from "../src/frontend/ts7/adapter.js"; +import { createProgram, getPreEmitDiagnostics } from "../src/frontend/ts7/program-adapter.js"; const repoRoot = join(import.meta.dirname, "../../.."); const shimRoot = join(repoRoot, "packages/compiler/shims/undici"); @@ -282,31 +285,29 @@ type _clientApi = ExpectNever, keyof real.Clie `; } -function check(files: Record): string[] { - const options: ts5.CompilerOptions = { +/** The diagnostics of one program, checked by the TypeScript 7 checker. */ +function check(name: string, text: string): string[] { + const dir = mkdtempSync(join(tmpdir(), "scriptc-undici-shim-types-")); + const file = join(dir, name); + writeFileSync(file, text); + const program = createProgram([file], { strict: true, noEmit: true, skipLibCheck: true, allowImportingTsExtensions: true, - module: ts5.ModuleKind.ESNext, - moduleResolution: ts5.ModuleResolutionKind.Bundler, - target: ts5.ScriptTarget.ES2022, + module: ad.ModuleKind.ESNext as number, + moduleResolution: ad.ModuleResolutionKind.Bundler as number, + target: ad.ScriptTarget.ES2022 as number, lib: ["lib.es2023.d.ts"], types: ["node"], typeRoots: [join(repoRoot, "node_modules/@types")], - }; - const host = ts5.createCompilerHost(options); - const readFile = host.readFile.bind(host); - const fileExists = host.fileExists.bind(host); - host.readFile = (name) => files[name] ?? readFile(name); - host.fileExists = (name) => name in files || fileExists(name); - const program = ts5.createProgram(Object.keys(files), options, host); - return ts5.getPreEmitDiagnostics(program).map((d) => { - const where = d.file - ? `${d.file.fileName}:${d.file.getLineAndCharacterOfPosition(d.start ?? 0).line + 1}` - : ""; - return `${where} ${ts5.flattenDiagnosticMessageText(d.messageText, "\n")}`; }); + try { + return getPreEmitDiagnostics(program).map((d) => `${d.fileName ?? ""}:${d.pos} ${d.text}`); + } finally { + program.dispose(); + rmSync(dir, { recursive: true, force: true }); + } } describe.each([5, 7])("undici %i.x shim types", (major) => { @@ -314,18 +315,15 @@ describe.each([5, 7])("undici %i.x shim types", (major) => { const shim = join(shimRoot, `v${major}.ts`); test("a program written against undici type-checks against the real declarations", () => { - const file = join(repoRoot, `.undici-usage-real-${major}.ts`); - expect(check({ [file]: helpers + usage(major).replace(/UNDICI/g, real) })).toEqual([]); + expect(check("usage-real.ts", helpers + usage(major).replace(/UNDICI/g, real))).toEqual([]); }); test("the same program type-checks against the shim", () => { - const file = join(repoRoot, `.undici-usage-shim-${major}.ts`); - expect(check({ [file]: helpers + usage(major).replace(/UNDICI/g, shim) })).toEqual([]); + expect(check("usage-shim.ts", helpers + usage(major).replace(/UNDICI/g, shim))).toEqual([]); }); test("the shim's exports and records are a compatible subset", () => { - const file = join(repoRoot, `.undici-shape-${major}.ts`); const text = helpers + shape(major).replace(/REAL/g, real).replace(/SHIM/g, shim); - expect(check({ [file]: text })).toEqual([]); + expect(check("shape.ts", text)).toEqual([]); }); }); diff --git a/tests/harness/node-types-timers.test.ts b/tests/harness/node-types-timers.test.ts index 257d644ff..faaafcca9 100644 --- a/tests/harness/node-types-timers.test.ts +++ b/tests/harness/node-types-timers.test.ts @@ -53,7 +53,7 @@ const program = `class Holder { } const holder = new Holder(); holder.timer = setTimeout(() => console.log("never"), 50); -const kept = setTimeout(() => console.log("fired", holder.timer === null), 10); +const kept = setTimeout(() => console.log("fired", holder.timer === null), 200); kept.unref().ref(); kept.refresh(); console.log("hasRef", kept.hasRef()); @@ -84,10 +84,14 @@ test("@types/node 20 class-declared timer handles compile and match Node", async }); if (!result.ok) throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); - const [node, native] = await Promise.all([ - execFileAsync(process.execPath, [join(dir, "main.ts")], { cwd: dir, encoding: "utf8" }), - execFileAsync(result.binaryPath, [], { cwd: dir, encoding: "utf8" }), - ]); + // One after the other, and the timer 200 ms out: under load Node's own + // startup can outlast a short timer, which then runs before the + // immediate. + const node = await execFileAsync(process.execPath, [join(dir, "main.ts")], { + cwd: dir, + encoding: "utf8", + }); + const native = await execFileAsync(result.binaryPath, [], { cwd: dir, encoding: "utf8" }); expect(native.stdout).toBe(node.stdout); expect(native.stdout).toBe("hasRef true\nimmediate\nfired true\n"); }, 180_000); diff --git a/tests/harness/npm.test.ts b/tests/harness/npm.test.ts index 54207f246..f6f31f35c 100644 --- a/tests/harness/npm.test.ts +++ b/tests/harness/npm.test.ts @@ -113,6 +113,11 @@ const KNOWN: Record = { const cases = shardSelect(npmCases(fixturesRoot), (c) => c.name); +/** Per-case timeouts past the default 120 s: the undici cases compile the + * built-in undici implementation's module graph, which a sanitized build + * on a CI runner takes minutes to compile. */ +const TIMEOUTS: Record = { undici5: 600_000, undici7: 600_000 }; + interface RunResult { stdout: Buffer; stderr: Buffer; @@ -235,7 +240,7 @@ describe(`npm native (${cases.length} programs${sanitize ? ", sanitized" : ""}${ } await expectMatchesNode(c, built.binary); }, - 120_000, + TIMEOUTS[c.name] ?? 120_000, ); } }); From 88e032bf65b3076f3eda5cc8f95809128ce5521a Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 20 Oct 2026 00:32:42 -0500 Subject: [PATCH 09/38] Keep counters a for loop bounds in integer shadows A loop that counts with number locals (`lines++`, `run = 0`, `longest = run`) kept each counter a double, so every increment was a dependent floating-point add, and conditional increments compiled to branches. The speculative integer copy only covered loops with products or variable steps. The range analysis now bounds for-loop cursors that are reset to an integer literal or set to another bounded binding, besides those it already bounded by their increments. A cursor's writes take its loop range, which holds every value it takes in the loop, so `changes + 1` stays proven at the limit of a typed array's length. The analysis reports these loop ranges per for loop. Where an exception leaves the function (no enclosing try or finally, no coroutine), the emitter keeps such cursors in i64 shadows for the loop: one copy of the loop, no checks, and each double is written back when the loop exits. A loop with a labeled break or continue keeps its doubles. --- .../src/backend/llvm/emit-function.ts | 6 +- .../src/backend/llvm/emit-statements.ts | 101 ++++++++++- packages/compiler/src/backend/llvm/emitter.ts | 9 +- .../backend/llvm/speculative-integers.test.ts | 72 ++++++++ .../src/backend/llvm/speculative-integers.ts | 106 +++++++++++- .../compiler/src/ir/integer-loop-facts.ts | 80 +++++++-- .../compiler/src/ir/integer-ranges.test.ts | 117 +++++++++++++ packages/compiler/src/ir/integer-ranges.ts | 83 ++++++++- ...5686-loop-cursor-resets-and-copies.ts.json | 6 + .../5686-loop-cursor-resets-and-copies.ts | 157 ++++++++++++++++++ ...5686-loop-cursor-resets-and-copies.ts.json | 1 + 11 files changed, 699 insertions(+), 39 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5686-loop-cursor-resets-and-copies.ts.json create mode 100644 tests/corpus/5686-loop-cursor-resets-and-copies.ts create mode 100644 tests/harness/corpus-costs/5686-loop-cursor-resets-and-copies.ts.json diff --git a/packages/compiler/src/backend/llvm/emit-function.ts b/packages/compiler/src/backend/llvm/emit-function.ts index 78f208ff1..c26c93b84 100644 --- a/packages/compiler/src/backend/llvm/emit-function.ts +++ b/packages/compiler/src/backend/llvm/emit-function.ts @@ -27,7 +27,7 @@ import { findMapEntryFusions } from "./map-entry-fusion.js"; import { findLocalStackUnions } from "./stack-unions.js"; import { findLocalStackObjects, findReassignedStackObjects } from "./stack-objects.js"; import { findLocalUnionStorage } from "./local-union-storage.js"; -import { INT32_RANGE, analyzeIntegerRanges } from "../../ir/integer-ranges.js"; +import { INT32_RANGE, analyzeIntegerRangeFacts } from "../../ir/integer-ranges.js"; import { findBytesBounds } from "./bytes-bounds.js"; import { findInt32Locals } from "./int32-locals.js"; import { findIntegerViews } from "./integer-views.js"; @@ -217,7 +217,9 @@ export function emitFunction(emitter: LlEmitter, fn: IrFunction): string { emitter.localUnionStorage = new Map(); emitter.walkBorrows = emitter.walkBorrowsByFunction.get(fn.name) ?? new Set(); const slotFacts = emitter.int32Slots.facts(fn.name); - emitter.integerRanges = analyzeIntegerRanges(numericFn, slotFacts); + const rangeFacts = analyzeIntegerRangeFacts(numericFn, slotFacts); + emitter.integerRanges = rangeFacts.ranges; + emitter.integerLoopRanges = rangeFacts.loopRanges; emitter.int32Elements = slotFacts.elements ?? null; emitter.bytesBounds = findBytesBounds(numericFn, emitter.integerRanges); // Debug builds keep every number local a double for the debugger's diff --git a/packages/compiler/src/backend/llvm/emit-statements.ts b/packages/compiler/src/backend/llvm/emit-statements.ts index e641f3ecd..1a6e73c17 100644 --- a/packages/compiler/src/backend/llvm/emit-statements.ts +++ b/packages/compiler/src/backend/llvm/emit-statements.ts @@ -50,7 +50,9 @@ import { SPECULATIVE_RANGE, emitSafeIntegerCheck, emitSpeculativeInteger, + planProvenIntegerLoop, planSpeculativeLoop, + type ProvenLoopPlan, speculativeReads, speculativeWriteTarget, } from "./speculative-integers.js"; @@ -1765,7 +1767,9 @@ export function emitLoopCopies( ) : null; if (!plan) { - emitCopy(exit); + const proven = allowed ? planProvenLoop(emitter, loop) : null; + if (proven) emitProvenLoop(emitter, proven, exit, emitCopy); + else emitCopy(exit); return; } const B = emitter.B; @@ -1844,7 +1848,7 @@ export function emitLoopCopies( const leave = B.newLabel("spec.leave"); emitCopy(leave); B.startBlock(leave); - emitter.writeBackSpeculativeGlobals(deferred); + emitter.writeBackDeferred(deferred); B.br(exit); } } finally { @@ -1853,10 +1857,91 @@ export function emitLoopCopies( } } +/** A proven integer loop (speculative-integers.ts) for this loop, or null. + * A shadowed local's double is stale until the loop is left, which only + * the loop's exit, a return and an exception do: the loop is planned only + * where an exception leaves the function (no enclosing try or finally, no + * coroutine frame), so no handler reads the stale double. */ +function planProvenLoop( + emitter: LlEmitter, + loop: Extract, +): ProvenLoopPlan | null { + if ( + loop.kind !== "for" || + !emitter.speculativeLoopsEnabled || + emitter.speculation !== null || + emitter.speculationResume !== null || + emitter.tryStack.length > 0 || + emitter.finallyStack.length > 0 || + emitter.currentCoro !== null + ) + return null; + const cursors = emitter.integerLoopRanges.get(loop); + if (!cursors) return null; + return planProvenIntegerLoop( + loop, + cursors, + (id) => emitter.currentLocals.has(id) && emitter.speculativeLocal(id), + (id) => emitter.integerLoopBindings.has(id) || emitter.int32Locals.has(id), + (e) => { + const range = emitter.integerRanges.get(e); + return ( + range !== undefined && + range !== null && + range.min >= -Number.MAX_SAFE_INTEGER && + range.max <= Number.MAX_SAFE_INTEGER + ); + }, + ); +} + +/** Emit a proven integer loop: its shadowed locals start from their + * doubles (integers in their loop ranges on entry), its planned writes + * compute integers into the shadows only, and leaving the loop through its + * exit writes each double back. */ +function emitProvenLoop( + emitter: LlEmitter, + plan: ProvenLoopPlan, + exit: string, + emitCopy: (exit: string) => void, +): void { + const B = emitter.B; + for (const [id, range] of plan.locals) { + const shadow = B.slot(); + B.entryAllocas.push( + `${shadow} = alloca i64 ; integer ${emitter.currentLocals.get(id)?.name ?? id}`, + ); + const value = B.tmp(), + integer = B.tmp(); + B.line(`${value} = load double, ptr ${emitter.binding(id).slot}`); + B.line(`${integer} = fptosi double ${value} to i64`); + B.line(`store i64 ${integer}, ptr ${shadow}`); + emitter.integerLoopBindings.set(id, { slot: shadow, type: "i64", signed: true, range }); + } + const deferred = new Set(); + for (const id of plan.locals.keys()) deferred.add(id); + emitter.speculation = { writes: plan.writes, resume: new Map(), deferred }; + try { + const leave = B.newLabel("integer.leave"); + emitCopy(leave); + B.startBlock(leave); + emitter.writeBackDeferred(deferred); + B.br(exit); + } finally { + emitter.speculation = null; + for (const id of plan.locals.keys()) emitter.integerLoopBindings.delete(id); + } +} + /** A speculated write in the fast copy: compute the value as a safe * integer, store it to the shadow and its exact double to the local. - * Any failed check re-executes the statement in the ordinary copy. */ -export function emitSpeculativeWrite(emitter: LlEmitter, s: IrStmt, resume: string): void { + * Any failed check re-executes the statement in the ordinary copy. A + * proven integer loop's write (no `resume`) has nothing to check. */ +export function emitSpeculativeWrite( + emitter: LlEmitter, + s: IrStmt, + resume: string | undefined, +): void { const B = emitter.B; const target = speculativeWriteTarget(s)!; const binding = emitter.integerLoopBindings.get(target.localId)!; @@ -1864,7 +1949,7 @@ export function emitSpeculativeWrite(emitter: LlEmitter, s: IrStmt, resume: stri let bails = false; emitter.declare("declare i1 @llvm.expect.i1(i1, i1)"); const bail = (condition: string): void => { - if (condition === "false") return; + if (condition === "false" || resume === undefined) return; bails = true; const expected = B.tmp(); B.line(`${expected} = call i1 @llvm.expect.i1(i1 ${condition}, i1 false)`); @@ -1893,7 +1978,7 @@ export function emitSpeculativeWrite(emitter: LlEmitter, s: IrStmt, resume: stri B.line(`${biased} = add i64 ${result}, ${Number.MAX_SAFE_INTEGER}`); B.line(`${outside} = icmp ugt i64 ${biased}, ${2 * Number.MAX_SAFE_INTEGER}`); bail(outside); - } else result = emitSpeculativeInteger(emitter, target.value, bail, reads); + } else result = emitSpeculativeInteger(emitter, target.value, bail, reads, resume === undefined); // Speculable values read only numbers past this point. if (emitter.frames.pop()!.length > 0) throw new InternalCompilerError("llvm emitter bug: speculative write owns a temporary"); @@ -1908,8 +1993,8 @@ export function emitSpeculativeWrite(emitter: LlEmitter, s: IrStmt, resume: stri const done = B.newLabel("spec.done"); B.br(done); B.startBlock(failed); - emitter.writeBackSpeculativeGlobals(deferred); - B.br(resume); + emitter.writeBackDeferred(deferred); + B.br(resume!); B.startBlock(done); } diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 1d745b981..adee90840 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -1022,6 +1022,8 @@ export class LlEmitter { * not a constructor (where they change), not async or a generator. */ finalFieldBorrows = false; integerRanges: IntegerRanges = new Map(); + /** The current function's bounded for-loop cursors (integer-ranges.ts). */ + integerLoopRanges: ReadonlyMap> = new Map(); /** Number arrays of the current function whose every present element is * an int32 (Int32[] contracts), or null when the module has none. */ int32Elements: ((arr: IrExpr) => boolean) | null = null; @@ -2653,7 +2655,8 @@ export class LlEmitter { return emitLoopCopies(this, loop, allowed, exit, emitCopy); } - writeBackSpeculativeGlobals(deferred: Set): void { + /** Store the double of each deferred shadow back to its binding. */ + writeBackDeferred(deferred: Set): void { const B = this.B; for (const id of deferred) { const integer = B.tmp(), @@ -2732,7 +2735,7 @@ export class LlEmitter { } return this.globalWriters.get(id) ?? new Set(); } - private emitSpeculativeWrite(s: IrStmt, resume: string): void { + private emitSpeculativeWrite(s: IrStmt, resume: string | undefined): void { return emitSpeculativeWrite(this, s, resume); } @@ -4378,7 +4381,7 @@ export class LlEmitter { this.B.startBlock(resume.label); } if (this.speculation?.writes.has(s)) { - this.emitSpeculativeWrite(s, this.speculation.resume.get(s)!); + this.emitSpeculativeWrite(s, this.speculation.resume.get(s)); return; } const previous = this.B.debugLocation; diff --git a/packages/compiler/src/backend/llvm/speculative-integers.test.ts b/packages/compiler/src/backend/llvm/speculative-integers.test.ts index ae9172298..3acfec0fb 100644 --- a/packages/compiler/src/backend/llvm/speculative-integers.test.ts +++ b/packages/compiler/src/backend/llvm/speculative-integers.test.ts @@ -2,6 +2,7 @@ import { expect, test } from "vitest"; import { F64, type IrExpr, type IrStmt } from "../../ir/ir.js"; import { isSpeculableValue, + planProvenIntegerLoop, planSpeculativeLoop, speculativeReads, } from "./speculative-integers.js"; @@ -194,3 +195,74 @@ test("loops that only count and compare keep one copy", () => { ]); expect(planSpeculativeLoop(scaled as never, all)?.locals.has("n")).toBe(true); }); + +test("bounded cursors with proven writes are shadowed in a proven integer loop", () => { + // for (i = 0; i < n; i++) { if (a < b) run = run + 1; else run = 0; + // if (run > longest) longest = run; } + const sum = bin("+", ref("run"), num(1)); + const step = assign("run", sum); + const reset = assign("run", num(0)); + const copy = assign("longest", ref("run")); + const forLoop = (body: IrStmt[]): IrStmt => ({ + kind: "for", + init: null, + cond: bin("<", ref("i"), ref("n")), + update: assign("i", bin("+", ref("i"), num(1))), + body, + loc, + }); + const branch = (then: IrStmt[], else_: IrStmt[] | null): IrStmt => ({ + kind: "if", + cond: bin("<", ref("a"), ref("b")), + then, + else_, + loc, + }); + const body = [branch([step], [reset]), branch([copy], null)]; + const range = { min: 0, max: 100 }; + const cursors = new Map([ + ["run", range], + ["longest", range], + ]); + const plan = planProvenIntegerLoop(forLoop(body) as never, cursors, all, () => false, all)!; + expect([...plan.locals]).toEqual([ + ["run", range], + ["longest", range], + ]); + expect(plan.writes).toEqual(new Set([step, reset, copy])); + // A binding the loop does not bound keeps its double. + const only = planProvenIntegerLoop( + forLoop(body) as never, + new Map([["run", range]]), + all, + () => false, + all, + )!; + expect([...only.locals.keys()]).toEqual(["run"]); + // An unproven sum leaves `run` unshadowed, and its copy reads `run` + // where nothing proves it integral. + const unproven = (e: IrExpr): boolean => e !== sum && e.kind !== "varRef"; + expect( + planProvenIntegerLoop(forLoop(body) as never, cursors, all, () => false, unproven), + ).toBeNull(); + // An expression-position write refuses the binding. + const inner: IrStmt = { + kind: "exprStmt", + expr: { kind: "incDec", op: "+", prefix: false, localId: "longest", type: F64, loc }, + loc, + }; + expect([ + ...planProvenIntegerLoop( + forLoop([...body, inner]) as never, + cursors, + all, + () => false, + all, + )!.locals.keys(), + ]).toEqual(["run"]); + // A labeled jump may leave past the write-back. + const jump: IrStmt = { kind: "break", label: "outer", loc }; + expect( + planProvenIntegerLoop(forLoop([...body, jump]) as never, cursors, all, () => false, all), + ).toBeNull(); +}); diff --git a/packages/compiler/src/backend/llvm/speculative-integers.ts b/packages/compiler/src/backend/llvm/speculative-integers.ts index cf1295fa2..a309a9785 100644 --- a/packages/compiler/src/backend/llvm/speculative-integers.ts +++ b/packages/compiler/src/backend/llvm/speculative-integers.ts @@ -3,7 +3,7 @@ import { everyExprChild, everyStmtChild } from "../../ir/traverse.js"; import { f64Lit } from "./common.js"; import type { LlValue, LlvmEmitterContext } from "./expr-context.js"; import { exactInteger, widenInteger } from "./integer-values.js"; -import { withinInt32 } from "../../ir/integer-ranges.js"; +import { type IntegerRange, withinInt32 } from "../../ir/integer-ranges.js"; /** Speculative integer loops. * @@ -410,6 +410,97 @@ export function planSpeculativeLoop( return { locals, writes: speculated, observerFree }; } +/** A proven integer loop. The bindings integer-ranges.ts bounds over a for + * loop (`loopRanges`) hold an integer on entry and at every write, so the + * loop keeps them in i64 shadow slots without a second copy or checks, and + * writes their doubles once, when it is left. */ +export interface ProvenLoopPlan { + /** Shadowed bindings and their loop ranges. */ + locals: Map; + /** Statement-level writes computed as integers. */ + writes: Set; +} + +/** Plan a proven integer loop. `cursors` are the loop's bounded bindings, + * `eligible` the locals a shadow may stand for, `integral` those the emitter + * already holds as integers and `proven` the expressions the range analysis + * proves integral. A shadowed binding's double is stale until the loop is + * left, so every exit has to pass the write-back: the caller plans only + * loops where an exception leaves the function, and a labeled jump (which + * may leave past the write-back) keeps the loop ordinary. */ +export function planProvenIntegerLoop( + loop: SpeculativeLoop, + cursors: ReadonlyMap, + eligible: (localId: string) => boolean, + integral: (localId: string) => boolean, + proven: (e: IrExpr) => boolean, +): ProvenLoopPlan | null { + if (cursors.size === 0) return null; + const writes = new Map(); + const refused = new Set(); + let rejected = false; + const expr = (e: IrExpr): boolean => { + if (e.kind === "assignExpr" || e.kind === "incDec") refused.add(e.localId); + return everyExprChild(e, expr, nested); + }; + const nested = (s: IrStmt): boolean => { + if (s.kind === "assign" || s.kind === "varDecl") refused.add(s.localId); + return everyStmtChild(s, expr, nested); + }; + const stmt = (s: IrStmt): boolean => { + if (rejected) return false; + if ( + !SIMPLE_STATEMENTS.has(s.kind) || + ((s.kind === "break" || s.kind === "continue") && s.label !== undefined) + ) { + rejected = true; + return false; + } + const target = s.kind === "varDecl" ? null : speculativeWriteTarget(s); + if (target) { + if (target.value !== null && cursors.has(target.localId)) { + const list = writes.get(target.localId) ?? []; + list.push(s); + writes.set(target.localId, list); + } else refused.add(target.localId); + return target.value === null || expr(target.value); + } + if (s.kind === "varDecl") refused.add(s.localId); + return everyStmtChild(s, expr, stmt); + }; + if (loop.cond) expr(loop.cond); + if (loop.kind === "for" && loop.update) nested(loop.update); + for (const s of loop.body) stmt(s); + if (rejected) return null; + const locals = new Map(); + for (const [id, list] of writes) { + if (refused.has(id) || !eligible(id)) continue; + const exact = list.every((s) => { + const value = speculativeWriteTarget(s)!.value!; + return isSpeculableValue(value, false) && proven(value); + }); + if (exact) locals.set(id, cursors.get(id)!); + } + // Every binding a shadowed write reads is shadowed, already an integer, + // or proven integral where it is read. + const grounded = (e: IrExpr): boolean => + e.kind === "varRef" + ? locals.has(e.localId) || integral(e.localId) || proven(e) + : everyExprChild(e, grounded, () => false); + for (let changed = true; changed;) { + changed = false; + for (const id of [...locals.keys()]) + if (!writes.get(id)!.every((s) => grounded(speculativeWriteTarget(s)!.value!))) { + locals.delete(id); + changed = true; + } + } + if (locals.size === 0) return null; + const planned = new Set(); + for (const id of locals.keys()) for (const s of writes.get(id)!) planned.add(s); + return { locals, writes: planned }; +} + const SAFE = Number.MAX_SAFE_INTEGER; /** The range every speculative shadow holds. */ @@ -501,11 +592,14 @@ export function emitSpeculativeInteger( e: IrExpr, bail: (condition: string) => void, reads: ReadonlyMap = new Map(), + integral = false, ): string { const B = host.B; // A value the range analysis already proves takes the ordinary integer - // emission, which needs no checks. - const proven = e.kind === "bin" ? host.integerRanges.get(e) : undefined; + // emission, which needs no checks. A proven integer loop's write + // (`integral`) is proven throughout and computes its operands as integers + // instead: the ordinary emission of a sum past int32 is a double. + const proven = e.kind === "bin" && !integral ? host.integerRanges.get(e) : undefined; if (proven && proven.min >= -SAFE && proven.max <= SAFE && speculativeReads(e).length === 0) return speculativeLeaf(host, host.emitExpr(e), e, bail); switch (e.kind) { @@ -533,8 +627,8 @@ export function emitSpeculativeInteger( } case "bin": { if (e.op !== "+" && e.op !== "-" && e.op !== "*" && e.op !== "%") break; - const a = emitSpeculativeInteger(host, e.left, bail, reads); - const b = emitSpeculativeInteger(host, e.right, bail, reads); + const a = emitSpeculativeInteger(host, e.left, bail, reads, integral); + const b = emitSpeculativeInteger(host, e.right, bail, reads, integral); const result = B.tmp(); if (e.op === "+" || e.op === "-") { // With one operand's range known, the other is checked against the @@ -590,7 +684,7 @@ export function emitSpeculativeInteger( } case "unary": { if (e.op !== "-") break; - const a = emitSpeculativeInteger(host, e.operand, bail, reads); + const a = emitSpeculativeInteger(host, e.operand, bail, reads, integral); const zero = B.tmp(), result = B.tmp(); // -0 is not an integer result. diff --git a/packages/compiler/src/ir/integer-loop-facts.ts b/packages/compiler/src/ir/integer-loop-facts.ts index 84fabf956..d207c4c12 100644 --- a/packages/compiler/src/ir/integer-loop-facts.ts +++ b/packages/compiler/src/ir/integer-loop-facts.ts @@ -36,6 +36,11 @@ function variableIncrement(id: string, value: IrExpr): { localId: string; sign: * filtering, breaks and continues can only reduce the budget. A nested * loop or opaque control region invalidates the bindings it writes. * + * A cursor may also be reset to an integer literal (`run = 0`) or set to + * another bounded binding (`longest = run`): every value it then holds is + * its entry value, a reset value or a value the source held, plus at most + * the increments of the whole loop. + * * The counter's step is a positive integer literal or a binding the loop * never writes whose entry fact is a positive integer range (the sieve's * `j += i`): every update then adds an exact integer in that range, so the @@ -68,6 +73,8 @@ export function boundedIntegerLoopFacts( refused = new Set(); const lower = new Map(), upper = new Map(); + const resets = new Map(), + copies = new Map>(); let opaque = 0; const write = (id: string, delta: number | null): void => { written.add(id); @@ -81,8 +88,29 @@ export function boundedIntegerLoopFacts( lower.set(id, lo); upper.set(id, hi); }; + const assignment = (id: string, value: IrExpr): void => { + const delta = increment(id, value); + if (opaque || delta !== null) write(id, delta); + else if ( + value.kind === "numLit" && + Number.isSafeInteger(value.value) && + !Object.is(value.value, -0) + ) { + written.add(id); + const range = resets.get(id); + resets.set(id, { + min: Math.min(range?.min ?? value.value, value.value), + max: Math.max(range?.max ?? value.value, value.value), + }); + } else if (value.kind === "varRef" && value.localId !== id) { + written.add(id); + const sources = copies.get(id) ?? new Set(); + sources.add(value.localId); + copies.set(id, sources); + } else write(id, null); + }; const expr = (e: IrExpr): boolean => { - if (e.kind === "assignExpr") write(e.localId, increment(e.localId, e.value)); + if (e.kind === "assignExpr") assignment(e.localId, e.value); if (e.kind === "incDec") write(e.localId, e.op === "+" ? 1 : -1); return everyExprChild(e, expr, stmt); }; @@ -95,7 +123,7 @@ export function boundedIntegerLoopFacts( s.kind === "tryCatch" || s.kind === "switch"; if (region) opaque++; - if (s.kind === "assign") write(s.localId, increment(s.localId, s.value)); + if (s.kind === "assign") assignment(s.localId, s.value); if (s.kind === "varDecl" || s.kind === "forOf") write(s.localId, null); const completed = everyStmtChild(s, expr, stmt); if (region) opaque--; @@ -147,15 +175,45 @@ export function boundedIntegerLoopFacts( // Header facts also cover the failing final condition and zero iterations. const maximum = Math.max(start.max, last + step.max); if (Number.isSafeInteger(maximum)) result.set(counter, { min: start.min, max: maximum }); - for (const [id, hi] of upper) { - const range = initial.get(id); - if (!range || refused.has(id) || id === counter) continue; - const decrement = iterations * lower.get(id)!; - const increment = iterations * hi; - if (!Number.isSafeInteger(decrement) || !Number.isSafeInteger(increment)) continue; - const min = range.min + decrement; - const max = range.max + increment; - if (Number.isSafeInteger(min) && Number.isSafeInteger(max)) result.set(id, { min, max }); + // Every value a copy reads lies in its source's range: a cursor bounded + // here, or a binding the loop never writes. + const source = (id: string): IntegerRange | undefined => + result.get(id) ?? (stable(id) ? initial.get(id) : undefined); + // Copies wait for their sources; a cycle of copies stays unbounded. + for (let changed = true; changed;) { + changed = false; + for (const id of written) { + const range = initial.get(id); + if (!range || refused.has(id) || id === counter || result.has(id)) continue; + let min = range.min, + max = range.max; + const reset = resets.get(id); + if (reset) { + min = Math.min(min, reset.min); + max = Math.max(max, reset.max); + } + let bounded = true; + const sources = copies.get(id); + if (sources) + for (const from of sources) { + const copied = source(from); + if (!copied) bounded = false; + else { + min = Math.min(min, copied.min); + max = Math.max(max, copied.max); + } + } + if (!bounded) continue; + const decrement = iterations * (lower.get(id) ?? 0); + const increment = iterations * (upper.get(id) ?? 0); + if (!Number.isSafeInteger(decrement) || !Number.isSafeInteger(increment)) continue; + min += decrement; + max += increment; + if (Number.isSafeInteger(min) && Number.isSafeInteger(max)) { + result.set(id, { min, max }); + changed = true; + } + } } return result; } diff --git a/packages/compiler/src/ir/integer-ranges.test.ts b/packages/compiler/src/ir/integer-ranges.test.ts index 8e1c97f66..6605e1d23 100644 --- a/packages/compiler/src/ir/integer-ranges.test.ts +++ b/packages/compiler/src/ir/integer-ranges.test.ts @@ -1,5 +1,6 @@ import { expect, test } from "vitest"; import { + analyzeIntegerRangeFacts, analyzeIntegerRanges, bitwiseIdentityOperand, bitwiseRange, @@ -1041,3 +1042,119 @@ test("bitwise identity operands follow ToInt32 and the shift-count mask", () => expect(identity("<<", num(0), x)).toBeNull(); expect(identity(">>>", num(0), x)).toBeNull(); }); + +test("a for loop bounds cursors it resets and copies, and their writes keep the loop range", () => { + // for (x = 0; x < 100; x++) { if (flag === 0) run = run + 1; else run = 0; + // if (run > longest) longest = run; changes = changes + 1; } + const sum = bin("+", ref("run"), num(1)), + copied = ref("run"), + counted = bin("+", ref("changes"), num(1)); + const write = (localId: string, value: IrExpr): IrStmt => ({ + loc, + kind: "assign", + localId, + value, + }); + const loop: IrStmt & { kind: "for" } = { + loc, + kind: "for", + init: { loc, kind: "varDecl", localId: "x", init: num(0) }, + cond: condition("<", ref(), num(100)), + update: assign(bin("+", ref(), num(1))), + body: [ + { + loc, + kind: "if", + cond: condition("===", ref("flag"), num(0)), + then: [write("run", sum)], + else_: [write("run", num(0))], + }, + { + loc, + kind: "if", + cond: condition(">", ref("run"), ref("longest")), + then: [write("longest", copied)], + else_: null, + }, + write("changes", counted), + ], + }; + const declare = (localId: string): IrStmt => ({ loc, kind: "varDecl", localId, init: num(0) }); + const f = fn([declare("run"), declare("longest"), declare("changes"), loop]); + f.locals.push( + ...["run", "longest", "changes", "flag"].map((id) => ({ + id, + name: id, + type: F64, + mutable: true, + })), + ); + const ranges = analyzeIntegerRanges(f); + // Each of the 100 passes adds at most one, so `run + 1` and + // `changes + 1` stay within the loop range though their operands reach it. + const range = { min: 0, max: 100 }; + expect(ranges.get(sum)).toEqual({ min: 1, max: 100 }); + expect(ranges.get(counted)).toEqual({ min: 1, max: 100 }); + expect(ranges.get(copied)).toEqual({ min: 1, max: 100 }); + expect(analyzeIntegerRangeFacts(f).loopRanges.get(loop)).toEqual( + new Map([ + ["run", range], + ["longest", range], + ["changes", range], + ]), + ); + for (const mutation of ["fraction", "cycle", "nested"] as const) { + const changed = structuredClone(f); + const body = (changed.body[3] as IrStmt & { kind: "for" }).body; + if (mutation === "fraction") body.push(write("run", num(0.5))); + // `run = longest` and `longest = run` bound neither. + if (mutation === "cycle") body.push(write("run", ref("longest"))); + if (mutation === "nested") + body.push({ + loc, + kind: "while", + cond: condition("<", ref("unknown"), num(1)), + body: [write("run", num(0))], + }); + const cursors = analyzeIntegerRangeFacts(changed).loopRanges.get(changed.body[3]!); + expect(cursors?.has("run"), mutation).toBe(false); + expect(cursors?.get("changes"), mutation).toEqual(range); + } +}); + +test("a cursor of a loop over a typed array keeps its writes exact up to 2^53 - 1", () => { + // for (x = 0; x < bytes.length; x++) changes = changes + 1; + const bytes: IrExpr = { + loc, + kind: "varRef", + localId: "bytes", + type: { kind: "bytes", elem: "u8" }, + }; + const length: IrExpr = { + loc, + kind: "bytesIntrinsic", + method: "length", + receiver: bytes, + args: [], + type: F64, + }; + const counted = bin("+", ref("changes"), num(1)); + const f = fn([ + { loc, kind: "varDecl", localId: "changes", init: num(0) }, + { + loc, + kind: "for", + init: { loc, kind: "varDecl", localId: "x", init: num(0) }, + cond: condition("<", ref(), length), + update: assign(bin("+", ref(), num(1))), + body: [{ loc, kind: "assign", localId: "changes", value: counted }], + }, + ]); + f.locals.push( + { id: "changes", name: "changes", type: F64, mutable: true }, + { id: "bytes", name: "bytes", type: { kind: "bytes", elem: "u8" }, mutable: false }, + ); + // The sum of the header range passes 2^53 - 1; the write takes the loop + // range instead. + expect(analyzeIntegerRanges(f).get(counted)).toEqual({ min: 0, max: Number.MAX_SAFE_INTEGER }); +}); diff --git a/packages/compiler/src/ir/integer-ranges.ts b/packages/compiler/src/ir/integer-ranges.ts index 5c365e2ca..8755bdac2 100644 --- a/packages/compiler/src/ir/integer-ranges.ts +++ b/packages/compiler/src/ir/integer-ranges.ts @@ -15,6 +15,13 @@ export interface IntegerRange { max: number; } export type IntegerRanges = ReadonlyMap; +/** Facts per expression, and per for loop the bindings its body writes only + * as bounded cursors (boundedIntegerLoopFacts): each holds an integer in its + * loop range on entry and at every write in the loop. */ +export interface IntegerRangeFacts { + ranges: IntegerRanges; + loopRanges: ReadonlyMap>; +} const SIGNED: IntegerRange = { min: -2147483648, max: 2147483647 }; const UNSIGNED: IntegerRange = { min: 0, max: 4294967295 }; /** Int32 and uint32 at once: the bound under which `%` (modeled only for @@ -227,8 +234,17 @@ export interface IntegerSlotFacts { * range those writes and its entry value span across the region. * Suspensions and captured bindings deliberately remain outside the proof. */ export function analyzeIntegerRanges(fn: IrFunction, slots: IntegerSlotFacts = {}): IntegerRanges { + return analyzeIntegerRangeFacts(fn, slots).ranges; +} + +/** analyzeIntegerRanges with the loop ranges of the function's for loops. */ +export function analyzeIntegerRangeFacts( + fn: IrFunction, + slots: IntegerSlotFacts = {}, +): IntegerRangeFacts { const ranges = new Map(); - if (fn.async || fn.generator) return ranges; + const loopRanges = new Map>(); + if (fn.async || fn.generator) return { ranges, loopRanges }; const captures = new Set( [...(fn.captures ?? []), ...(fn.classCaptures ?? [])].map((c) => c.localId), ); @@ -436,6 +452,26 @@ export function analyzeIntegerRanges(fn: IrFunction, slots: IntegerSlotFacts = { return result; } + // Loop ranges of the bindings the enclosing for loops bound + // (boundedIntegerLoopFacts) while their bodies are analyzed. + let counterRanges: ReadonlyMap = new Map(); + /** The value a write stores in a binding a for loop bounds. Every value + * the binding holds in the loop lies in its loop range, so the written + * value takes that range even when its operands' ranges add past it + * (`changes = changes + 1` with `changes` at the loop's limit). */ + function boundedWrite(id: string, value: IrExpr, facts: Facts): IntegerRange | null { + const bound = counterRanges.get(id); + const before = ranges.get(value); + const computed = expr(value, facts); + if (!bound) return computed; + const range = computed + ? { min: Math.max(computed.min, bound.min), max: Math.min(computed.max, bound.max) } + : bound; + if (range.min > range.max) return computed; + ranges.set(value, before === undefined ? range : before ? union(before, range) : null); + return range; + } + function remember(e: IrExpr, range: IntegerRange | null): IntegerRange | null { // Shared IR objects have to satisfy the proof at every occurrence. const previous = ranges.get(e); @@ -565,12 +601,17 @@ export function analyzeIntegerRanges(fn: IrFunction, slots: IntegerSlotFacts = { range = facts.get(e.localId) ?? boxedRange(e.localId); break; case "assignExpr": - range = expr(e.value, facts); + range = boundedWrite(e.localId, e.value, facts); set(facts, e.localId, range); break; case "incDec": { const before = facts.get(e.localId) ?? null; - const after = before ? integerArithmeticRange(e.op, before, { min: 1, max: 1 }) : null; + const bound = counterRanges.get(e.localId); + const sum = before ? integerArithmeticRange(e.op, before, { min: 1, max: 1 }) : null; + const after = + bound && sum + ? { min: Math.max(sum.min, bound.min), max: Math.min(sum.max, bound.max) } + : sum; set(facts, e.localId, after); range = e.prefix ? after : before; break; @@ -756,11 +797,11 @@ export function analyzeIntegerRanges(fn: IrFunction, slots: IntegerSlotFacts = { for (const s of stmts) { switch (s.kind) { case "varDecl": - case "assign": { - const value = s.kind === "varDecl" ? s.init : s.value; - set(facts, s.localId, value ? expr(value, facts) : null); + set(facts, s.localId, s.init ? expr(s.init, facts) : null); + break; + case "assign": + set(facts, s.localId, boundedWrite(s.localId, s.value, facts)); break; - } case "exprStmt": expr(s.expr, facts); break; @@ -831,7 +872,31 @@ export function analyzeIntegerRanges(fn: IrFunction, slots: IntegerSlotFacts = { if (s.cond) expr(s.cond, loop); const inside = new Map(loop); if (s.cond) refine(s.cond, true, inside); - body(s.body, inside); + // Only the body's writes are the ones the bound counted. + const outer = counterRanges; + const headWrites = s.cond ? writes(s.cond) : new Set(); + if (s.update) for (const id of writes(s.update)) headWrites.add(id); + const cursors = new Map(); + for (const [id, range] of bounded) + if (id !== counter?.id && !headWrites.has(id)) cursors.set(id, range); + // A loop analyzed again (inside another loop) keeps the bindings + // every analysis bounds, with the union of their ranges. + const previous = loopRanges.get(s); + if (!previous) loopRanges.set(s, cursors); + else + for (const [id, range] of previous) { + const other = cursors.get(id); + if (other) previous.set(id, union(range, other)); + else previous.delete(id); + } + const active = new Map(outer); + for (const [id, range] of cursors) active.set(id, range); + counterRanges = active; + try { + body(s.body, inside); + } finally { + counterRanges = outer; + } // Continue can bypass any body assignment. The update therefore // starts from header invariants rather than the body's exit facts, // plus what the condition proved about bindings the body never @@ -919,5 +984,5 @@ export function analyzeIntegerRanges(fn: IrFunction, slots: IntegerSlotFacts = { const entry: Facts = new Map(); if (slots.params) for (const [id, range] of slots.params) set(entry, id, range); body(fn.body, entry); - return ranges; + return { ranges, loopRanges }; } diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5686-loop-cursor-resets-and-copies.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5686-loop-cursor-resets-and-copies.ts.json new file mode 100644 index 000000000..8aef0174d --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5686-loop-cursor-resets-and-copies.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5686-loop-cursor-resets-and-copies.ts" + ], + "diags": [] +} diff --git a/tests/corpus/5686-loop-cursor-resets-and-copies.ts b/tests/corpus/5686-loop-cursor-resets-and-copies.ts new file mode 100644 index 000000000..ae5ff3589 --- /dev/null +++ b/tests/corpus/5686-loop-cursor-resets-and-copies.ts @@ -0,0 +1,157 @@ +// Loop cursors that a for loop resets, copies and counts (`run = 0`, +// `longest = run`, `lines++`): typed-array and array scans, exits through +// break, return and a caught exception, a labeled continue, loops that run +// zero times, cursors near 2^53, decrements and reads inside the loop. + +function longestRun(values: Int32Array): number { + let longest = 0; + let run = 0; + let changes = 0; + let previous = 0; + for (let i = 0; i < values.length; i++) { + const v = values[i]!; + if (v >= 0) { + run++; + if (run > longest) longest = run; + } else run = 0; + if (v < 0 !== previous < 0) changes++; + previous = v; + } + return longest * 1000 + changes; +} + +function countBytes(bytes: Uint8Array): string { + let lines = 0; + let words = 0; + let digits = 0; + let inWord = false; + for (let i = 0; i < bytes.length; i++) { + const b = bytes[i]!; + if (b === 10) lines++; + if (b === 10 || b === 32) inWord = false; + else if (!inWord) { + inWord = true; + words++; + } + if (b >= 48 && b <= 57) digits++; + } + return `${lines} ${words} ${digits}`; +} + +function firstLongRun(values: number[], limit: number): number { + let run = 0; + let at = -1; + for (let i = 0; i < values.length; i++) { + if (values[i]! > 0) run++; + else run = 0; + if (run === limit) { + at = i; + break; + } + } + return at * 100 + run; +} + +function returnInside(values: number[]): number { + let count = 0; + for (let i = 0; i < values.length; i++) { + if (values[i] === 0) return count; + count++; + } + return -count; +} + +function throwInside(values: number[]): number { + let count = 0; + for (let i = 0; i < values.length; i++) { + if (values[i]! < 0) throw new Error(`negative after ${count}`); + count++; + } + return count; +} + +function caughtInside(values: number[]): string { + let count = 0; + try { + for (let i = 0; i < values.length; i++) { + if (values[i]! < 0) throw new Error("negative"); + count++; + } + } catch (error) { + return `caught after ${count}`; + } + return `done ${count}`; +} + +function labeled(rows: number[][]): number { + let total = 0; + let skipped = 0; + outer: for (let r = 0; r < rows.length; r++) { + let run = 0; + for (let i = 0; i < rows[r]!.length; i++) { + if (rows[r]![i]! < 0) { + skipped++; + continue outer; + } + run++; + } + total += run; + } + return total * 100 + skipped; +} + +function nearLimit(): number[] { + let up = 9007199254740981; + let down = -9007199254740981; + let copy = 0; + for (let i = 0; i < 10; i++) { + up++; + down--; + copy = up; + } + let untouched = 9007199254740991; + for (let i = 0; i < 0; i++) untouched = untouched - 1; + return [up, down, copy, up - 9007199254740981, untouched]; +} + +function readsInside(n: number): number { + let count = 0; + let sum = 0; + for (let i = 0; i < n; i++) { + if (i % 3 === 0) count++; + sum += count * 2 + i; + } + return sum; +} + +function countDown(values: Float64Array): number { + let remaining = values.length; + let below = 0; + for (let i = 0; i < values.length; i++) { + remaining = remaining - 1; + if (values[i]! < 0.5) below = below + 2; + else below = below - 1; + } + return remaining * 1000 + below; +} + +const ints = new Int32Array(64); +for (let i = 0; i < ints.length; i++) ints[i] = ((i * 37) % 11) - 3; +console.log("longestRun", longestRun(ints), longestRun(new Int32Array(0))); +const text = new TextEncoder().encode("const value = 42;\nreturn x_y 7 plus 8\n\nend"); +console.log("countBytes", countBytes(text), countBytes(new Uint8Array(0))); +console.log("firstLongRun", firstLongRun([1, 2, 0, 3, 4, 5, 6, 0], 3), firstLongRun([1, 0, 1], 5)); +console.log("returnInside", returnInside([4, 5, 0, 6]), returnInside([1, 2, 3])); +try { + throwInside([1, 2, -1, 4]); +} catch (error) { + console.log("throwInside", (error as Error).message); +} +console.log("throwInside", throwInside([])); +console.log("caughtInside", caughtInside([3, 2, -1]), caughtInside([1, 2])); +console.log("labeled", labeled([[1, 2, 3], [4, -1, 5], [], [6, 7]])); +console.log("nearLimit", nearLimit().join(",")); +console.log("readsInside", readsInside(10), readsInside(0)); +const samples = new Float64Array(20); +for (let i = 0; i < samples.length; i++) samples[i] = (i * 0.37) % 1; +console.log("countDown", countDown(samples), countDown(new Float64Array(0))); diff --git a/tests/harness/corpus-costs/5686-loop-cursor-resets-and-copies.ts.json b/tests/harness/corpus-costs/5686-loop-cursor-resets-and-copies.ts.json new file mode 100644 index 000000000..738b5b70e --- /dev/null +++ b/tests/harness/corpus-costs/5686-loop-cursor-resets-and-copies.ts.json @@ -0,0 +1 @@ +{ "plain": 2650, "san": 2200 } From e71a1763bb80efc050dd077f8f5e16658eccdd7b Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Mon, 19 Oct 2026 23:17:06 -0700 Subject: [PATCH 10/38] Promisify untyped functions at run time util.promisify(fn) over a function typed any (an npm package method typed by its JavaScript, a bound method of an untyped value) was refused, because the promisified signature is only known at run time. A native wrapper now calls the function with the call's receiver, its arguments and a callback, and answers a promise: a truthy error rejects, otherwise the first value fulfills, the first settlement wins, and a synchronous throw rejects. util.promisify.custom functions are not consulted. The wrapper lives in its own runtime unit (scr_util_promisify.c), linked by the new utilPromisify feature, so other programs are unchanged. Co-Authored-By: Claude Opus 5.5 --- .../src/backend/executable-features.ts | 1 + packages/compiler/src/backend/library-pack.ts | 1 + .../compiler/src/backend/llvm/lib-shared.ts | 2 + .../src/backend/native-link-info-core.ts | 1 + .../compiler/src/backend/native/contracts.ts | 4 + .../compiler/src/backend/native/executable.ts | 2 + .../src/backend/runtime-pack-targets.test.ts | 1 + .../compiler/src/backend/runtime-pack.test.ts | 1 + packages/compiler/src/backend/warm-cache.ts | 1 + .../src/executable/executable-cache.test.ts | 1 + .../src/executable/executable-cache.ts | 2 + .../lowering/builtins/callable-aliases.ts | 8 +- .../lowering/builtins/promisify-callback.ts | 21 ++++- packages/compiler/src/index.ts | 1 + packages/compiler/src/ir/builtin-effects.ts | 1 + .../compiler/src/ir/builtin-signatures.ts | 3 + packages/compiler/src/ir/runtime-features.ts | 8 +- packages/compiler/src/native/cli.ts | 1 + packages/compiler/src/native/prepare.ts | 1 + .../compiler/src/native/sanitizer.test.ts | 1 + .../6452-promisify-untyped-functions.ts.json | 6 ++ .../runtime-pack-matrix.mjs | 2 + packages/runtime/src/scr_qs.h | 4 +- packages/runtime/src/scr_util_promisify.c | 77 +++++++++++++++++++ .../6452-promisify-untyped-functions.ts | 47 +++++++++++ .../self-hosting-native-toolchain.test.ts | 1 + .../self-hosting-runtime-features.test.ts | 1 + .../harness/self-hosting-runtime-pack.test.ts | 1 + 28 files changed, 196 insertions(+), 5 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6452-promisify-untyped-functions.ts.json create mode 100644 packages/runtime/src/scr_util_promisify.c create mode 100644 tests/corpus/6452-promisify-untyped-functions.ts diff --git a/packages/compiler/src/backend/executable-features.ts b/packages/compiler/src/backend/executable-features.ts index eeb6c6900..cba658b00 100644 --- a/packages/compiler/src/backend/executable-features.ts +++ b/packages/compiler/src/backend/executable-features.ts @@ -31,6 +31,7 @@ export function executableLinkFeatures(mod: IrModule): NativeLinkFeatures { bigint: features.bigint, searchParams: features.searchParams, qs: features.qs, + utilPromisify: features.utilPromisify, parseArgs: features.parseArgs, stream: features.stream, net: features.net, diff --git a/packages/compiler/src/backend/library-pack.ts b/packages/compiler/src/backend/library-pack.ts index 5070a47a8..4037f082a 100644 --- a/packages/compiler/src/backend/library-pack.ts +++ b/packages/compiler/src/backend/library-pack.ts @@ -42,6 +42,7 @@ export async function compilePackedLibrary( bigint: options.bigint ?? false, searchParams: options.searchParams ?? false, qs: false, + utilPromisify: false, parseArgs: false, stream: false, net: false, diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index e48d52160..5ccb29d67 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -82,6 +82,7 @@ export const LIB_FN_SYMS: Record = { "util.inherits": "scr_util_inherits", "util.deprecate": "scr_util_deprecate", "util.callbackify": "scr_util_callbackify", + "util.promisifyDyn": "scr_util_promisify", "util.debuglog": "scr_util_debuglog", "util.typeIs": "scr_dyn_util_type_is", "util.typeValue": "scr_dyn_util_type_value", @@ -1497,6 +1498,7 @@ export const USES_TIMERS_LIB_FNS = new Set([ // The wrappers schedule warnings and callbacks on the tick queue. "util.deprecate", "util.callbackify", + "util.promisifyDyn", "util.debuglog", "worker.new", "worker.parentPort", diff --git a/packages/compiler/src/backend/native-link-info-core.ts b/packages/compiler/src/backend/native-link-info-core.ts index 86146259d..091696986 100644 --- a/packages/compiler/src/backend/native-link-info-core.ts +++ b/packages/compiler/src/backend/native-link-info-core.ts @@ -46,6 +46,7 @@ export interface NativeLinkFeatures { bigint: boolean; searchParams: boolean; qs: boolean; + utilPromisify: boolean; parseArgs: boolean; stream: boolean; net: boolean; diff --git a/packages/compiler/src/backend/native/contracts.ts b/packages/compiler/src/backend/native/contracts.ts index 33b13960e..11096a08c 100644 --- a/packages/compiler/src/backend/native/contracts.ts +++ b/packages/compiler/src/backend/native/contracts.ts @@ -150,6 +150,10 @@ export interface CcOptions { * line (escape-only programs ride the always-linked component encoder * and never flip this). */ qs?: boolean; + /** The program promisifies an untyped function (util.promisify over + * `any`): compiles scr_util_promisify.c, which also references the + * dyn-async and dyn-invoke units. */ + utilPromisify?: boolean; /** The program uses native util parsing, comparison, or styling. Optional * utility units also require their symbol, inspection, and warning features. */ parseArgs?: boolean; diff --git a/packages/compiler/src/backend/native/executable.ts b/packages/compiler/src/backend/native/executable.ts index 5ebb4796e..18891798d 100644 --- a/packages/compiler/src/backend/native/executable.ts +++ b/packages/compiler/src/backend/native/executable.ts @@ -235,6 +235,8 @@ const RUNTIME_UNITS: readonly RuntimeUnit[] = [ (u) => u.opts.parseArgs && u.opts.inspect && u.opts.dynAsync && u.opts.dynInvoke && u.regex, "scr_util_callable.c", ), + // util.promisify over an untyped function: dyn promises and calls. + unit((u) => u.opts.utilPromisify && u.opts.dynAsync && u.opts.dynInvoke, "scr_util_promisify.c"), unit((u) => u.opts.stream, "scr_stream.c", "scr_fs_stream.c"), // The zlib stream classes reference both the zlib engine and the // stream unit. diff --git a/packages/compiler/src/backend/runtime-pack-targets.test.ts b/packages/compiler/src/backend/runtime-pack-targets.test.ts index fd57cb49d..2b083b59e 100644 --- a/packages/compiler/src/backend/runtime-pack-targets.test.ts +++ b/packages/compiler/src/backend/runtime-pack-targets.test.ts @@ -107,6 +107,7 @@ const features = { bigint: false, searchParams: false, qs: false, + utilPromisify: false, parseArgs: false, stream: false, net: false, diff --git a/packages/compiler/src/backend/runtime-pack.test.ts b/packages/compiler/src/backend/runtime-pack.test.ts index 61d1dba5e..80317ce2b 100644 --- a/packages/compiler/src/backend/runtime-pack.test.ts +++ b/packages/compiler/src/backend/runtime-pack.test.ts @@ -44,6 +44,7 @@ const BASE: NativeLinkFeatures = { bigint: false, searchParams: false, qs: false, + utilPromisify: false, parseArgs: false, stream: false, net: false, diff --git a/packages/compiler/src/backend/warm-cache.ts b/packages/compiler/src/backend/warm-cache.ts index 2cd111060..cdea8a52c 100644 --- a/packages/compiler/src/backend/warm-cache.ts +++ b/packages/compiler/src/backend/warm-cache.ts @@ -50,6 +50,7 @@ export async function warmNativeCaches( bigint: false, searchParams: false, qs: false, + utilPromisify: false, parseArgs: false, stream: false, net: false, diff --git a/packages/compiler/src/executable/executable-cache.test.ts b/packages/compiler/src/executable/executable-cache.test.ts index 1b42c3eca..2767bedac 100644 --- a/packages/compiler/src/executable/executable-cache.test.ts +++ b/packages/compiler/src/executable/executable-cache.test.ts @@ -45,6 +45,7 @@ const native: EarlyExecutableNativeFeatures = { bigint: false, searchParams: false, qs: false, + utilPromisify: false, parseArgs: false, stream: false, net: false, diff --git a/packages/compiler/src/executable/executable-cache.ts b/packages/compiler/src/executable/executable-cache.ts index 8c67c1006..e842520e6 100644 --- a/packages/compiler/src/executable/executable-cache.ts +++ b/packages/compiler/src/executable/executable-cache.ts @@ -71,6 +71,7 @@ export interface EarlyExecutableNativeFeatures { bigint: boolean; searchParams: boolean; qs: boolean; + utilPromisify: boolean; parseArgs: boolean; stream: boolean; net: boolean; @@ -185,6 +186,7 @@ const BOOLEAN_NATIVE_KEYS = [ "bigint", "searchParams", "qs", + "utilPromisify", "parseArgs", "stream", "net", diff --git a/packages/compiler/src/frontend/lowering/builtins/callable-aliases.ts b/packages/compiler/src/frontend/lowering/builtins/callable-aliases.ts index afb221d04..8724827e4 100644 --- a/packages/compiler/src/frontend/lowering/builtins/callable-aliases.ts +++ b/packages/compiler/src/frontend/lowering/builtins/callable-aliases.ts @@ -6,7 +6,7 @@ import { registerHttpClientFnBinding } from "../lower-server.js"; import { isPromisifiedZlibMember } from "./compression.js"; import { isPromisifiedCryptoMember } from "./crypto.js"; import { isPromisifiedDnsMember } from "../lower-dgram.js"; -import { callbackPromisifyShape } from "./promisify-callback.js"; +import { callbackPromisifyShape, untypedPromisifyArgument } from "./promisify-callback.js"; /** Compile-time util.promisify projections. The immutable result binding * owns no runtime slot: calls rewrite to the target's promise lowering, @@ -167,7 +167,11 @@ export function registerPromisifiedBuiltinDecl( : null; // A typed callback-last function value: the call lowers to a native // promisified function, and the binding holds it like any closure. - if (!projection && callbackPromisifyShape(lowerer, e, locOf(e)) !== null) return false; + if ( + !projection && + (callbackPromisifyShape(lowerer, e, locOf(e)) !== null || untypedPromisifyArgument(lowerer, e)) + ) + return false; if (!projection) { lowerer.noLowering( "util.promisify of this target", diff --git a/packages/compiler/src/frontend/lowering/builtins/promisify-callback.ts b/packages/compiler/src/frontend/lowering/builtins/promisify-callback.ts index 693079d3f..9e10000ee 100644 --- a/packages/compiler/src/frontend/lowering/builtins/promisify-callback.ts +++ b/packages/compiler/src/frontend/lowering/builtins/promisify-callback.ts @@ -3,6 +3,7 @@ import { type Lowerer } from "../lowerer.js"; import { varRef } from "../../../ir/build.js"; import { BOOL, + DYN, type IrExpr, type IrLocal, type IrParam, @@ -90,6 +91,14 @@ function callbackShape( return { fnT, resultT, args, callbackT, callbackParamT: callbackParamT!, inner, error }; } +/** Whether `promisify(fn)`'s one argument is typed `any`: the function is + * only known at run time, so the promisified function is built there. */ +export function untypedPromisifyArgument(lowerer: Lowerer, call: ts.CallExpression): boolean { + if (call.questionDotToken || call.arguments.nodes.length !== 1) return false; + const argNode = call.arguments.nodes[0]!; + return !ts.isSpreadElement(argNode) && (lowerer.typeOf(argNode).flags & ts.TypeFlags.Any) !== 0; +} + /** The callback shape of a `promisify(fn)` call, or null. */ export function callbackPromisifyShape( lowerer: Lowerer, @@ -131,7 +140,17 @@ export function lowerCallbackPromisify( loc: SrcLoc, ): IrExpr | null { const shape = callbackPromisifyShape(lowerer, call, loc); - if (shape === null) return null; + if (shape === null) { + // An untyped (`any`) function: Node's promisify at run time. + if (!untypedPromisifyArgument(lowerer, call)) return null; + return { + kind: "libCall", + fn: "util.promisifyDyn", + args: [lowerer.lowerExprExpecting(call.arguments.nodes[0]!, DYN)], + type: DYN, + loc, + }; + } const argNode = call.arguments.nodes[0]!; let fn = lowerer.lowerExpr(argNode); if (!typeEquals(fn.type, shape.fnT)) fn = lowerer.coerceToExpected(fn, shape.fnT); diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index 7d8d88889..c1ec92b11 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -543,6 +543,7 @@ async function compileExecutableNative( bigint: features.bigint, searchParams: features.searchParams, qs: features.qs, + utilPromisify: features.utilPromisify, parseArgs: features.parseArgs, stream: features.stream, net: features.net, diff --git a/packages/compiler/src/ir/builtin-effects.ts b/packages/compiler/src/ir/builtin-effects.ts index 7f709f099..1adac1d56 100644 --- a/packages/compiler/src/ir/builtin-effects.ts +++ b/packages/compiler/src/ir/builtin-effects.ts @@ -503,6 +503,7 @@ export const MAY_THROW_LIB_FNS: ReadonlySet = new Set([ "util.inherits", "util.deprecate", "util.callbackify", + "util.promisifyDyn", "util.debuglog", // The URI codecs throw the spec's URIError: decoding on bad hex/invalid // UTF-8 octets, encoding (and querystring.escape) on a lone surrogate. diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index 7f2748341..5de86f637 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -1004,6 +1004,9 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ "util.inherits": { argTypes: [DYN, DYN], result: VOID }, "util.deprecate": { argTypes: [DYN, DYN, DYN, DYN], result: DYN }, "util.callbackify": { argTypes: [DYN], result: DYN }, + /** util.promisify over an untyped function value: a native dyn function + * whose calls answer a dyn promise the appended callback settles. */ + "util.promisifyDyn": { argTypes: [DYN], result: DYN }, "util.debuglog": { argTypes: [DYN, DYN], result: DYN }, "util.typeIs": { argTypes: [DYN, STRING], result: BOOL }, "util.typeValue": { argTypes: [DYN, STRING], result: DYN }, diff --git a/packages/compiler/src/ir/runtime-features.ts b/packages/compiler/src/ir/runtime-features.ts index 4d59a6241..a5dcfe9a2 100644 --- a/packages/compiler/src/ir/runtime-features.ts +++ b/packages/compiler/src/ir/runtime-features.ts @@ -61,6 +61,7 @@ const DYN_ASYNC_LIB_FNS: ReadonlySet = new Set([ "util.inherits", "util.deprecate", "util.callbackify", + "util.promisifyDyn", "util.debuglog", "als.new", "als.get", @@ -493,7 +494,7 @@ const RUNTIME_FEATURE_RULES = { }), dynInvoke: rule({ initial: (mod) => mod.workers === true, - fns: [...DYN_INVOKE_LIB_FNS, ...UTIL_CALLABLE_LIB_FNS], + fns: [...DYN_INVOKE_LIB_FNS, ...UTIL_CALLABLE_LIB_FNS, "util.promisifyDyn"], expr: (node, _tables) => node.kind === "dynInvoke", }), /** The program has dynamic-tier jsval operations (js.* builtins, the @@ -610,6 +611,10 @@ const RUNTIME_FEATURE_RULES = { }), tls: rule({ prefixes: ["tls.", "https.", "http2."], types: ["secureCtx"] }), tlsCa: rule({ prefixes: ["tlsca."] }), + /** util.promisify over an untyped function (scr_util_promisify.c): its + * promises go through the dyn-async unit and its calls through the + * dyn-invoke unit. */ + utilPromisify: rule({ fns: ["util.promisifyDyn"] }), /** process.emitWarning is assigned (process.setEmitWarning). */ warningReplace: rule({ fns: ["process.setEmitWarning"] }), /** crypto.subtle importKey/sign/verify/digest and CryptoKey: @@ -662,6 +667,7 @@ function noRuntimeFeatures(): RuntimeFeatures { symbol: false, tls: false, tlsCa: false, + utilPromisify: false, warningReplace: false, webcrypto: false, zlib: false, diff --git a/packages/compiler/src/native/cli.ts b/packages/compiler/src/native/cli.ts index 7d00acbc8..7cb191622 100644 --- a/packages/compiler/src/native/cli.ts +++ b/packages/compiler/src/native/cli.ts @@ -89,6 +89,7 @@ async function main(): Promise { bigint: false, searchParams: false, qs: false, + utilPromisify: false, parseArgs: false, stream: false, net: false, diff --git a/packages/compiler/src/native/prepare.ts b/packages/compiler/src/native/prepare.ts index 54100a413..0335ace48 100644 --- a/packages/compiler/src/native/prepare.ts +++ b/packages/compiler/src/native/prepare.ts @@ -143,6 +143,7 @@ function validFeatures(value: unknown): value is NativeLinkFeatures { "bigint", "searchParams", "qs", + "utilPromisify", "parseArgs", "stream", "net", diff --git a/packages/compiler/src/native/sanitizer.test.ts b/packages/compiler/src/native/sanitizer.test.ts index 6ca84af6f..fd7572db6 100644 --- a/packages/compiler/src/native/sanitizer.test.ts +++ b/packages/compiler/src/native/sanitizer.test.ts @@ -95,6 +95,7 @@ test.skipIf(process.platform !== "linux" && process.platform !== "darwin")( bigint: false, searchParams: false, qs: false, + utilPromisify: false, parseArgs: false, stream: false, net: false, diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6452-promisify-untyped-functions.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6452-promisify-untyped-functions.ts.json new file mode 100644 index 000000000..bf040a1ec --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6452-promisify-untyped-functions.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6452-promisify-untyped-functions.ts" + ], + "diags": [] +} diff --git a/packages/runtime-pack-common/runtime-pack-matrix.mjs b/packages/runtime-pack-common/runtime-pack-matrix.mjs index 1e959eccc..be8a5632c 100644 --- a/packages/runtime-pack-common/runtime-pack-matrix.mjs +++ b/packages/runtime-pack-common/runtime-pack-matrix.mjs @@ -88,6 +88,8 @@ export const OPTIONAL_RUNTIME_SOURCES = [ ["scr_util_compare.c", all("parseArgs", "symbol")], ["scr_util_style.c", all("parseArgs", "inspect", "dynAsync")], ["scr_util_callable.c", all("parseArgs", "inspect", "dynAsync", "dynInvoke", "regex")], + // util.promisify over an untyped function creates dyn promises and calls. + ["scr_util_promisify.c", all("utilPromisify", "dynAsync", "dynInvoke")], ["scr_stream.c", "stream"], // The zlib stream classes reference both the zlib engine and the stream unit. ["scr_zlib_stream.c", all("zlib", "stream")], diff --git a/packages/runtime/src/scr_qs.h b/packages/runtime/src/scr_qs.h index 3d00dfae6..fc5ca5da5 100644 --- a/packages/runtime/src/scr_qs.h +++ b/packages/runtime/src/scr_qs.h @@ -58,12 +58,14 @@ ScrStr *scr_util_style_text(const struct ScrDyn *format, const struct ScrDyn *te * once the stream is a terminal (scr_util_style.c). */ bool scr_util_terminal_colors(void); /* util.inherits, util.deprecate, util.callbackify and util.debuglog - * (scr_util_callable.c). Arguments are borrowed; function results are +1 + * (scr_util_callable.c), and util.promisify over an untyped function + * (scr_util_promisify.c). Arguments are borrowed; function results are +1 * native dyn functions, or NULL with Node's TypeError pending. */ void scr_util_inherits(struct ScrDyn *ctor, struct ScrDyn *super_ctor); struct ScrDyn *scr_util_deprecate(struct ScrDyn *fn, struct ScrDyn *msg, struct ScrDyn *code, struct ScrDyn *options); struct ScrDyn *scr_util_callbackify(struct ScrDyn *original); +struct ScrDyn *scr_util_promisify(struct ScrDyn *original); struct ScrDyn *scr_util_debuglog(struct ScrDyn *section, struct ScrDyn *callback); /* Node/libuv system error utilities. Arguments are checked without coercion; diff --git a/packages/runtime/src/scr_util_promisify.c b/packages/runtime/src/scr_util_promisify.c new file mode 100644 index 000000000..7ca250a92 --- /dev/null +++ b/packages/runtime/src/scr_util_promisify.c @@ -0,0 +1,77 @@ +/* util.promisify over an untyped function. The wrapper is a native dyn + * function: each call creates a promise, calls the original with the + * call's receiver and arguments plus an error-first callback, and answers + * the promise. Typed functions keep the compiler's static wrapper; this + * unit links only into programs that promisify an `any` value. All + * arguments are borrowed. */ +#include "scr_runtime.h" + +#include + +static ScrBox *up_box(ScrDyn *value /* moves in */) { + ScrBox *box = scr_box_new_obj(scr_dyn_retain_v, scr_dyn_release_v, scr_dyn_trace_v); + scr_box_set_ref(box, value); + return box; +} + +static ScrDyn *up_cap(ScrClosure *closure, size_t index) { + return scr_box_get_ref(closure->caps[index]); /* +1 */ +} + +/* ── util.promisify over an untyped function ──────────────────────────── + * The wrapper calls original with the call's receiver and arguments plus a + * callback, and answers a promise the callback settles: a truthy first + * argument rejects with it, otherwise the second argument fulfills. A + * synchronous throw rejects, as Node's executor does. The first settlement + * wins. util.promisify.custom functions are not consulted. */ +static ScrDyn *up_callback(ScrClosure *closure, ScrDyn *const *args, size_t argc) { + ScrDyn *settles = up_cap(closure, 0); + ScrPromise *promise = scr_dyn_promise_of(settles); + if (promise && scr_promise_is_pending(promise)) { + ScrDyn *err = argc ? args[0] : scr_dyn_undefined(); + if (scr_dyn_truthy(err)) { + scr_dyn_throw(scr_dyn_retain(err)); + scr_promise_reject_pending(promise); + } else { + scr_promise_resolve_dyn(promise, scr_dyn_retain(argc > 1 ? args[1] : scr_dyn_undefined())); + } + } + scr_dyn_release(settles); + return scr_dyn_retain(scr_dyn_undefined()); +} + +static ScrDyn *up_call(ScrClosure *closure, ScrDyn *const *args, size_t argc) { + ScrDyn *original = up_cap(closure, 0); + ScrDyn *receiver = scr_dyn_this_get(); + ScrPromise *promise = scr_promise_new(); + ScrDyn *result = scr_dyn_new_promise(promise); + ScrClosure *settle = scr_closure_new(NULL, 1); + settle->caps[0] = up_box(scr_dyn_retain(result)); + ScrDyn *callback = scr_dyn_new_func(settle, up_callback, 2, "native:util.promisify", ""); + ScrDyn **full = malloc((argc + 1) * sizeof *full); + if (!full) scr_trap("out of memory"); + for (size_t i = 0; i < argc; i++) full[i] = args[i]; + full[argc] = callback; + scr_dyn_this_push_dyn(receiver); + ScrDyn *returned = scr_dyn_call(original, full, argc + 1, "function"); + scr_dyn_this_pop(); + free(full); + if (returned) scr_dyn_release(returned); + else if (scr_promise_is_pending(promise)) scr_promise_reject_pending(promise); + else scr_exc_clear(); + scr_dyn_release(callback); + scr_dyn_release(receiver); + scr_dyn_release(original); + scr_promise_release(promise); + return result; +} + +ScrDyn *scr_util_promisify(ScrDyn *original) { + if (!scr_dyn_is_callable(original)) { + scr_dyn_arg_type_fail("original", "of type function", original); + return NULL; + } + ScrClosure *closure = scr_closure_new(NULL, 1); + closure->caps[0] = up_box(scr_dyn_retain(original)); + return scr_dyn_new_func(closure, up_call, 0, "native:util.promisify", ""); +} diff --git a/tests/corpus/6452-promisify-untyped-functions.ts b/tests/corpus/6452-promisify-untyped-functions.ts new file mode 100644 index 000000000..aaae0ee3b --- /dev/null +++ b/tests/corpus/6452-promisify-untyped-functions.ts @@ -0,0 +1,47 @@ +// util.promisify over untyped (`any`) functions: the callback's error +// rejects, its first value fulfills, a synchronous throw rejects, only the +// first settlement counts, and the call's receiver reaches the function. +import { promisify } from "node:util"; + +const client: any = { + name: "db", + close(callback: any) { + setImmediate(() => callback()); + }, + fetch(key: any, callback: any) { + setImmediate(() => (key ? callback(null, `${this.name}:${key}`, "ignored") : callback(new Error("no key")))); + }, + twice(callback: any) { + callback(null, "first"); + callback(new Error("second")); + }, + boom(_callback: any) { + throw new TypeError("synchronous failure"); + }, +}; + +async function main(): Promise { + await promisify(client.close.bind(client))(); + console.log("closed"); + const fetchAsync = promisify(client.fetch.bind(client)); + console.log(await fetchAsync("a")); + try { + await fetchAsync(""); + } catch (error) { + console.log("rejected:", (error as Error).message); + } + console.log("twice:", await promisify(client.twice)()); + try { + await promisify(client.boom)(); + } catch (error) { + console.log("sync:", (error as Error).name, (error as Error).message); + } + const unbound = promisify(client.fetch); + console.log("receiver:", await unbound.call(client, "b")); + const order: string[] = []; + const pending = promisify(client.close.bind(client))().then(() => order.push("settled")); + order.push("returned"); + await pending; + console.log(order.join(", ")); +} +void main(); diff --git a/tests/harness/self-hosting-native-toolchain.test.ts b/tests/harness/self-hosting-native-toolchain.test.ts index fa71b5a08..8b399e288 100644 --- a/tests/harness/self-hosting-native-toolchain.test.ts +++ b/tests/harness/self-hosting-native-toolchain.test.ts @@ -30,6 +30,7 @@ const features: NativeLinkFeatures = { bigint: false, searchParams: false, qs: false, + utilPromisify: false, parseArgs: false, stream: false, net: false, diff --git a/tests/harness/self-hosting-runtime-features.test.ts b/tests/harness/self-hosting-runtime-features.test.ts index d2c2b589b..7e11316e5 100644 --- a/tests/harness/self-hosting-runtime-features.test.ts +++ b/tests/harness/self-hosting-runtime-features.test.ts @@ -50,6 +50,7 @@ const featureCalls: readonly [string, IrLibFn][] = [ ["bigint", "bigint.parse"], ["searchParams", "sp.new"], ["qs", "qs.parse"], + ["utilPromisify", "util.promisifyDyn"], ["parseArgs", "util.parseArgs"], ["parseArgs", "util.isDeepStrictEqual"], ["parseArgs", "util.styleText"], diff --git a/tests/harness/self-hosting-runtime-pack.test.ts b/tests/harness/self-hosting-runtime-pack.test.ts index c95d6d9a0..6cd004295 100644 --- a/tests/harness/self-hosting-runtime-pack.test.ts +++ b/tests/harness/self-hosting-runtime-pack.test.ts @@ -31,6 +31,7 @@ const base: NativeLinkFeatures = { bigint: false, searchParams: false, qs: false, + utilPromisify: false, parseArgs: false, stream: false, net: false, From c2ae71a6faf307e5661a4a6bcd73ac17f6ef657a Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 20 Oct 2026 02:07:00 -0500 Subject: [PATCH 11/38] Bound full-batch poller drains to 48 per poll phase A full batch of 64 events polled again without returning to the event loop. When synchronous socket callbacks kept 64 events ready on every drain, timers and setImmediate callbacks never ran. The poll phase now drains at most 48 full batches, the count libuv's uv__io_poll uses, and the budget carries across the reopen that lets microtasks run, so the remaining events wait for the next loop iteration. --- packages/runtime/src/scr_async.c | 13 +++++++++++-- packages/runtime/src/scr_net.c | 8 +++++--- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/packages/runtime/src/scr_async.c b/packages/runtime/src/scr_async.c index de48d6b61..2ab35e85d 100644 --- a/packages/runtime/src/scr_async.c +++ b/packages/runtime/src/scr_async.c @@ -4153,13 +4153,19 @@ bool scr_loop_has_ready(void) { return scr_ready_len > 0 || scr_nt_head != NULL; * drains the poller at most once between check phases. I/O that becomes * ready while this iteration's callbacks run (a loopback connect started * by one of them) waits for the next iteration, so immediates queued in - * the meantime fire first — Node's poll → check order. */ + * the meantime fire first — Node's poll → check order. A full batch polls + * again without waiting, at most SCR_LOOP_IO_BATCHES times per phase (the + * count libuv's uv__io_poll uses); later events wait for the next + * iteration, so timers and immediates still run while sockets stay busy. */ +enum { SCR_LOOP_IO_BATCHES = 48 }; static SCR_TL bool scr_loop_io_open = true; +static SCR_TL int scr_loop_io_batches = 0; bool scr_loop_io_take(void) { bool open = scr_loop_io_open; scr_loop_io_open = false; return open; } +bool scr_loop_io_more(void) { return ++scr_loop_io_batches < SCR_LOOP_IO_BATCHES; } void scr_loop_io_reopen(void) { scr_loop_io_open = true; } /* The process.nextTick queue to exhaustion, ticks queued meanwhile @@ -4668,7 +4674,10 @@ static bool scr_loop_run_pass(ScrPromise *top_level, bool first_checkpoint) { * had nothing to run). A check phase deferred for pending microtasks * keeps it closed, so the next iteration reaches the check phase * before polling again. */ - if (check_ran || scr_pending_immediates == 0) scr_loop_io_open = true; + if (check_ran || scr_pending_immediates == 0) { + scr_loop_io_open = true; + scr_loop_io_batches = 0; + } } return rejection_failed; } diff --git a/packages/runtime/src/scr_net.c b/packages/runtime/src/scr_net.c index 9684aebe9..f12840d91 100644 --- a/packages/runtime/src/scr_net.c +++ b/packages/runtime/src/scr_net.c @@ -4909,6 +4909,7 @@ static int scr_net_pollfd(void) { } bool scr_loop_io_take(void); +bool scr_loop_io_more(void); void scr_loop_io_reopen(void); static void scr_net_dispatch(void) { @@ -4917,8 +4918,9 @@ static void scr_net_dispatch(void) { return; } /* One poller drain per loop iteration (scr_loop_io_take); a full batch - * keeps draining, like libuv's poll phase. Deferred emits (the sweep) - * still run on every call. */ + * keeps draining within the phase's batch budget (scr_loop_io_more), like + * libuv's poll phase. Deferred emits (the sweep) still run on every + * call. */ bool may_drain = scr_loop_io_take(); for (;;) { scr_net_sweep(); @@ -4928,7 +4930,7 @@ static void scr_net_dispatch(void) { int n = 0; if (may_drain) { n = scrp_drain(scr_net_poller, evs, 64); - may_drain = n == 64; + may_drain = n == 64 && scr_loop_io_more(); } for (int i = 0; i < n; i++) { void *udata = evs[i].udata; From 835f6baed8edb28264aea948c736633c76e1fc1d Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Tue, 20 Oct 2026 03:29:12 -0700 Subject: [PATCH 12/38] Refuse untyped promisify originals with own properties Node's promisify copies the original's own property descriptors and prototype onto the wrapper and consults util.promisify.custom. The runtime wrapper now copies name and length and refuses, with SC2020 where util.promisify runs, an original with any other own property, instead of returning a wrapper that lacks them. The node24-util-adapters npm case now gets past its promisify scenarios and stops at an unrelated deferred Object.create refusal; its known state records that. Co-Authored-By: Claude Opus 5.5 --- .../limitations/builtin-function-values.mdx | 3 + packages/runtime/src/scr_util_promisify.c | 66 ++++++++++++++++++- tests/harness/npm.test.ts | 2 +- 3 files changed, 69 insertions(+), 2 deletions(-) diff --git a/docs/content/limitations/builtin-function-values.mdx b/docs/content/limitations/builtin-function-values.mdx index c08540c1a..b505b3715 100644 --- a/docs/content/limitations/builtin-function-values.mdx +++ b/docs/content/limitations/builtin-function-values.mdx @@ -8,6 +8,9 @@ Some builtins can be stored and passed as functions: Other immutable aliases of table-backed Node builtins support direct calls, but cannot be passed elsewhere. `util.promisify` supports compile-time conversions of `child_process.execFile`, `child_process.exec`, `setTimeout` and `setImmediate` (to the `timers/promises` functions, as in Node), `stream.finished` and `stream.pipeline` (to the `stream/promises` functions, as in Node), the `zlib` callback one-shots `deflate`, `inflate`, `deflateRaw`, `inflateRaw`, `gzip`, `gunzip` and `unzip` (with or without options), the `crypto` callback functions `randomBytes`, `pbkdf2`, `scrypt`, `hkdf`, `randomFill`, `randomInt` and `generateKey`, `dns.lookup` (settling with `{ address, family }`, or the address array with `all: true`) and `dns.lookupService` (settling with `{ hostname, service }`), and the `fs` callback functions that have an `fs/promises` counterpart (`access`, `appendFile`, `chmod`, `chown`, `copyFile`, `lchown`, `link`, `lstat`, `lutimes`, `mkdir`, `mkdtemp`, `readFile`, `readdir`, `readlink`, `realpath`, `rename`, `rm`, `rmdir`, `stat`, `statfs`, `symlink`, `truncate`, `unlink`, `utimes`, and `writeFile`); calls through the converted function lower as the `fs/promises` call. +`util.promisify` over an untyped (`any`) function returns a native wrapper that follows Node's callback rules and carries the original's `name` and `length`. +Such a function with other own properties, `util.promisify.custom` among them, throws `SC2020` where `util.promisify` runs, and the wrapper does not report Node's DEP0174 warning when the original returns a promise. + `util.styleText` supports standard formats and aliases, `#RGB` and `#RRGGBB` hex colors, arrays of formats, nested ANSI resets, and explicit color enabling with `{ validateStream: false }`. Native calls accept standard process streams; custom native streams and mutations to `util.inspect.colors` remain unsupported. diff --git a/packages/runtime/src/scr_util_promisify.c b/packages/runtime/src/scr_util_promisify.c index 7ca250a92..3acd4cb05 100644 --- a/packages/runtime/src/scr_util_promisify.c +++ b/packages/runtime/src/scr_util_promisify.c @@ -7,6 +7,7 @@ #include "scr_runtime.h" #include +#include static ScrBox *up_box(ScrDyn *value /* moves in */) { ScrBox *box = scr_box_new_obj(scr_dyn_retain_v, scr_dyn_release_v, scr_dyn_trace_v); @@ -66,12 +67,75 @@ static ScrDyn *up_call(ScrClosure *closure, ScrDyn *const *args, size_t argc) { return result; } +/* Node's promisify also copies the original's prototype and own property + * descriptors onto the wrapper, consults util.promisify.custom, and marks + * the wrapper with it. The plain case is a function whose only own + * properties are name, length and prototype: the wrapper takes the name + * and length descriptors. Other own properties (util.promisify.custom + * among them) refuse with SC2020 rather than answering differently. The + * wrapper's prototype is Function.prototype even when the original's was + * replaced. */ +static bool up_plain_original(ScrDyn *original) { + ScrDyn *keys = scr_dyn_own_keys(original); + if (!keys) return false; + bool plain = true; + double count = scr_dyn_arr_len(keys); + for (double i = 0; i < count && plain; i++) { + ScrDyn *key = scr_dyn_index_get(keys, i); + if (!key) { + plain = false; + break; + } + plain = key->kind == SCR_DYN_STR && + ((key->v.str->len == 4 && memcmp(key->v.str->data, "name", 4) == 0) || + (key->v.str->len == 6 && memcmp(key->v.str->data, "length", 6) == 0) || + (key->v.str->len == 9 && memcmp(key->v.str->data, "prototype", 9) == 0)); + scr_dyn_release(key); + } + scr_dyn_release(keys); + return plain; +} + +/* wrapper[key] = original[key] as a non-writable, non-enumerable, + * configurable data property, as getOwnPropertyDescriptors copies the + * name and length of an ordinary function. */ +static bool up_copy_descriptor(ScrDyn *wrapper, ScrDyn *original, const char *key) { + ScrStr *name = scr_str_new(key, strlen(key)); + ScrDyn *property = scr_dyn_new_str(name); + scr_str_release(name); + ScrDyn *value = scr_dyn_fn_get(original, key, strlen(key)); + ScrDyn *descriptor = scr_dyn_new_obj(); + scr_dyn_obj_set(descriptor, "value", 5, value ? value : scr_dyn_retain(scr_dyn_undefined())); + scr_dyn_obj_set(descriptor, "writable", 8, scr_dyn_new_bool(false)); + scr_dyn_obj_set(descriptor, "enumerable", 10, scr_dyn_new_bool(false)); + scr_dyn_obj_set(descriptor, "configurable", 12, scr_dyn_new_bool(true)); + ScrDyn *defined = scr_dyn_define_property(wrapper, property, descriptor); + scr_dyn_release(descriptor); + scr_dyn_release(property); + if (!defined) return false; + scr_dyn_release(defined); + return true; +} + ScrDyn *scr_util_promisify(ScrDyn *original) { if (!scr_dyn_is_callable(original)) { scr_dyn_arg_type_fail("original", "of type function", original); return NULL; } + if (!up_plain_original(original)) { + if (scr_exc_pending()) return NULL; + static const char message[] = + "util.promisify over an untyped function with own properties (such as util.promisify.custom) " + "is not supported yet"; + scr_throw_error_msg_code(SCR_ERR_ERROR, message, sizeof message - 1, "SC2020"); + return NULL; + } ScrClosure *closure = scr_closure_new(NULL, 1); closure->caps[0] = up_box(scr_dyn_retain(original)); - return scr_dyn_new_func(closure, up_call, 0, "native:util.promisify", ""); + ScrDyn *wrapper = scr_dyn_new_func(closure, up_call, 0, "native:util.promisify", ""); + if (!up_copy_descriptor(wrapper, original, "name") || !up_copy_descriptor(wrapper, original, "length")) { + scr_dyn_release(wrapper); + return NULL; + } + return wrapper; } diff --git a/tests/harness/npm.test.ts b/tests/harness/npm.test.ts index 54207f246..d15b38b6e 100644 --- a/tests/harness/npm.test.ts +++ b/tests/harness/npm.test.ts @@ -88,7 +88,7 @@ const KNOWN: Record = { "node24-timers-unref": { run: "SC2020" }, "node24-url-factories": { run: "SC2020" }, "node24-url-mutation": { run: "SC1090" }, - "node24-util-adapters": { run: "SC2020" }, + "node24-util-adapters": { run: "Object.create with this value is not supported yet" }, "node24-util-types": { run: "SC2009" }, "node24-web-stream-encoding": { build: "SC2013" }, "node24-web-stream-errors": { build: "SC2013" }, From 32f2b5d28b18cec28ae84918d0dd3cd13195e73b Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 20 Oct 2026 08:50:23 -0500 Subject: [PATCH 13/38] Run death-triggered full collections at the event loop's idle moments --- packages/runtime/src/scr_async.c | 3 ++ packages/runtime/src/scr_cycle.c | 41 ++++++++++++++++++++++++++++ packages/runtime/src/scr_rc.h | 6 ++++ packages/runtime/test/test_cycle.c | 44 ++++++++++++++++++++++++++++++ 4 files changed, 94 insertions(+) diff --git a/packages/runtime/src/scr_async.c b/packages/runtime/src/scr_async.c index 8185422af..3057c1d74 100644 --- a/packages/runtime/src/scr_async.c +++ b/packages/runtime/src/scr_async.c @@ -4408,6 +4408,8 @@ static bool scr_loop_run_pass(ScrPromise *top_level, bool first_checkpoint) { #ifdef SCR_WORKERS if (scr_workers_ready_fn && scr_workers_ready_fn()) due = now; #endif + /* An idle moment: a full pass the death rule deferred runs now. */ + if (due > now && scr_cyc_collect_idle()) now = scr_now_ms(); #if !defined(_WIN32) && !defined(__wasi__) if (due > now) scr_fiber_stacks_clear(); #endif @@ -4661,6 +4663,7 @@ void scr_loop_commonjs_entry(void) { scr_loop_cjs_entry = true; } bool scr_loop_run(ScrPromise *top_level) { /* The main module's body has returned: an enterWith it made ends here. */ scr_loop_started = true; + scr_cyc_note_event_loop(); if (scr_als_main_slot != NULL) scr_als_main_clear(); bool first_checkpoint = !scr_loop_cjs_entry; bool rejection_failed; diff --git a/packages/runtime/src/scr_cycle.c b/packages/runtime/src/scr_cycle.c index fd5d64c81..3922586f6 100644 --- a/packages/runtime/src/scr_cycle.c +++ b/packages/runtime/src/scr_cycle.c @@ -118,6 +118,10 @@ * growth baselines but not lower them: the heap regrowing * to where it was before such a pass reclaimed a dropped * structure is a rebuild, not growth. + * Once the thread's event loop runs, the pass waits for + * the loop's next idle moment (scr_cyc_collect_idle), or + * until the heap has grown the base fraction past where + * it was asked for. * mature the same two rules one level down: a growth fraction * measured from the last mature-or-full pass and scheduled * by everything mature passes free, and a backlog rule over @@ -225,6 +229,8 @@ typedef struct { size_t full_after; unsigned char death_shift; bool falling; + bool idle_points; + size_t death_due_live; bool keep_base; size_t trigger; size_t alloc_trigger; @@ -274,6 +280,8 @@ static _Thread_local ScrCycThread scr_cyc_thread = {.gen_limit = SCR_CYC_OLD}; #define scr_cyc_full_after (scr_cyc_thread.full_after) #define scr_cyc_death_shift (scr_cyc_thread.death_shift) #define scr_cyc_falling (scr_cyc_thread.falling) +#define scr_cyc_idle_points (scr_cyc_thread.idle_points) +#define scr_cyc_death_due_live (scr_cyc_thread.death_due_live) #define scr_cyc_keep_base (scr_cyc_thread.keep_base) #define scr_cyc_kinds (scr_cyc_thread.kinds) #define scr_cyc_kinds_cap (scr_cyc_thread.kinds_cap) @@ -569,6 +577,19 @@ static SCR_TL unsigned char scr_cyc_death_shift = 0; static SCR_TL bool scr_cyc_falling = false, scr_cyc_keep_base = false; #endif +/* A death pass waits for an idle moment of the thread's event loop + * (scr_cyc_collect_idle) once the loop runs (`idle_points`): the dropped + * structure it reclaims is not urgent, and a program that drops one in the + * middle of its work (a compiler starting a rebuild in watch mode) would + * otherwise walk its whole heap on the way. `death_due_live`: the live + * count when the death rule first asked for the pass that waits, 0 when + * none does. If the heap grows the base fraction past that before an idle + * moment comes, the pass runs where it is due. */ +#ifndef SCR_WORKERS +static SCR_TL bool scr_cyc_idle_points = false; +static SCR_TL size_t scr_cyc_death_due_live = 0; +#endif + static size_t scr_cyc_nursery_threshold(void) { static SCR_TL size_t cached = 0; if (cached == 0) { @@ -827,6 +848,10 @@ static SCR_CYC_SCHEDULING void scr_cyc_collect_due(void) { unsigned gen = SCR_CYC_OLD; bool by_growth = scr_cyc_grown(SCR_CYC_OLD); bool by_death = !by_growth && scr_cyc_died(); + if (by_death && scr_cyc_idle_points) { + if (!scr_cyc_death_due_live) scr_cyc_death_due_live = scr_cyc_live; + by_death = scr_cyc_live > scr_cyc_death_due_live + scr_cyc_death_due_live / SCR_CYC_GROWTH_DIV; + } if (!by_growth && !by_death && scr_roots[SCR_CYC_OLD].n < scr_cyc_backlog_threshold(SCR_CYC_OLD)) { gen = SCR_CYC_MATURE; by_growth = scr_cyc_grown(SCR_CYC_MATURE); @@ -864,6 +889,16 @@ void scr_cyc_collect_scheduled(void) { scr_cyc_collect_due(); } +void scr_cyc_note_event_loop(void) { scr_cyc_idle_points = true; } + +bool scr_cyc_collect_idle(void) { + if (!scr_cyc_death_due_live || scr_collecting) return false; + scr_cyc_note_fall(); + if (!scr_cyc_died()) return false; + scr_cyc_scheduled_pass(SCR_CYC_OLD, false, true); + return true; +} + /* ── level bypass ─────────────────────────────────────────────────────── */ /* A nursery pass costs about the same for every candidate it walks, and @@ -1874,6 +1909,7 @@ static SCR_CYC_COMPACT size_t scr_cyc_settle(unsigned gen_limit, bool doomed) { if (gen_limit >= SCR_CYC_OLD) { scr_cyc_scheduled_mature_age = 0; scr_cyc_fallen = 0; + scr_cyc_death_due_live = 0; scr_cyc_full_after = scr_cyc_live; } } @@ -2112,6 +2148,11 @@ void scr_cyc_forget_backlog_at(unsigned gen) { scr_cyc_backlog_shift[gen] = 0; scr_cyc_backlog_judged[gen] = false; } +/* Tests only: the thread's event loop stops counting as running. */ +void scr_cyc_forget_event_loop_at(void) { + scr_cyc_idle_points = false; + scr_cyc_death_due_live = 0; +} #endif size_t scr_cyc_collect_set(const ScrCycSet *set, unsigned gen) { diff --git a/packages/runtime/src/scr_rc.h b/packages/runtime/src/scr_rc.h index 8611741da..7dd8deadc 100644 --- a/packages/runtime/src/scr_rc.h +++ b/packages/runtime/src/scr_rc.h @@ -573,6 +573,12 @@ void scr_cyc_context_cleanup(void); * turns). Cheap: usually a nursery pass; a waiting mature backlog ages into * a bounded full pass so sparse roots cannot float forever. */ void scr_cyc_collect_scheduled(void); +/* The thread's event loop runs (scr_loop_run): from now on a full pass the + * death rule asks for waits for scr_cyc_collect_idle. */ +void scr_cyc_note_event_loop(void); +/* The event loop is about to wait with nothing ready: runs the full pass + * the death rule asked for, if one waits; returns whether it ran. */ +bool scr_cyc_collect_idle(void); /* Publication (@scriptc/threads publish; scr_cycle.c): makes everything a * root reaches immortal and immutable. Compiler-emitted per-type walkers diff --git a/packages/runtime/test/test_cycle.c b/packages/runtime/test/test_cycle.c index d4ea39a93..ab44ab08c 100644 --- a/packages/runtime/test/test_cycle.c +++ b/packages/runtime/test/test_cycle.c @@ -1311,6 +1311,49 @@ static size_t grow_until_watched(Node **keep, size_t limit, size_t watched) { return n; } +void scr_cyc_forget_event_loop_at(void); + +/* Once the thread's event loop runs, the full pass a fall asks for waits + * for the loop's next idle moment, unless the heap grows another quarter + * first. */ +static void check_death_waits_for_idle(void) { + enum { FLOOR = 4096, RING = 64, LEAVES = 1024 }; + static ScrCycType watched_type = { &node_trace, &watched_free, 0 }; + if (configured_growth_cap() < 4) return; /* growth alone is due about as soon */ + size_t limit = 2 * FLOOR + 2 * configured_nursery_threshold() + 2; + size_t before_due = FLOOR + FLOOR / 4 + 2 * configured_nursery_threshold() + 2; + Node **leaves = calloc(LEAVES, sizeof(*leaves)); + Node **keep = calloc(limit, sizeof(*keep)); + if (!leaves || !keep) scr_trap("cycle test: out of memory\n"); + for (int i = 0; i < 4; i++) grow_dead_cycle_until_full_pass(0, false); + scr_cyc_note_event_loop(); + for (int round = 0; round < 2; round++) { + Node *ring = make_ring_of(RING, &watched_type); + age_to_old(ring); + for (size_t i = 0; i < LEAVES; i++) leaves[i] = make_leaf(); + scr_collect_cycles(); + watched_freed = 0; + drop_leaves(leaves, LEAVES); + scr_cyc_collect_scheduled(); + release_live(ring); + size_t grown = grow_until_watched(keep, before_due, RING); + check(watched_freed == 0, "a fall brought a full pass while the event loop was busy"); + if (round == 0) { + check(scr_cyc_collect_idle(), "the idle moment did not run the waiting full pass"); + check(watched_freed == RING, "the waiting full pass did not reclaim the dead old ring"); + } else { + grown += grow_until_watched(keep + grown, limit - grown, RING); + check(watched_freed == RING, "the waiting full pass did not run once the heap grew another quarter"); + check(!scr_cyc_collect_idle(), "a full pass ran at the idle moment after the due one had run"); + } + drop_leaves(keep, grown); + scr_collect_cycles(); + } + scr_cyc_forget_event_loop_at(); + free(leaves); + free(keep); +} + static void check_death_brings_full_pass(void) { enum { FLOOR = 4096, RING = 64, LEAVES = 1024 }; static ScrCycType watched_type = { &node_trace, &watched_free, 0 }; @@ -1384,6 +1427,7 @@ int main(void) { check_growth_schedule_follows_yield(); check_growth_cap_bounds_floating_garbage(); check_mixed_dead_structure_narrows(); + check_death_waits_for_idle(); check_death_brings_full_pass(); check_nursery_bypass(); check_mature_bypass(); From 4470bab3082ed93c8fbbab836d1c6faec493d126 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Tue, 20 Oct 2026 07:58:16 -0700 Subject: [PATCH 14/38] Give the undici shim's Dispatcher an explicit constructor The implicit constructor forwarded its arguments to EventEmitter's options-taking constructor, which compiles to a deferred runtime fence. Dispatcher takes no options, so it calls super() directly. Co-Authored-By: Claude Opus 5.5 --- packages/compiler/shims/undici/core/dispatcher.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/compiler/shims/undici/core/dispatcher.ts b/packages/compiler/shims/undici/core/dispatcher.ts index cd195b5da..d9d35c33f 100644 --- a/packages/compiler/shims/undici/core/dispatcher.ts +++ b/packages/compiler/shims/undici/core/dispatcher.ts @@ -38,6 +38,12 @@ export interface ResponseData { const defaultProfile: Profile = { major: 7 }; export class Dispatcher extends EventEmitter { + // An explicit constructor: the implicit one forwards its arguments to + // EventEmitter's options-taking constructor, which stays a runtime fence. + constructor() { + super(); + } + dispatch(_options: DispatchOptions, _handler: DispatchHandlers): boolean { throw new Error("not implemented"); } From 8aca6d653db32d942890fa8cc66b8db513662671 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 20 Oct 2026 10:48:36 -0500 Subject: [PATCH 15/38] Abort the request of a redirect that fetch follows or rejects A redirect fetch followed or rejected was marked to drain: its body was read and dropped until the server ended it, and the abort listener was already removed. A body that never ends kept the connection busy after cancellation and stalled a pool of one connection. Following or rejecting a redirect now aborts its request, so the connection closes. undici 7 leaves that connection open with its body paused, which stalls the same pool; the follow-up request here gets a new connection. --- packages/compiler/shims/undici/core/fetch.ts | 9 +++++++-- .../compiler/test/ts7/resolver-parity.test.ts | 20 ++++++++++--------- .../package-shims/undici-hostile-peer.mjs | 15 ++++++++++++++ .../fixtures/package-shims/undici-redirect.ts | 19 ++++++++++++++++++ tests/harness/package-shims.test.ts | 17 +++++++++++++++- 5 files changed, 68 insertions(+), 12 deletions(-) create mode 100644 tests/fixtures/package-shims/undici-redirect.ts diff --git a/packages/compiler/shims/undici/core/fetch.ts b/packages/compiler/shims/undici/core/fetch.ts index 54a0ce0d5..ebca55d71 100644 --- a/packages/compiler/shims/undici/core/fetch.ts +++ b/packages/compiler/shims/undici/core/fetch.ts @@ -117,8 +117,9 @@ interface RawResponse { status: number; statusText: string; headers: string[]; - /** null when the response has no body to expose (null-body status, HEAD, - * or a redirect fetch follows); its bytes are then read and dropped. */ + /** null when the response has no body to expose: a null-body status or + * HEAD (its bytes, if any, are read and dropped), or a redirect fetch + * follows or rejects (the request is aborted, closing its connection). */ body: ReadableStream | null; } @@ -520,6 +521,10 @@ function dispatchOnce( dump = true; stopListening(); resolve({ status: statusCode, statusText: text, headers: rawHeaders, body: null }); + // A redirect that is followed or rejected never reads its body: + // abort the request so the connection is released instead of + // draining a body that may never end. + if (follows) abortRequest(new DOMException("The operation was aborted.", "AbortError")); return true; } const codings = diff --git a/packages/compiler/test/ts7/resolver-parity.test.ts b/packages/compiler/test/ts7/resolver-parity.test.ts index b3190ae9f..b5d5875e7 100644 --- a/packages/compiler/test/ts7/resolver-parity.test.ts +++ b/packages/compiler/test/ts7/resolver-parity.test.ts @@ -147,15 +147,17 @@ test("relative and bare specifiers across the whole test tree resolve identicall // The deliberate package-shim delta: a package scriptc implements // itself (package-shims.ts) answers its shim module here, where // 5.9.3 answers the installed package as an external import; the - // installed package's answer is still compared below. - const shimmed = packageShimFor(file, spec) !== null; - const theirsProj = shimmed - ? reference && reference.isExternalLibraryImport - ? packageShimFor(file, spec) - : null - : reference && !reference.isExternalLibraryImport - ? reference.resolvedFileName - : null; + // installed package's answer is still compared below. Resolved + // answers use forward slashes on every platform, as 5.9.3's do. + const shim = packageShimFor(file, spec); + const theirsProj = + shim !== null + ? reference && reference.isExternalLibraryImport + ? shim.replaceAll("\\", "/") + : null + : reference && !reference.isExternalLibraryImport + ? reference.resolvedFileName + : null; if (oursProj !== theirsProj) { failures.push(`${file} '${spec}' (project): ours=${oursProj} ts5=${theirsProj}`); } diff --git a/tests/fixtures/package-shims/undici-hostile-peer.mjs b/tests/fixtures/package-shims/undici-hostile-peer.mjs index ce6c5881c..b55e3226f 100644 --- a/tests/fixtures/package-shims/undici-hostile-peer.mjs +++ b/tests/fixtures/package-shims/undici-hostile-peer.mjs @@ -3,6 +3,9 @@ // /zeros a chunk size with 64 MiB of leading zeros, // /big a 64 MiB body, as fast as the socket takes it, // /bomb 64 MiB of zeros, gzipped, +// /redirect-endless a 302 to /redirect-closed whose body never ends, +// /redirect-closed "closed" once the last /redirect-endless connection +// closed (waiting up to a second), else "open", // and exits when its stdin closes. import * as net from "node:net"; import * as zlib from "node:zlib"; @@ -24,6 +27,8 @@ function stream(s, head, piece, tail) { pump(); } +let redirectClosed = Promise.resolve(); + const server = net.createServer((s) => { s.on("error", () => {}); let head = ""; @@ -39,6 +44,16 @@ const server = net.createServer((s) => { stream(s, chunked, Buffer.alloc(64 * 1024, 48), "5\r\nhello\r\n0\r\n\r\n"); } else if (path === "/big") { stream(s, `HTTP/1.1 200 OK\r\nContent-Length: ${size}\r\n\r\n`, Buffer.alloc(64 * 1024, 97), ""); + } else if (path === "/redirect-endless") { + redirectClosed = new Promise((resolve) => s.on("close", resolve)); + s.write("HTTP/1.1 302 Found\r\nLocation: /redirect-closed\r\nTransfer-Encoding: chunked\r\n\r\n"); + const timer = setInterval(() => s.write("4\r\ndata\r\n"), 5); + s.on("close", () => clearInterval(timer)); + } else if (path === "/redirect-closed") { + const timeout = new Promise((resolve) => setTimeout(() => resolve("open"), 1000)); + void Promise.race([redirectClosed.then(() => "closed"), timeout]).then((state) => { + s.write(`HTTP/1.1 200 OK\r\nContent-Length: ${state.length}\r\n\r\n${state}`); + }); } else { s.write(`HTTP/1.1 200 OK\r\nContent-Encoding: gzip\r\nContent-Length: ${bomb.length}\r\n\r\n`); s.write(bomb); diff --git a/tests/fixtures/package-shims/undici-redirect.ts b/tests/fixtures/package-shims/undici-redirect.ts new file mode 100644 index 000000000..de9700fcf --- /dev/null +++ b/tests/fixtures/package-shims/undici-redirect.ts @@ -0,0 +1,19 @@ +// Follows and rejects redirects whose bodies never end +// (undici-hostile-peer.mjs, port in argv[2]) through a pool of one +// connection. The discarded redirect's request is aborted, so its +// connection closes, the follow-up request gets a connection, and the peer +// reports the redirect's connection closed. +import { Agent, fetch } from "undici"; + +const origin = `http://127.0.0.1:${process.argv[2]}`; +const dispatcher = new Agent({ connections: 1 }); +const followed = await fetch(`${origin}/redirect-endless`, { dispatcher }); +console.log("follow", followed.status, await followed.text()); +try { + await fetch(`${origin}/redirect-endless`, { dispatcher, redirect: "error" }); +} catch (err) { + console.log("error mode", (err as Error).name, (err as Error).message); +} +const after = await fetch(`${origin}/redirect-closed`, { dispatcher }); +console.log("after error mode", await after.text()); +await dispatcher.close(); diff --git a/tests/harness/package-shims.test.ts b/tests/harness/package-shims.test.ts index 0be52d2df..447052b81 100644 --- a/tests/harness/package-shims.test.ts +++ b/tests/harness/package-shims.test.ts @@ -10,7 +10,10 @@ * - the undici shims keep memory bounded against a peer that sends endless * chunk-size lines and large or compressed bodies, and the 5.x profile * rejects a Host value with CR or LF as undici 5.29 does (the vendored - * 5.12 writes it into the request). + * 5.12 writes it into the request); + * - fetch aborts a redirect it follows or rejects, so a redirect body that + * never ends does not keep its connection busy (undici 7 leaves that + * connection open with its body paused). */ import { execFile, spawn } from "node:child_process"; import { copyFileSync, cpSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; @@ -164,6 +167,18 @@ describe("package shims", () => { }, 300_000); } + test("undici7: fetch closes the connection of a redirect it follows or rejects", async () => { + const { dir, binary } = await buildInCase("undici7", "undici-redirect.ts"); + expect(await runAgainstPeer(binary, dir)).toBe( + [ + "follow 200 closed", + "error mode TypeError fetch failed", + "after error mode closed", + "", + ].join("\n"), + ); + }, 180_000); + test("undici5: a Host value with CR or LF throws before the request is written", async () => { const { dir, binary } = await buildInCase("undici5", "undici5-host.ts"); // undici 5.29.0's output. From 33354cc4817fcad1726937085babbee55ab80722 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 20 Oct 2026 10:50:19 -0500 Subject: [PATCH 16/38] Abort the request of a redirect that fetch follows or rejects A redirect fetch followed or rejected was marked to drain: its body was read and dropped until the server ended it, and the abort listener was already removed. A body that never ends kept the connection busy after cancellation and stalled a pool of one connection. Following or rejecting a redirect now aborts its request, so the connection closes. undici 7 leaves that connection open with its body paused, which stalls the same pool; the follow-up request here gets a new connection. --- packages/compiler/shims/undici/core/fetch.ts | 9 +++++++-- .../package-shims/undici-hostile-peer.mjs | 15 +++++++++++++++ .../fixtures/package-shims/undici-redirect.ts | 19 +++++++++++++++++++ tests/harness/package-shims.test.ts | 17 ++++++++++++++++- 4 files changed, 57 insertions(+), 3 deletions(-) create mode 100644 tests/fixtures/package-shims/undici-redirect.ts diff --git a/packages/compiler/shims/undici/core/fetch.ts b/packages/compiler/shims/undici/core/fetch.ts index 54a0ce0d5..ebca55d71 100644 --- a/packages/compiler/shims/undici/core/fetch.ts +++ b/packages/compiler/shims/undici/core/fetch.ts @@ -117,8 +117,9 @@ interface RawResponse { status: number; statusText: string; headers: string[]; - /** null when the response has no body to expose (null-body status, HEAD, - * or a redirect fetch follows); its bytes are then read and dropped. */ + /** null when the response has no body to expose: a null-body status or + * HEAD (its bytes, if any, are read and dropped), or a redirect fetch + * follows or rejects (the request is aborted, closing its connection). */ body: ReadableStream | null; } @@ -520,6 +521,10 @@ function dispatchOnce( dump = true; stopListening(); resolve({ status: statusCode, statusText: text, headers: rawHeaders, body: null }); + // A redirect that is followed or rejected never reads its body: + // abort the request so the connection is released instead of + // draining a body that may never end. + if (follows) abortRequest(new DOMException("The operation was aborted.", "AbortError")); return true; } const codings = diff --git a/tests/fixtures/package-shims/undici-hostile-peer.mjs b/tests/fixtures/package-shims/undici-hostile-peer.mjs index ce6c5881c..b55e3226f 100644 --- a/tests/fixtures/package-shims/undici-hostile-peer.mjs +++ b/tests/fixtures/package-shims/undici-hostile-peer.mjs @@ -3,6 +3,9 @@ // /zeros a chunk size with 64 MiB of leading zeros, // /big a 64 MiB body, as fast as the socket takes it, // /bomb 64 MiB of zeros, gzipped, +// /redirect-endless a 302 to /redirect-closed whose body never ends, +// /redirect-closed "closed" once the last /redirect-endless connection +// closed (waiting up to a second), else "open", // and exits when its stdin closes. import * as net from "node:net"; import * as zlib from "node:zlib"; @@ -24,6 +27,8 @@ function stream(s, head, piece, tail) { pump(); } +let redirectClosed = Promise.resolve(); + const server = net.createServer((s) => { s.on("error", () => {}); let head = ""; @@ -39,6 +44,16 @@ const server = net.createServer((s) => { stream(s, chunked, Buffer.alloc(64 * 1024, 48), "5\r\nhello\r\n0\r\n\r\n"); } else if (path === "/big") { stream(s, `HTTP/1.1 200 OK\r\nContent-Length: ${size}\r\n\r\n`, Buffer.alloc(64 * 1024, 97), ""); + } else if (path === "/redirect-endless") { + redirectClosed = new Promise((resolve) => s.on("close", resolve)); + s.write("HTTP/1.1 302 Found\r\nLocation: /redirect-closed\r\nTransfer-Encoding: chunked\r\n\r\n"); + const timer = setInterval(() => s.write("4\r\ndata\r\n"), 5); + s.on("close", () => clearInterval(timer)); + } else if (path === "/redirect-closed") { + const timeout = new Promise((resolve) => setTimeout(() => resolve("open"), 1000)); + void Promise.race([redirectClosed.then(() => "closed"), timeout]).then((state) => { + s.write(`HTTP/1.1 200 OK\r\nContent-Length: ${state.length}\r\n\r\n${state}`); + }); } else { s.write(`HTTP/1.1 200 OK\r\nContent-Encoding: gzip\r\nContent-Length: ${bomb.length}\r\n\r\n`); s.write(bomb); diff --git a/tests/fixtures/package-shims/undici-redirect.ts b/tests/fixtures/package-shims/undici-redirect.ts new file mode 100644 index 000000000..de9700fcf --- /dev/null +++ b/tests/fixtures/package-shims/undici-redirect.ts @@ -0,0 +1,19 @@ +// Follows and rejects redirects whose bodies never end +// (undici-hostile-peer.mjs, port in argv[2]) through a pool of one +// connection. The discarded redirect's request is aborted, so its +// connection closes, the follow-up request gets a connection, and the peer +// reports the redirect's connection closed. +import { Agent, fetch } from "undici"; + +const origin = `http://127.0.0.1:${process.argv[2]}`; +const dispatcher = new Agent({ connections: 1 }); +const followed = await fetch(`${origin}/redirect-endless`, { dispatcher }); +console.log("follow", followed.status, await followed.text()); +try { + await fetch(`${origin}/redirect-endless`, { dispatcher, redirect: "error" }); +} catch (err) { + console.log("error mode", (err as Error).name, (err as Error).message); +} +const after = await fetch(`${origin}/redirect-closed`, { dispatcher }); +console.log("after error mode", await after.text()); +await dispatcher.close(); diff --git a/tests/harness/package-shims.test.ts b/tests/harness/package-shims.test.ts index 0be52d2df..447052b81 100644 --- a/tests/harness/package-shims.test.ts +++ b/tests/harness/package-shims.test.ts @@ -10,7 +10,10 @@ * - the undici shims keep memory bounded against a peer that sends endless * chunk-size lines and large or compressed bodies, and the 5.x profile * rejects a Host value with CR or LF as undici 5.29 does (the vendored - * 5.12 writes it into the request). + * 5.12 writes it into the request); + * - fetch aborts a redirect it follows or rejects, so a redirect body that + * never ends does not keep its connection busy (undici 7 leaves that + * connection open with its body paused). */ import { execFile, spawn } from "node:child_process"; import { copyFileSync, cpSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; @@ -164,6 +167,18 @@ describe("package shims", () => { }, 300_000); } + test("undici7: fetch closes the connection of a redirect it follows or rejects", async () => { + const { dir, binary } = await buildInCase("undici7", "undici-redirect.ts"); + expect(await runAgainstPeer(binary, dir)).toBe( + [ + "follow 200 closed", + "error mode TypeError fetch failed", + "after error mode closed", + "", + ].join("\n"), + ); + }, 180_000); + test("undici5: a Host value with CR or LF throws before the request is written", async () => { const { dir, binary } = await buildInCase("undici5", "undici5-host.ts"); // undici 5.29.0's output. From fec779abbb14e96d28cefcec21a4cd9cfdbc813e Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Tue, 20 Oct 2026 11:13:09 -0700 Subject: [PATCH 17/38] Use the events module and EventEmitter as untyped constructor values In JavaScript, `ReleaseEmitter = EventEmitter` over `const EventEmitter = require("events")` was refused with SC1090 ("a runtime-provided class value as an untyped value"), and `class Channel extends require("events")` failed on the require call as a value. Bundled npm code does both, for example a small semaphore that aliases EventEmitter and constructs its release channel through the alias. - The events module binding and the EventEmitter or stream class itself turn untyped as the runtime's untyped constructor that constructor functions already use (scr_dyn_builtin_class.c). `new` through it builds an emitter object with Node's listener API, and `===` against the class compares identity. - `x instanceof EventEmitter` on an untyped value also recognizes a boxed compiled emitter (an instance of a program class extending EventEmitter), which the prototype test alone answered false for. The same holds through an untyped binding narrowed to the class. - `class X extends require("events")` extends the runtime emitter, as through a require binding. - A typed array of the class built from such values (`[EventEmitter]`) is refused with SC1090 rather than failing its class check at run time. Co-Authored-By: Claude Opus 5.5 --- .../limitations/builtin-function-values.mdx | 2 + .../lowering/builtins/builtin-class-dyn.ts | 64 +++++++++++++++- .../frontend/lowering/expressions/binary.ts | 20 +++++ .../lowering/expressions/relational.ts | 18 ++++- .../src/frontend/lowering/lower-classes.ts | 23 +++++- .../src/frontend/lowering/lower-modules.ts | 4 +- .../compiler/src/frontend/lowering/lowerer.ts | 51 ++++++++++++- .../6571-events-module-class-value.cjs.json | 6 ++ .../6572-extends-require-events.cjs.json | 6 ++ .../6573-semaphore-release-emitter.cjs.json | 6 ++ .../2595-bundled-chunk-emitter/main.ts.json | 6 ++ .../corpus/6571-events-module-class-value.cjs | 30 ++++++++ tests/corpus/6572-extends-require-events.cjs | 27 +++++++ .../corpus/6573-semaphore-release-emitter.cjs | 62 +++++++++++++++ .../cases/2595-bundled-chunk-emitter/main.ts | 11 +++ .../node_modules/gtchunkee/dist/index.d.ts | 2 + .../npm/node_modules/gtchunkee/dist/index.js | 75 +++++++++++++++++++ .../npm/node_modules/gtchunkee/package.json | 7 ++ tests/harness/npm-cases.ts | 5 +- tests/harness/npm-static.test.ts | 9 +++ 20 files changed, 424 insertions(+), 10 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6571-events-module-class-value.cjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6572-extends-require-events.cjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6573-semaphore-release-emitter.cjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2595-bundled-chunk-emitter/main.ts.json create mode 100644 tests/corpus/6571-events-module-class-value.cjs create mode 100644 tests/corpus/6572-extends-require-events.cjs create mode 100644 tests/corpus/6573-semaphore-release-emitter.cjs create mode 100644 tests/fixtures/npm/cases/2595-bundled-chunk-emitter/main.ts create mode 100644 tests/fixtures/npm/node_modules/gtchunkee/dist/index.d.ts create mode 100644 tests/fixtures/npm/node_modules/gtchunkee/dist/index.js create mode 100644 tests/fixtures/npm/node_modules/gtchunkee/package.json diff --git a/docs/content/limitations/builtin-function-values.mdx b/docs/content/limitations/builtin-function-values.mdx index 541c9fb2b..a99f0d65c 100644 --- a/docs/content/limitations/builtin-function-values.mdx +++ b/docs/content/limitations/builtin-function-values.mdx @@ -8,6 +8,8 @@ Some builtins can be stored and passed as functions: In JavaScript, `setTimeout`, `setInterval`, `setImmediate`, `clearTimeout`, `clearInterval`, `clearImmediate`, `queueMicrotask`, and `process.nextTick` are function values wherever they are stored, including conditional and logical expressions and assignments: `typeof` answers `"function"`, identity holds across spellings, and calls schedule as the direct calls do, returning the same numeric timer handles. Calls through these values bypass `node:test` mock timers. Other builtin globals stored in JavaScript remain opaque identity values. +In JavaScript, `require("events")` and the `EventEmitter` class assigned to an untyped binding or stored in an object (`ReleaseEmitter = EventEmitter`) are the untyped `EventEmitter` constructor that constructor functions inherit from (see below): `new` through it builds such an emitter object, and identity and `instanceof` agree with the class. +A class can extend `require("events")` directly. Other immutable aliases of table-backed Node builtins support direct calls, but cannot be passed elsewhere. `util.promisify` supports compile-time conversions of `child_process.execFile`, `child_process.exec`, `setTimeout` and `setImmediate` (to the `timers/promises` functions, as in Node), `stream.finished` and `stream.pipeline` (to the `stream/promises` functions, as in Node), the `zlib` callback one-shots `deflate`, `inflate`, `deflateRaw`, `inflateRaw`, `gzip`, `gunzip` and `unzip` (with or without options), the `crypto` callback functions `randomBytes`, `pbkdf2`, `scrypt`, `hkdf`, `randomFill`, `randomInt` and `generateKey`, `dns.lookup` (settling with `{ address, family }`, or the address array with `all: true`) and `dns.lookupService` (settling with `{ hostname, service }`), and the `fs` callback functions that have an `fs/promises` counterpart (`access`, `appendFile`, `chmod`, `chown`, `copyFile`, `lchown`, `link`, `lstat`, `lutimes`, `mkdir`, `mkdtemp`, `readFile`, `readdir`, `readlink`, `realpath`, `rename`, `rm`, `rmdir`, `stat`, `statfs`, `symlink`, `truncate`, `unlink`, `utimes`, and `writeFile`); calls through the converted function lower as the `fs/promises` call. diff --git a/packages/compiler/src/frontend/lowering/builtins/builtin-class-dyn.ts b/packages/compiler/src/frontend/lowering/builtins/builtin-class-dyn.ts index ac73924ad..6242f3481 100644 --- a/packages/compiler/src/frontend/lowering/builtins/builtin-class-dyn.ts +++ b/packages/compiler/src/frontend/lowering/builtins/builtin-class-dyn.ts @@ -1,8 +1,18 @@ import { numLit } from "../../../ir/build.js"; -import { BOOL, DYN, RUNTIME_EMITTER_CLASS, type IrExpr, type SrcLoc } from "../../../ir/ir.js"; +import { + BOOL, + DYN, + RUNTIME_EMITTER_CLASS, + type IrExpr, + type IrStmt, + type SrcLoc, +} from "../../../ir/ir.js"; import * as ts from "../../ts7/adapter.js"; import { locOf } from "../../program.js"; import type { Lowerer } from "../lowerer.js"; +import type { ClassInfo } from "../lower-classes.js"; +import { classInstanceOf } from "../class-dynamic-dispatch.js"; +import { isSafeToRepeat } from "../expressions/evaluation-safety.js"; /* EventEmitter and the stream classes as untyped constructor values, the * bases of JavaScript constructor functions: @@ -147,3 +157,55 @@ export function builtinClassInstanceOf(value: IrExpr, kind: number, loc: SrcLoc) loc, }; } + +/** `value instanceof Base` for an untyped value that may also hold a + * compiled instance: an object built by a constructor function inherits + * the untyped constructor's prototype, and an instance of a program class + * extending Base boxes as a typed reference the class test recognizes. + * With `callee`, the right-hand side is an untyped value and the class + * test applies only when it is Base's untyped constructor. */ +export function builtinClassInstanceOfAny( + lowerer: Lowerer, + value: IrExpr, + kind: number, + info: ClassInfo, + loc: SrcLoc, + callee?: IrExpr, +): IrExpr { + const stmts: IrStmt[] = []; + const once = (expr: IrExpr, name: string): IrExpr => { + if (isSafeToRepeat(expr)) return expr; + const local = lowerer.declareHiddenLocal(name, DYN); + stmts.push({ kind: "varDecl", localId: local.id, init: expr, loc }); + return { kind: "varRef", localId: local.id, type: DYN, loc }; + }; + const tested = once(value, "%instanceValue"); + const ctor = callee ? once(callee, "%instanceCallee") : null; + const compiled = classInstanceOf(lowerer, tested, info, loc); + const result: IrExpr = { + kind: "logical", + op: "||", + left: ctor + ? { kind: "libCall", fn: "dyn.functionInstanceOf", args: [tested, ctor], type: BOOL, loc } + : builtinClassInstanceOf(tested, kind, loc), + right: ctor + ? { + kind: "logical", + op: "&&", + left: { + kind: "dynScalarEq", + left: ctor, + right: builtinClassValue(kind, loc), + type: BOOL, + loc, + }, + right: compiled, + type: BOOL, + loc, + } + : compiled, + type: BOOL, + loc, + }; + return stmts.length ? { kind: "seqExpr", stmts, result, type: BOOL, loc } : result; +} diff --git a/packages/compiler/src/frontend/lowering/expressions/binary.ts b/packages/compiler/src/frontend/lowering/expressions/binary.ts index 8ef631f60..d556245b9 100644 --- a/packages/compiler/src/frontend/lowering/expressions/binary.ts +++ b/packages/compiler/src/frontend/lowering/expressions/binary.ts @@ -45,6 +45,7 @@ import { lowerElementCompoundParts } from "./element-access.js"; import { InternalCompilerError } from "../../../errors.js"; import { isJsSourceFile, locOf } from "../../program.js"; import { storedClassValueType } from "../lower-classes.js"; +import { builtinClassKind } from "../builtins/builtin-class-dyn.js"; import { hasRuntimeStatics } from "../class-runtime-statics.js"; import { tryLowerExpression } from "./try-lower-expression.js"; import { fenceNodeModuleMutation, lowerRequireCacheHas } from "../lower-node-module.js"; @@ -1486,6 +1487,25 @@ export function lowerBinary(lowerer: Lowerer, expr: ts.BinaryExpression): IrExpr loc, }; } + // EventEmitter or a stream class against an untyped value in + // JavaScript: the untyped value can hold the class's untyped + // constructor (coerceToExpected), which compares by identity. + if ( + dynSide.type.kind === "dyn" && + isJsSourceFile(expr.getSourceFile()) && + ((scalarSide.type.kind === "moduleNs" && scalarSide.type.moduleId === "builtin:events") || + (scalarSide.kind === "classRef" && builtinClassKind(scalarSide.className) !== null)) + ) { + const boxed = lowerer.coerceToExpected(scalarSide, DYN); + return { + kind: "dynScalarEq", + left: scalarSide === left ? boxed : left, + right: scalarSide === right ? boxed : right, + ...(negated ? { negated: true as const } : {}), + type: BOOL, + loc, + }; + } if ( dynSide.type.kind === "dyn" && (scalarSide.type.kind === "f64" || diff --git a/packages/compiler/src/frontend/lowering/expressions/relational.ts b/packages/compiler/src/frontend/lowering/expressions/relational.ts index 91d1468c2..b29b67f8b 100644 --- a/packages/compiler/src/frontend/lowering/expressions/relational.ts +++ b/packages/compiler/src/frontend/lowering/expressions/relational.ts @@ -1,6 +1,6 @@ /* instanceof and in, including private-name and runtime-key in. */ import * as ts from "../../ts7/adapter.js"; -import { builtinClassInstanceOf, builtinClassKind } from "../builtins/builtin-class-dyn.js"; +import { builtinClassInstanceOfAny, builtinClassKind } from "../builtins/builtin-class-dyn.js"; import { checkedPrivateIn, localPrivateBrandClass, @@ -401,6 +401,16 @@ export function lowerInstanceOf(lowerer: Lowerer, expr: ts.BinaryExpression, loc } const left = lowerer.lowerExpr(expr.left); if (left.type.kind === "dyn") { + // An untyped binding the checker narrowed to EventEmitter or a + // stream class (`ReleaseEmitter = EventEmitter` in JavaScript): + // the binding holds an untyped constructor, so the test is + // Function.prototype[Symbol.hasInstance] over it. + const builtinKind = targetInfo.def.runtime ? builtinClassKind(targetInfo.def.name) : null; + if (builtinKind !== null) { + const right = lowerer.lowerExpr(expr.right); + if (right.type.kind === "dyn") + return builtinClassInstanceOfAny(lowerer, left, builtinKind, targetInfo, loc, right); + } if (targetInfo.def.runtime) { lowerer.unsupported( "SC1090", @@ -600,11 +610,11 @@ export function lowerInstanceOf(lowerer: Lowerer, expr: ts.BinaryExpression, loc } // `u instanceof EventEmitter` (or a stream class): an untyped object // built by a constructor function inherits the untyped constructor's - // prototype (builtin-class-dyn.ts). Compiled emitters and streams have - // no untyped form, so the prototype chain answers for every value. + // prototype (builtin-class-dyn.ts), and a compiled emitter or stream + // boxes as a typed reference the class test answers for. const builtinKind = builtinClassKind(target.def.name); if (left.type.kind === "dyn" && builtinKind !== null) { - return builtinClassInstanceOf(left, builtinKind, loc); + return builtinClassInstanceOfAny(lowerer, left, builtinKind, target, loc); } if (left.type.kind === "dyn") { lowerer.unsupported( diff --git a/packages/compiler/src/frontend/lowering/lower-classes.ts b/packages/compiler/src/frontend/lowering/lower-classes.ts index 99d854a6a..f63851546 100644 --- a/packages/compiler/src/frontend/lowering/lower-classes.ts +++ b/packages/compiler/src/frontend/lowering/lower-classes.ts @@ -89,14 +89,17 @@ import { withUnitArm, } from "../type-mapper.js"; import { + canonicalBuiltinModule, cjsClassExprWholeExportOf, isCjsJsFile, isJsSourceFile, isModuleExportsAccess, locOf, + requireSpecOf, } from "../program.js"; import { PoisonError, dynFallbackType, jsFuncNameOf, newFnCtx, own } from "./lowerer.js"; import { isAssertClassNew } from "./assert-instances.js"; +import { isCjsRequireGlobal } from "./lower-node-module.js"; import { lowerArrayConstructor } from "./containers/array-construction.js"; import { lowerMapSeedNew, lowerSetSeedNew } from "./containers/collection-methods.js"; import { lowerObjectConstructor } from "./containers/indexed-objects.js"; @@ -978,6 +981,19 @@ export function registerBuiltinEmitterClass(lowerer: Lowerer): void { lowerer.classes.set(RUNTIME_EMITTER_CLASS, info); } +/** `class Bus extends require("events") {}` in a CommonJS module: the + * events module is the EventEmitter class itself, so the base is the + * runtime emitter, exactly as through a `require("events")` binding. */ +export function builtinRequireBaseOf(lowerer: Lowerer, node: ts.Expression): ClassInfo | null { + let expr = node; + while (ts.isParenthesizedExpression(expr)) expr = expr.expression; + if (!ts.isCallExpression(expr) || !isJsSourceFile(expr.getSourceFile())) return null; + const spec = requireSpecOf(expr); + if (spec === null || canonicalBuiltinModule(spec) !== "events") return null; + if (!isCjsRequireGlobal(lowerer, expr.expression)) return null; + return lowerer.classes.get(RUNTIME_EMITTER_CLASS) ?? null; +} + /** The runtime-provided node:stream classes as eagerly-registered * ClassInfos (the emitter story): mapType names `%Readable` et al the * moment a stream type appears, so the infos must exist before any @@ -2092,6 +2108,11 @@ export function collectClassShapeInner( // callee is NOT a mixin function keeps the computed-expression // fence below. if (t && ts.isCallExpression(t.expression) && !t.typeArguments?.nodes) { + const requiredBase = builtinRequireBaseOf(lowerer, t.expression); + if (requiredBase) { + base = requiredBase; + continue; + } const mixinBase = lowerer.mixinCallClassInfoOf(t.expression); if (mixinBase) { base = mixinBase; @@ -5724,7 +5745,7 @@ export function lowerClassExpressionInfo( )?.types?.nodes[0]?.expression; if ( heritage && - (ts.isCallExpression(heritage) || + ((ts.isCallExpression(heritage) && !builtinRequireBaseOf(lowerer, heritage)) || (ts.isIdentifier(heritage) && (lowerer.peekLocal(heritage) ?? lowerer.globalOf(heritage)) != null)) ) { diff --git a/packages/compiler/src/frontend/lowering/lower-modules.ts b/packages/compiler/src/frontend/lowering/lower-modules.ts index 12c7d32ff..b0b07235c 100644 --- a/packages/compiler/src/frontend/lowering/lower-modules.ts +++ b/packages/compiler/src/frontend/lowering/lower-modules.ts @@ -132,6 +132,7 @@ import { recordTextCodecClass } from "../../ir/ir.js"; import { isUnitOnlyTsType, unitOnlyUnion } from "../type-mapper.js"; import type { ClassInfo } from "./lower-classes.js"; import { + builtinRequireBaseOf, collectInheritedJsFieldWrites, collectVirtualJsMethods, decoratorNodesOf, @@ -373,7 +374,8 @@ export function collectProgram(lowerer: Lowerer, parts: FileParts[]): void { const heritage = decl.heritageClauses?.nodes.find( (clause) => clause.token === ts.SyntaxKind.ExtendsKeyword, )?.types?.nodes[0]?.expression; - if (heritage && ts.isCallExpression(heritage)) computed.add(decl); + if (heritage && ts.isCallExpression(heritage) && !builtinRequireBaseOf(lowerer, heritage)) + computed.add(decl); } // Descendants need the computed ancestor's completed layout before // their own fields and inherited methods can be collected. diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index a5f376fc0..069eddd6c 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -6,7 +6,12 @@ import { } from "./general-streams.js"; import { extendAssertedRecordMembers } from "./assertion-members.js"; import { jsDynamicDeclarations } from "./js-dynamic-objects.js"; -import { builtinClassValue, builtinMemberClassKind } from "./builtins/builtin-class-dyn.js"; +import { isJsSourceFileName } from "../tsc-codes.js"; +import { + builtinClassKind, + builtinClassValue, + builtinMemberClassKind, +} from "./builtins/builtin-class-dyn.js"; import { lifetimeRegionType, lowerLifetimeRegionCall, @@ -8925,7 +8930,33 @@ export class Lowerer { } // A runtime-provided class value has no checked-dynamic form that // inspects as Node's does (`[Function: EventEmitter] { ... }`). + // JavaScript sources already read EventEmitter and the stream + // classes as the runtime's untyped constructors (builtin-class-dyn.ts), + // so one of those classes turning untyped there becomes the same + // constructor. `require("events")` is the EventEmitter class itself, + // so its module binding is too (`ReleaseEmitter = EventEmitter` over + // `const EventEmitter = require("events")`). + if ( + expr.type.kind === "moduleNs" && + expr.type.moduleId === "builtin:events" && + isJsSourceFileName(expr.loc.file) + ) { + const value = builtinClassValue(0, expr.loc); + if (expr.kind === "moduleNsRef") return value; + return { + kind: "seqExpr", + stmts: [{ kind: "exprStmt", expr, loc: expr.loc }], + result: value, + type: DYN, + loc: expr.loc, + }; + } if (expr.type.kind === "classval" && this.classes.get(expr.type.className)?.def.runtime) { + // Only the class itself: a slot of the class's type may hold a + // program subclass, which this constructor would misidentify. + const kind = builtinClassKind(expr.type.className); + if (kind !== null && expr.kind === "classRef" && isJsSourceFileName(expr.loc.file)) + return builtinClassValue(kind, expr.loc); this.pushDiag( unsupportedDiag( "SC1090", @@ -9005,6 +9036,24 @@ export class Lowerer { ); const handleOk = DYN_HANDLE_KINDS.has(expected.kind); noteUntypedStreamFlow(this, expected); + // `[EventEmitter]` in JavaScript builds an untyped array of the + // untyped constructor, which a typed array of the class cannot + // hold: no class check admits that constructor. + if ( + expr.kind === "dynArrLit" && + expected.kind === "array" && + expected.elem.kind === "classval" && + expr.elems.some((elem) => elem.kind === "libCall" && elem.fn === "dyn.builtinClass") + ) { + this.pushDiag( + unsupportedDiag( + "SC1090", + expr.loc, + `holding runtime-provided class values ('${expected.elem.className.slice(1)}') in a typed array`, + ), + ); + throw new PoisonError(); + } if ( canDynCheckTo( expected, diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6571-events-module-class-value.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6571-events-module-class-value.cjs.json new file mode 100644 index 000000000..c8eaf1451 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6571-events-module-class-value.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6571-events-module-class-value.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6572-extends-require-events.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6572-extends-require-events.cjs.json new file mode 100644 index 000000000..2fc3d1039 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6572-extends-require-events.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6572-extends-require-events.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6573-semaphore-release-emitter.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6573-semaphore-release-emitter.cjs.json new file mode 100644 index 000000000..f1dd30da9 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6573-semaphore-release-emitter.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6573-semaphore-release-emitter.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2595-bundled-chunk-emitter/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2595-bundled-chunk-emitter/main.ts.json new file mode 100644 index 000000000..ee60a3f46 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2595-bundled-chunk-emitter/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/2595-bundled-chunk-emitter/main.ts" + ], + "diags": [] +} diff --git a/tests/corpus/6571-events-module-class-value.cjs b/tests/corpus/6571-events-module-class-value.cjs new file mode 100644 index 000000000..9df037a6d --- /dev/null +++ b/tests/corpus/6571-events-module-class-value.cjs @@ -0,0 +1,30 @@ +// require("events") is the EventEmitter class. Assigned to an untyped +// binding or stored in an object, it stays a constructor: construction +// through it builds an emitter with the whole listener API, and identity, +// instanceof, typeof and .name agree with the class. +var EventEmitter = require("events"), ReleaseEmitter; +ReleaseEmitter = EventEmitter; + +const e = new ReleaseEmitter(); +e.on("x", (v) => console.log("x", v)); +e.prependListener("x", (v) => console.log("first x", v)); +e.once("x", (v) => console.log("once x", v)); +console.log("emit", e.emit("x", 1), e.emit("x", 2), e.emit("missing")); +console.log("count", e.listenerCount("x"), e.eventNames(), e.getMaxListeners()); +console.log("instanceof", e instanceof EventEmitter, e instanceof ReleaseEmitter); +console.log("identity", ReleaseEmitter === EventEmitter, ReleaseEmitter !== EventEmitter); +console.log("typeof", typeof ReleaseEmitter, ReleaseEmitter.name); + +const holder = { E: EventEmitter }; +const h = new holder.E(); +h.on("error", (err) => console.log("error", err.message)); +h.emit("error", new Error("boom")); +console.log("holder", h instanceof EventEmitter, h instanceof holder.E, holder.E === EventEmitter); + +try { + new ReleaseEmitter().emit("error", new Error("unhandled")); +} catch (err) { + console.log("caught", err.message); +} +e.removeAllListeners("x"); +console.log("removed", e.listenerCount("x"), e.emit("x", 3)); diff --git a/tests/corpus/6572-extends-require-events.cjs b/tests/corpus/6572-extends-require-events.cjs new file mode 100644 index 000000000..06e2f1c9a --- /dev/null +++ b/tests/corpus/6572-extends-require-events.cjs @@ -0,0 +1,27 @@ +// A class whose heritage is the require call itself extends EventEmitter, +// as through a require binding. The class is a value: stored in an untyped +// binding, construction through it runs the subclass constructor. +class Channel extends require("events") { + constructor(name) { + super(); + this.name = name; + } + send(text) { + return this.emit("message", this.name, text); + } +} + +const a = new Channel("a"); +a.on("message", (from, text) => console.log("message", from, text)); +console.log("sent", a.send("hello"), new Channel("b").send("nobody")); + +let Ctor; +Ctor = Channel; +const c = new Ctor("c"); +c.once("message", (from, text) => console.log("once", from, text)); +c.send("one"); +c.send("two"); + +const EventEmitter = require("events"); +console.log("instanceof", a instanceof EventEmitter, c instanceof EventEmitter, c instanceof Channel); +console.log("class", typeof Ctor, Ctor.name, Ctor === Channel, a.listenerCount("message")); diff --git a/tests/corpus/6573-semaphore-release-emitter.cjs b/tests/corpus/6573-semaphore-release-emitter.cjs new file mode 100644 index 000000000..fea3e284d --- /dev/null +++ b/tests/corpus/6573-semaphore-release-emitter.cjs @@ -0,0 +1,62 @@ +// A token semaphore whose constructor builds its release channel from the +// events module, the way small npm semaphores do: once through a subclass +// declared beside the require, once through an untyped alias of the class. +// acquire() hands out free tokens and queues the rest; release() emits, and +// the listener wakes the oldest waiter or returns the token. +var EventEmitter = require("events"), Subclassed = class extends EventEmitter {}, Aliased; +Aliased = EventEmitter; + +function defaultInit() { + return "1"; +} + +class Semaphore { + constructor(nr, { initFn = defaultInit, aliased = false } = {}) { + this.nrTokens = nr; + this.free = []; + this.waiting = []; + this.releaseEmitter = aliased ? new Aliased() : new Subclassed(); + this.noTokens = initFn === defaultInit; + this.releaseEmitter.on("release", (token) => { + const p = this.waiting.shift(); + if (p) p.resolve(token); + else this.free.push(token); + }); + for (let i = 0; i < nr; i++) this.free.push(initFn()); + } + async acquire() { + const token = this.free.pop(); + if (token !== undefined) return token; + return new Promise((resolve, reject) => { + this.waiting.push({ resolve, reject }); + }); + } + release(token) { + this.releaseEmitter.emit("release", this.noTokens ? "1" : token); + } + nrWaiting() { + return this.waiting.length; + } +} + +async function run(label, sema) { + const order = []; + const tasks = [1, 2, 3, 4].map(async (n) => { + const token = await sema.acquire(); + order.push(`${n}:${token}`); + await new Promise((resolve) => setTimeout(resolve, 2)); + sema.release(token); + }); + console.log(label, "waiting", sema.nrWaiting()); + await Promise.all(tasks); + console.log(label, order.join(" "), "free", sema.free.length, "waiting", sema.nrWaiting()); +} + +(async () => { + await run("plain", new Semaphore(2)); + let next = 0; + await run("tokens", new Semaphore(2, { initFn: () => `t${next++}` })); + await run("aliased", new Semaphore(1, { aliased: true })); + const s = new Semaphore(1); + console.log("channel", s.releaseEmitter instanceof EventEmitter, s.releaseEmitter instanceof Subclassed); +})(); diff --git a/tests/fixtures/npm/cases/2595-bundled-chunk-emitter/main.ts b/tests/fixtures/npm/cases/2595-bundled-chunk-emitter/main.ts new file mode 100644 index 000000000..e684af394 --- /dev/null +++ b/tests/fixtures/npm/cases/2595-bundled-chunk-emitter/main.ts @@ -0,0 +1,11 @@ +// A bundled CommonJS chunk whose top-level var statement mixes +// require("events"), require("util"), a sibling chunk's require and a class +// expression extending the events binding, followed by an untyped alias of +// the events module. The require bindings own no storage: the semaphore's +// constructor builds its release channel through them, its queue through +// the sibling chunk's constructor function, and both channels emit. +import { aliased, limited } from "gtchunkee"; + +console.log(aliased()); +console.log(await limited("two slots", 4, 2)); +console.log(await limited("one slot", 3, 1)); diff --git a/tests/fixtures/npm/node_modules/gtchunkee/dist/index.d.ts b/tests/fixtures/npm/node_modules/gtchunkee/dist/index.d.ts new file mode 100644 index 000000000..f9f2d1535 --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtchunkee/dist/index.d.ts @@ -0,0 +1,2 @@ +export declare function limited(label: string, jobs: number, slots: number): Promise; +export declare function aliased(): string; diff --git a/tests/fixtures/npm/node_modules/gtchunkee/dist/index.js b/tests/fixtures/npm/node_modules/gtchunkee/dist/index.js new file mode 100644 index 000000000..59040f238 --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtchunkee/dist/index.js @@ -0,0 +1,75 @@ +var __create = Object.create; +var __defProp = Object.defineProperty; +var __getOwnPropDesc = Object.getOwnPropertyDescriptor; +var __getOwnPropNames = Object.getOwnPropertyNames; +var __getProtoOf = Object.getPrototypeOf; +var __hasOwnProp = Object.prototype.hasOwnProperty; +var __commonJS = (cb, mod) => function __require() { + return mod || (0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports; +}; +var __export = (target, all) => { + for (var name in all) + __defProp(target, name, { get: all[name], enumerable: true }); +}; +var __copyProps = (to, from, except, desc) => { + if (from && typeof from === "object" || typeof from === "function") { + for (let key of __getOwnPropNames(from)) + if (!__hasOwnProp.call(to, key) && key !== except) + __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable }); + } + return to; +}; +var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps( + isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target, + mod +)); +var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod); + +// node_modules/queue/index.js +var require_queue = __commonJS({ + "node_modules/queue/index.js"(exports2, module2) { + "use strict";function Queue(capacity){this._items=[],this._capacity=capacity||16}Queue.prototype.push=function(item){return this._items.push(item)};Queue.prototype.shift=function(){return this._items.shift()};Queue.prototype.pop=function(){return this._items.pop()};Object.defineProperty(Queue.prototype,"length",{get:function(){return this._items.length}});module2.exports=Queue; + } +}); + +// node_modules/sema/index.js +var require_sema = __commonJS({ + "node_modules/sema/index.js"(exports2) { + var EventEmitter=require("events"),util=require("util"),Queue=require_queue(),ReleaseEmitter=class extends EventEmitter{},AliasedEmitter;AliasedEmitter=EventEmitter;function defaultInit(){return"1"}class Sema{constructor(nr,{initFn=defaultInit,aliased=!1}={}){this.nrTokens=nr,this.free=new Queue(nr),this.waiting=new Queue(10),this.releaseEmitter=aliased?new AliasedEmitter:new ReleaseEmitter,this.noTokens=initFn===defaultInit,this.releaseEmitter.on("release",token=>{let p=this.waiting.shift();p?p.resolve(token):this.free.push(token)});for(let i=0;i{this.waiting.push({resolve,reject})})}release(token){this.releaseEmitter.emit("release",this.noTokens?"1":token)}nrWaiting(){return this.waiting.length}describe(){return util.format("%d tokens, %s",this.nrTokens,this.releaseEmitter instanceof EventEmitter?"emitter":"other")}}exports2.create=function(nr,options){return new Sema(nr,options)}; + } +}); + +// src/index.ts +var index_exports = {}; +__export(index_exports, { + aliased: () => aliased, + limited: () => limited +}); +module.exports = __toCommonJS(index_exports); +var import_sema = __toESM(require_sema()); +async function limited(label, jobs, slots) { + let next = 0; + const sema = (0, import_sema.create)(slots, { initFn: () => "t" + next++ }); + const order = []; + const tasks = []; + for (let i = 0; i < jobs; i++) { + tasks.push((async () => { + const token = await sema.acquire(); + order.push(i + ":" + token); + await new Promise((resolve) => setTimeout(resolve, 2)); + sema.release(token); + })()); + } + const queued = sema.nrWaiting(); + await Promise.all(tasks); + return label + " queued " + queued + " order " + order.join(" ") + " (" + sema.describe() + ")"; +} +function aliased() { + const sema = (0, import_sema.create)(1, { aliased: true }); + return sema.describe() + " waiting " + sema.nrWaiting(); +} +// Annotate the CommonJS export names for ESM import in node: +0 && (module.exports = { + aliased, + limited +}); diff --git a/tests/fixtures/npm/node_modules/gtchunkee/package.json b/tests/fixtures/npm/node_modules/gtchunkee/package.json new file mode 100644 index 000000000..e47378c0e --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtchunkee/package.json @@ -0,0 +1,7 @@ +{ + "name": "gtchunkee", + "version": "1.0.0", + "description": "esbuild-style CommonJS bundle (--bundle --platform=node --format=cjs, whitespace+syntax minified chunk bodies): a semaphore chunk whose top-level var statement mixes require(\"events\"), require(\"util\"), a sibling chunk require and a class extending the events binding", + "main": "./dist/index.js", + "types": "./dist/index.d.ts" +} diff --git a/tests/harness/npm-cases.ts b/tests/harness/npm-cases.ts index 01718abf9..43044ee60 100644 --- a/tests/harness/npm-cases.ts +++ b/tests/harness/npm-cases.ts @@ -19,13 +19,14 @@ export function npmCases(fixturesRoot: string): NpmCase[] { return [ ...globSync(join(fixturesRoot, "npm/cases/*/main.ts")) .sort() - // 2465-2469, 2556-2567, 2571 and 2580-2584 are the --npm-static bundler-emitted-CJS + // 2465-2469, 2556-2567, 2571, 2580-2584 and 2595 are the --npm-static bundler-emitted-CJS // cases (npm-static.test.ts drives them with the opt-in): their // consumers exercise the static rewrite's surface (lexer-visible // names the shipped .d.ts never declares, the __toESM interop // family), so they stay out of the flagless island lane by design. .filter( - (entry) => !/\/(246[5-9]|255[6-9]|256[0-7]|2571|258[0-4])-[^/]+\/main\.ts$/.test(entry), + (entry) => + !/\/(246[5-9]|255[6-9]|256[0-7]|2571|258[0-4]|2595)-[^/]+\/main\.ts$/.test(entry), ) .map((entry) => ({ name: entry.split("/").at(-2)!, diff --git a/tests/harness/npm-static.test.ts b/tests/harness/npm-static.test.ts index c16567f47..499c50f17 100644 --- a/tests/harness/npm-static.test.ts +++ b/tests/harness/npm-static.test.ts @@ -1438,6 +1438,15 @@ describe(`npm-static pilots${sanitize ? " (sanitized)" : ""}`, () => { ]); }, 180_000); + // 2595: a minified chunk whose top-level var statement mixes require + // bindings with a class extending the events binding; the bindings own no + // storage, so the constructor reaches the modules through them. + test("bundled chunk 2595-bundled-chunk-emitter builds its emitters through require bindings", async () => { + await expectNodeParity(join(fixturesRoot, "npm/cases/2595-bundled-chunk-emitter/main.ts"), [ + "gtchunkee", + ]); + }, 180_000); + // A whitespace/syntax-minified bundle whose helpers keep their names is // judged by the recognizers, not refused up front as minified. test("--npm-static=auto opts a minified bundle with named helpers in", () => { From e060eeae2571f3caf8ba208c4c5d7ded29ac773a Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Tue, 20 Oct 2026 15:17:40 -0700 Subject: [PATCH 18/38] Iterate untyped values with for await, and delegate with yield* in async generators - A JavaScript generator whose inferred return is `any` (it delegates to itself) is a generator with checked-dynamic channels. - for await over a checked-dynamic value: a compiled async generator, the value's Symbol.asyncIterator result, or its synchronous iterator with awaited values; break and throws close it through an awaited return(). A value that is not iterable throws V8's TypeError. - yield* in async generators delegates through the same protocol and completes with the delegate's return value. The runtime part is a new unit, scr_dyn_for_await.c, linked only by programs that use it. Co-Authored-By: Claude Opus 5.5 --- .../src/backend/executable-features.ts | 1 + .../compiler/src/backend/llvm/lib-shared.ts | 2 + .../src/backend/native-link-info-core.ts | 3 + .../compiler/src/backend/native/contracts.ts | 2 + .../compiler/src/backend/native/executable.ts | 2 + .../compiler/src/backend/runtime-pack-core.ts | 2 + .../src/executable/executable-cache.ts | 1 + .../src/frontend/lowering/call-signatures.ts | 14 +- .../frontend/lowering/generator-delegation.ts | 192 +++++++++++++++++ .../src/frontend/lowering/lower-generators.ts | 17 +- .../src/frontend/lowering/lower-stmts.ts | 193 ++++++++++++++++-- packages/compiler/src/index.ts | 1 + packages/compiler/src/ir/builtin-effects.ts | 1 + .../compiler/src/ir/builtin-signatures.ts | 5 + packages/compiler/src/ir/runtime-features.ts | 3 + packages/compiler/src/ir/validate.ts | 4 +- .../compiler/src/native/sanitizer.test.ts | 1 + .../6580-npm-dyn-for-await/main.ts.json | 6 + .../runtime-pack-matrix.mjs | 2 + packages/runtime/src/scr_dyn.h | 3 + packages/runtime/src/scr_dyn_for_await.c | 45 ++++ tests/corpus/6580-npm-dyn-for-await/main.ts | 25 +++ .../node_modules/async-walk/index.d.ts | 2 + .../node_modules/async-walk/index.js | 84 ++++++++ .../node_modules/async-walk/package.json | 1 + 25 files changed, 592 insertions(+), 20 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6580-npm-dyn-for-await/main.ts.json create mode 100644 packages/runtime/src/scr_dyn_for_await.c create mode 100644 tests/corpus/6580-npm-dyn-for-await/main.ts create mode 100644 tests/corpus/6580-npm-dyn-for-await/node_modules/async-walk/index.d.ts create mode 100644 tests/corpus/6580-npm-dyn-for-await/node_modules/async-walk/index.js create mode 100644 tests/corpus/6580-npm-dyn-for-await/node_modules/async-walk/package.json diff --git a/packages/compiler/src/backend/executable-features.ts b/packages/compiler/src/backend/executable-features.ts index 0fcf86ada..cdf14b6aa 100644 --- a/packages/compiler/src/backend/executable-features.ts +++ b/packages/compiler/src/backend/executable-features.ts @@ -27,6 +27,7 @@ export function executableLinkFeatures(mod: IrModule): NativeLinkFeatures { dynAsync: features.dynAsync, ...(features.dynTier ? { dynTier: true } : {}), ...(features.streamDyn ? { streamDyn: true } : {}), + ...(features.dynForAwait ? { dynForAwait: true } : {}), ...(features.eventsOnceSignal ? { eventsOnceSignal: true } : {}), events: features.processEvents, emitter: features.emitter, diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index 8cd40240f..657b3ad0b 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -941,6 +941,8 @@ export const LIB_FN_SYMS: Record = { "dyn.iterPack": "scr_dyn_iter_pack", "dyn.destructureRest": "scr_dyn_destructure_rest", "dyn.iterator": "scr_dyn_iterator", + "dyn.asyncIteratorOpen": "scr_dyn_async_iterator_open", + "dyn.asyncIteratorWasSync": "scr_dyn_async_iterator_was_sync", "dyn.arrayFromIterator": "scr_dyn_array_from_iterator", "dyn.iteratorResult": "scr_dyn_iterator_result", "dyn.iteratorCanStep": "scr_dyn_iterator_can_step", diff --git a/packages/compiler/src/backend/native-link-info-core.ts b/packages/compiler/src/backend/native-link-info-core.ts index 84f448e93..069bf80ac 100644 --- a/packages/compiler/src/backend/native-link-info-core.ts +++ b/packages/compiler/src/backend/native-link-info-core.ts @@ -46,6 +46,9 @@ export interface NativeLinkFeatures { /** Checked-dynamic stream slots: the scr_stream_dyn.c unit. Absent (not * false) for programs without them. */ streamDyn?: boolean; + /** for await over checked-dynamic values: scr_dyn_for_await.c. Absent + * (false) for programs without it. */ + dynForAwait?: boolean; /** events.once with options: scr_events_once_signal.c. Absent (not * false) for programs without it. */ eventsOnceSignal?: boolean; diff --git a/packages/compiler/src/backend/native/contracts.ts b/packages/compiler/src/backend/native/contracts.ts index a9c93d3df..759ea478c 100644 --- a/packages/compiler/src/backend/native/contracts.ts +++ b/packages/compiler/src/backend/native/contracts.ts @@ -116,6 +116,8 @@ export interface CcOptions { /** Checked-dynamic stream slots (stream.dynNative / stream.pipeDyn): * compiles scr_stream_dyn.c. */ streamDyn?: boolean; + /** for await over checked-dynamic values: compiles scr_dyn_for_await.c. */ + dynForAwait?: boolean; /** events.once with options (events.onceSignal): compiles * scr_events_once_signal.c. */ eventsOnceSignal?: boolean; diff --git a/packages/compiler/src/backend/native/executable.ts b/packages/compiler/src/backend/native/executable.ts index 949f03839..7c627e57c 100644 --- a/packages/compiler/src/backend/native/executable.ts +++ b/packages/compiler/src/backend/native/executable.ts @@ -268,6 +268,8 @@ const RUNTIME_UNITS: readonly RuntimeUnit[] = [ // Checked-dynamic stream slots test values and call JavaScript stream // members through the dyn-invoke unit. unit((u) => u.opts.streamDyn === true, "scr_stream_dyn.c"), + // for await over checked-dynamic values. + unit((u) => u.opts.dynForAwait === true, "scr_dyn_for_await.c"), // FileHandle streams reference both the stream unit and FileHandle. unit((u) => u.opts.stream && u.opts.fileHandle, "scr_fs_stream_fh.c"), // The readiness-poller backends (scr_platform.h): kqueue on macOS/BSD, diff --git a/packages/compiler/src/backend/runtime-pack-core.ts b/packages/compiler/src/backend/runtime-pack-core.ts index 5c9d417c5..42ea18c79 100644 --- a/packages/compiler/src/backend/runtime-pack-core.ts +++ b/packages/compiler/src/backend/runtime-pack-core.ts @@ -85,6 +85,7 @@ export interface RuntimeFeatureSet extends NativeLinkFeatures { wasm: boolean; dynTier: boolean; streamDyn: boolean; + dynForAwait: boolean; eventsOnceSignal: boolean; warningReplace: boolean; stdioExtended: boolean; @@ -278,6 +279,7 @@ export function effectiveRuntimeFeatures( wasm: features.wasm === true, dynTier: features.dynTier === true, streamDyn: features.streamDyn === true, + dynForAwait: features.dynForAwait === true, eventsOnceSignal: features.eventsOnceSignal === true, warningReplace: features.warningReplace === true, stdioExtended: features.stdioExtended === true, diff --git a/packages/compiler/src/executable/executable-cache.ts b/packages/compiler/src/executable/executable-cache.ts index 064b1f867..018b12b4f 100644 --- a/packages/compiler/src/executable/executable-cache.ts +++ b/packages/compiler/src/executable/executable-cache.ts @@ -67,6 +67,7 @@ export interface EarlyExecutableNativeFeatures { dynAsync: boolean; dynTier?: boolean; streamDyn?: boolean; + dynForAwait?: boolean; eventsOnceSignal?: boolean; events: boolean; emitter: boolean; diff --git a/packages/compiler/src/frontend/lowering/call-signatures.ts b/packages/compiler/src/frontend/lowering/call-signatures.ts index 47cdd834f..889409031 100644 --- a/packages/compiler/src/frontend/lowering/call-signatures.ts +++ b/packages/compiler/src/frontend/lowering/call-signatures.ts @@ -808,7 +808,19 @@ export function collectSignatureInner(lowerer: Lowerer, decl: ts.FunctionDeclara decl.parameters.nodes.length > 0 ? "arguments" : "dynRest", ); const nameBlame: ts.Node = decl.name ?? decl; - const returnType = lowerer.declaredReturnType(decl, nameBlame); + let returnType = lowerer.declaredReturnType(decl, nameBlame); + // A JavaScript generator whose inferred return the checker leaves `any` + // (it delegates to itself: `yield* walkTree(sub)`) is still a generator: + // its channels are checked-dynamic values. + if (isGenerator && returnType.kind === "dyn" && isJsSourceFile(ts.getSourceFileOfNode(decl))) { + returnType = { + kind: "generator", + ...(isAsync ? { async: true as const } : {}), + yieldT: DYN, + retT: DYN, + nextT: DYN, + }; + } if (isAsync && !isGenerator && returnType.kind !== "promise") { lowerer.badType(nameBlame, lowerer.typeOf(nameBlame)); } diff --git a/packages/compiler/src/frontend/lowering/generator-delegation.ts b/packages/compiler/src/frontend/lowering/generator-delegation.ts index cfe13007f..f2ead3b4a 100644 --- a/packages/compiler/src/frontend/lowering/generator-delegation.ts +++ b/packages/compiler/src/frontend/lowering/generator-delegation.ts @@ -2,6 +2,7 @@ import { dynUndefinedExpr, numLit, varRef } from "../../ir/build.js"; import * as ts from "../ts7/adapter.js"; import { BOOL, + CAUGHT, DYN, F64, STRING, @@ -247,6 +248,197 @@ export function lowerCheckedDelegation( return { kind: "seqExpr", stmts, result, generatorDelegate: true, type: result.type, loc }; } +/** `yield* e` in an async generator: the async iterator protocol (a compiled + * async generator, Symbol.asyncIterator's result, or a synchronous iterator + * whose values are awaited). Each step's result is awaited, its values are + * yielded, and the expression completes with the final result's value. An + * abrupt completion at the yield (a consumer's return() or throw()) closes + * the delegate through its return(), as for-await does; next() arguments + * are not forwarded to the delegate. */ +export function lowerAsyncDelegation( + lowerer: Lowerer, + expr: ts.YieldExpression, + source: IrExpr, +): IrExpr { + const loc = locOf(expr); + const gen = lowerer.ctx.generator!; + const slot = (name: string, type: IrType, mutable = false) => { + const local = lowerer.declareHiddenLocal(name, type); + local.mutable = mutable; + return { local, ref: varRef(local.id, type, loc) }; + }; + const iterator = slot("%adelegate", DYN); + const sync = slot("%adelegateSync", BOOL); + const next = slot("%adelegateNext", DYN); + const step = slot("%adelegateStep", DYN, true); + const needsClose = slot("%adelegateClose", BOOL, true); + const returned = slot("%adelegateReturn", DYN); + const error = slot("%adelegateError", CAUGHT); + const get = (value: IrExpr, key: string): IrExpr => ({ + kind: "dynKeyGet", + value, + key: { kind: "strLit", value: key, type: STRING, loc }, + type: DYN, + loc, + }); + const awaitDyn = (value: IrExpr): IrExpr => ({ + kind: "libCall", + fn: "async.awaitDyn", + args: [value], + type: DYN, + loc, + }); + const invoke = (callee: IrExpr): IrExpr => + awaitDyn({ + kind: "dynCall", + callee, + receiver: iterator.ref, + calleeName: "iterator method", + args: [], + type: DYN, + loc, + }); + const stepNext: IrExpr = { + kind: "libCall", + fn: "dyn.iteratorResult", + args: [invoke(next.ref)], + type: DYN, + loc, + }; + const setClose = (value: boolean): IrStmt => ({ + kind: "assign", + localId: needsClose.local.id, + value: { kind: "boolLit", value, type: BOOL, loc }, + loc, + }); + const close = (): IrStmt[] => [ + { kind: "varDecl", localId: returned.local.id, init: get(iterator.ref, "return"), loc }, + { + kind: "if", + cond: { + kind: "unary", + op: "!", + operand: { kind: "dynTest", test: "nullish", value: returned.ref, type: BOOL, loc }, + type: BOOL, + loc, + }, + then: [{ kind: "exprStmt", expr: invoke(returned.ref), loc }], + else_: null, + loc, + }, + ]; + const yielded: IrExpr = { + kind: "ternary", + cond: sync.ref, + then: awaitDyn(get(step.ref, "value")), + else_: get(step.ref, "value"), + type: DYN, + loc, + }; + const stmts: IrStmt[] = [ + { + kind: "varDecl", + localId: iterator.local.id, + init: { + kind: "libCall", + fn: "dyn.iteratorResult", + args: [ + { + kind: "libCall", + fn: "dyn.asyncIteratorOpen", + args: [source, { kind: "strLit", value: "", type: STRING, loc }], + type: DYN, + loc, + }, + ], + type: DYN, + loc, + }, + loc, + }, + { + kind: "varDecl", + localId: sync.local.id, + init: { kind: "libCall", fn: "dyn.asyncIteratorWasSync", args: [], type: BOOL, loc }, + loc, + }, + { kind: "varDecl", localId: next.local.id, init: get(iterator.ref, "next"), loc }, + { + kind: "varDecl", + localId: needsClose.local.id, + init: { kind: "boolLit", value: false, type: BOOL, loc }, + loc, + }, + { kind: "varDecl", localId: step.local.id, init: stepNext, loc }, + { + kind: "tryCatch", + tryBody: [ + { + kind: "while", + cond: { + kind: "unary", + op: "!", + operand: { + kind: "dynTest", + test: "truthy", + value: get(step.ref, "done"), + type: BOOL, + loc, + }, + type: BOOL, + loc, + }, + body: [ + setClose(true), + { + kind: "exprStmt", + expr: { + kind: "yieldExpr", + value: lowerer.coerceToExpected(yielded, gen.yieldT), + type: gen.nextT.kind === "undefinedT" ? VOID : gen.nextT, + loc, + }, + loc, + }, + setClose(false), + { kind: "assign", localId: step.local.id, value: stepNext, loc }, + ], + loc, + }, + ], + catchBody: [ + { + kind: "if", + cond: needsClose.ref, + then: [ + setClose(false), + { + kind: "tryCatch", + tryBody: close(), + catchBody: [], + catchLocalId: null, + finallyBody: null, + loc, + }, + ], + else_: null, + loc, + }, + { kind: "rethrow", localId: error.local.id, loc }, + ], + catchLocalId: error.local.id, + finallyBody: [{ kind: "if", cond: needsClose.ref, then: close(), else_: null, loc }], + loc, + }, + ]; + const expected = lowerer.mapTypeOf(lowerer.typeOf(expr)); + const result = + expected && expected.kind !== "void" + ? lowerer.coerceToExpected(get(step.ref, "value"), expected) + : get(step.ref, "value"); + return { kind: "seqExpr", stmts, result, generatorDelegate: true, type: result.type, loc }; +} + /** `yield* xs` over an array, a string, a Set or a Map: the built-in * iterator walked in place, with the same live view as for-of (an array * re-reads its length every step, a string steps by code point, a Map diff --git a/packages/compiler/src/frontend/lowering/lower-generators.ts b/packages/compiler/src/frontend/lowering/lower-generators.ts index 19a75a0b4..b4e3ac102 100644 --- a/packages/compiler/src/frontend/lowering/lower-generators.ts +++ b/packages/compiler/src/frontend/lowering/lower-generators.ts @@ -26,7 +26,11 @@ import { import { locOf } from "../program.js"; import { genResultRecord } from "../type-mapper.js"; import { forOfVarTarget, lowerDestructuringAssign } from "./lower-stmts.js"; -import { lowerBuiltinDelegation, lowerCheckedDelegation } from "./generator-delegation.js"; +import { + lowerAsyncDelegation, + lowerBuiltinDelegation, + lowerCheckedDelegation, +} from "./generator-delegation.js"; export type GenType = IrType & { kind: "generator" }; @@ -818,14 +822,21 @@ export function lowerYieldStarStatement(lowerer: Lowerer, expr: ts.Expression): function lowerYieldStar(lowerer: Lowerer, expr: ts.YieldExpression): IrExpr { const gen = lowerer.ctx.generator; if (!gen) lowerer.unsupported("SC1071", expr); + if (!expr.expression) lowerer.unsupported("SC1071", expr, "'yield*' with no operand"); + // An async generator delegates through the async iterator protocol: a + // compiled async generator operand crosses as its checked capsule. if (lowerer.ctx.isAsync) { + const source = lowerer.lowerExpr(expr.expression); + const delegate = + source.kind === "dynCheck" && source.value.type.kind === "dyn" ? source.value : source; + if (delegate.type.kind === "dyn" || lowerer.dynConvertible(delegate.type)) + return lowerAsyncDelegation(lowerer, expr, lowerer.coerceToExpected(delegate, DYN)); lowerer.unsupported( "SC1071", expr, - "'yield*' in async generators (use 'for await' and yield each value explicitly)", + "'yield*' in async generators over this operand (use 'for await' and yield each value explicitly)", ); } - if (!expr.expression) lowerer.unsupported("SC1071", expr, "'yield*' with no operand"); const loc = locOf(expr); // An array literal operand builds as its own array type: the yield* // context would otherwise type it by the generator. diff --git a/packages/compiler/src/frontend/lowering/lower-stmts.ts b/packages/compiler/src/frontend/lowering/lower-stmts.ts index 69f6cc99b..f9a852409 100644 --- a/packages/compiler/src/frontend/lowering/lower-stmts.ts +++ b/packages/compiler/src/frontend/lowering/lower-stmts.ts @@ -9086,6 +9086,14 @@ function lowerArrayAssignInto( * specific diagnostics; other iterables fail on their types. The loop * variable is a fresh const binding per iteration (its own scope, like a * loop body). */ +/** Whether a `for await` head is untyped: the checker answers any or + * unknown, or a type that maps to a checked-dynamic value. */ +function isUntypedIterable(lowerer: Lowerer, node: ts.Expression): boolean { + const type = lowerer.typeOf(node); + if ((type.flags & (ts.TypeFlags.Any | ts.TypeFlags.Unknown)) !== 0) return true; + return lowerer.mapTypeOf(type)?.kind === "dyn"; +} + export function lowerForOf(lowerer: Lowerer, stmt: ts.ForOfStatement): IrStmt { // Labels consume HERE (nested statements must never see them). The // plain array/string paths carry them; the container/matchAll/stdin @@ -9147,6 +9155,32 @@ export function lowerForOf(lowerer: Lowerer, stmt: ts.ForOfStatement): IrStmt { if (dynStream) return dynStream; const builtin = lowerForAwaitBuiltin(lowerer, stmt); if (builtin) return builtin; + // An untyped value (a JavaScript async generator whose recursive + // return the checker types `any`): a compiled async generator steps + // natively; any other value takes the checked-dynamic async protocol. + // A synchronous generator, and in JavaScript any other head that + // reaches here, takes the same protocol: values of a synchronous + // iterator are awaited as the async-from-sync iterator awaits them, and + // a value that is not iterable throws Node's TypeError. + const headT = lowerer.mapTypeOf(lowerer.typeOf(stmt.expression)); + const syncGenerator = + (headT?.kind === "generator" && headT.async !== true) || isJsSourceFile(stmt.getSourceFile()); + if (lowerer.ctx.isAsync && (syncGenerator || isUntypedIterable(lowerer, stmt.expression))) { + const iterable = lowerer.lowerExpr(stmt.expression); + if (iterable.type.kind === "generator" && iterable.type.async) + return lowerForAwaitGenerator( + lowerer, + stmt, + iterable as IrExpr & { type: GenType }, + labels, + ); + if (iterable.type.kind === "dyn") return lowerForOfDyn(lowerer, stmt, iterable, labels, true); + if ( + (iterable.type.kind === "generator" || isJsSourceFile(stmt.getSourceFile())) && + lowerer.dynConvertible(iterable.type) + ) + return lowerForOfDyn(lowerer, stmt, lowerer.coerceToExpected(iterable, DYN), labels, true); + } // A NodeJS.ReadableStream iterates as the general Readable. requestGeneralStreamSurface(lowerer, stmt.expression); lowerer.unsupported( @@ -10442,6 +10476,7 @@ function lowerForOfDyn( stmt: ts.ForOfStatement, iterable: IrExpr, labels?: string[], + asyncLoop = false, ): IrStmt { const loc = locOf(stmt); const headText = (e: ts.Expression): string | null => { @@ -10454,11 +10489,12 @@ function lowerForOfDyn( }; const name = headText(stmt.expression); const callee = ts.isCallExpression(stmt.expression) ? headText(stmt.expression.expression) : null; + const iterableWord = asyncLoop ? "async iterable" : "iterable"; const spell = name !== null - ? `${name} is not iterable` + ? `${name} is not ${iterableWord}` : callee !== null - ? `${callee} is not a function or its return value is not iterable` + ? `${callee} is not a function or its return value is not ${iterableWord}` : ""; const iterator = lowerer.declareHiddenLocal("%dofIterator", DYN); const next = lowerer.declareHiddenLocal("%dofNext", DYN); @@ -10470,6 +10506,14 @@ function lowerForOfDyn( const needsClose = lowerer.declareHiddenLocal("%dofNeedsClose", BOOL); needsClose.mutable = true; const error = lowerer.declareHiddenLocal("%dofError", CAUGHT); + const syncIterator = asyncLoop ? lowerer.declareHiddenLocal("%dofSync", BOOL) : null; + const awaitDyn = (value: IrExpr): IrExpr => ({ + kind: "libCall", + fn: "async.awaitDyn", + args: [value], + type: DYN, + loc, + }); const get = (source: IrExpr, key: string): IrExpr => ({ kind: "dynKeyGet", value: source, @@ -10522,7 +10566,23 @@ function lowerForOfDyn( type: BOOL, loc, }, - then: [{ kind: "exprStmt", expr: call(varRef(returned.id, DYN, loc)), loc }], + then: [ + { + kind: "exprStmt", + expr: asyncLoop + ? awaitDyn({ + kind: "dynCall", + callee: varRef(returned.id, DYN, loc), + receiver: varRef(iterator.id, DYN, loc), + calleeName: "iterator method", + args: [], + type: DYN, + loc, + }) + : call(varRef(returned.id, DYN, loc)), + loc, + }, + ], else_: null, loc, }, @@ -10615,15 +10675,24 @@ function lowerForOfDyn( { kind: "varDecl", localId: value.id, - init: iteratorValue( - lowerer, - varRef(iterator.id, DYN, loc), - varRef(next.id, DYN, loc), - varRef(fast.id, BOOL, loc), - done, - loc, - "iterator method", - ), + init: syncIterator + ? asyncIteratorValue( + lowerer, + varRef(iterator.id, DYN, loc), + varRef(next.id, DYN, loc), + varRef(syncIterator.id, BOOL, loc), + done, + loc, + ) + : iteratorValue( + lowerer, + varRef(iterator.id, DYN, loc), + varRef(next.id, DYN, loc), + varRef(fast.id, BOOL, loc), + done, + loc, + "iterator method", + ), loc, }, { @@ -10648,13 +10717,29 @@ function lowerForOfDyn( localId: iterator.id, init: check({ kind: "libCall", - fn: "dyn.iterator", + fn: asyncLoop ? "dyn.asyncIteratorOpen" : "dyn.iterator", args: [iterable, { kind: "strLit", value: spell, type: STRING, loc }], type: DYN, loc, }), loc, }, + ...(syncIterator + ? [ + { + kind: "varDecl", + localId: syncIterator.id, + init: { + kind: "libCall", + fn: "dyn.asyncIteratorWasSync", + args: [], + type: BOOL, + loc, + }, + loc, + } satisfies IrStmt, + ] + : []), { kind: "varDecl", localId: next.id, @@ -10664,7 +10749,9 @@ function lowerForOfDyn( { kind: "varDecl", localId: fast.id, - init: iteratorCanStep(varRef(iterator.id, DYN, loc), varRef(next.id, DYN, loc), loc), + init: asyncLoop + ? { kind: "boolLit", value: false, type: BOOL, loc } + : iteratorCanStep(varRef(iterator.id, DYN, loc), varRef(next.id, DYN, loc), loc), loc, }, { @@ -10716,6 +10803,84 @@ function lowerForOfDyn( } } +/** One step of `for await` over a checked-dynamic iterator: await next()'s + * result, read done before value, and await the value of a synchronous + * iterator, as the async-from-sync iterator does. */ +function asyncIteratorValue( + lowerer: Lowerer, + iterator: IrExpr, + next: IrExpr, + sync: IrExpr, + done: IrLocal, + loc: SrcLoc, +): IrExpr { + const result = lowerer.declareHiddenLocal("%asyncIteratorResult", DYN); + const get = (key: string): IrExpr => ({ + kind: "dynKeyGet", + value: varRef(result.id, DYN, loc), + key: { kind: "strLit", value: key, type: STRING, loc }, + type: DYN, + loc, + }); + const awaitDyn = (value: IrExpr): IrExpr => ({ + kind: "libCall", + fn: "async.awaitDyn", + args: [value], + type: DYN, + loc, + }); + return { + kind: "seqExpr", + type: DYN, + loc, + stmts: [ + { + kind: "varDecl", + localId: result.id, + init: { + kind: "libCall", + fn: "dyn.iteratorResult", + args: [ + awaitDyn({ + kind: "dynCall", + callee: next, + receiver: iterator, + calleeName: "iterator method", + args: [], + type: DYN, + loc, + }), + ], + type: DYN, + loc, + }, + loc, + }, + { + kind: "assign", + localId: done.id, + value: { kind: "dynTest", test: "truthy", value: get("done"), type: BOOL, loc }, + loc, + }, + ], + result: { + kind: "ternary", + cond: varRef(done.id, BOOL, loc), + then: dynUndefinedExpr(loc), + else_: { + kind: "ternary", + cond: sync, + then: awaitDyn(get("value")), + else_: get("value"), + type: DYN, + loc, + }, + type: DYN, + loc, + }, + }; +} + function lowerForOfString( lowerer: Lowerer, stmt: ts.ForOfStatement, diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index 0f144ca2e..af9e37198 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -546,6 +546,7 @@ async function compileExecutableNative( dynAsync: features.dynAsync, ...(features.dynTier ? { dynTier: true } : {}), ...(features.streamDyn ? { streamDyn: true } : {}), + ...(features.dynForAwait ? { dynForAwait: true } : {}), ...(features.eventsOnceSignal ? { eventsOnceSignal: true } : {}), events: features.events, emitter: features.emitter, diff --git a/packages/compiler/src/ir/builtin-effects.ts b/packages/compiler/src/ir/builtin-effects.ts index d4e96a35a..36d8613b0 100644 --- a/packages/compiler/src/ir/builtin-effects.ts +++ b/packages/compiler/src/ir/builtin-effects.ts @@ -627,6 +627,7 @@ export const MAY_THROW_LIB_FNS: ReadonlySet = new Set([ "dyn.destructureRest", "dyn.arrayFromIterator", "dyn.iterator", + "dyn.asyncIteratorOpen", "dyn.iteratorResult", "dyn.iteratorStep", "dyn.mapSeedEntries", diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index cd3db784d..a4903c5ac 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -219,6 +219,11 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ "dyn.destructureRest": { argTypes: [DYN, STRING], result: DYN }, "dyn.arrayFromIterator": { argTypes: [DYN], result: DYN }, "dyn.iterator": { argTypes: [DYN, STRING], result: DYN }, + /** for await over a checked-dynamic value: the loop's iterator (a + * compiled async generator, Symbol.asyncIterator's result, or the sync + * iterator), and whether that iterator is synchronous. */ + "dyn.asyncIteratorOpen": { argTypes: [DYN, STRING], result: DYN }, + "dyn.asyncIteratorWasSync": { argTypes: [], result: BOOL }, "dyn.iteratorResult": { argTypes: [DYN], result: DYN }, "dyn.iteratorCanStep": { argTypes: [DYN, DYN], result: BOOL }, "dyn.iteratorStep": { argTypes: [DYN], result: DYN }, diff --git a/packages/compiler/src/ir/runtime-features.ts b/packages/compiler/src/ir/runtime-features.ts index fd7c98cf6..b875e144d 100644 --- a/packages/compiler/src/ir/runtime-features.ts +++ b/packages/compiler/src/ir/runtime-features.ts @@ -573,6 +573,8 @@ const RUNTIME_FEATURE_RULES = { streamDyn: rule({ fns: ["stream.dynNative", "stream.dynNativePipe", "stream.pipeDyn", "stream.unpipeDyn"], }), + /** for await over checked-dynamic values (scr_dyn_for_await.c). */ + dynForAwait: rule({ fns: ["dyn.asyncIteratorOpen", "dyn.asyncIteratorWasSync"] }), /** The stream unit emits through the emitter unit, also when the * program constructs no stream (a pipeline of functions). */ emitter: rule({ @@ -756,6 +758,7 @@ function noRuntimeFeatures(): RuntimeFeatures { stdioExtended: false, stream: false, streamDyn: false, + dynForAwait: false, symbol: false, tls: false, tlsCa: false, diff --git a/packages/compiler/src/ir/validate.ts b/packages/compiler/src/ir/validate.ts index f75f6f46e..a581a6ff9 100644 --- a/packages/compiler/src/ir/validate.ts +++ b/packages/compiler/src/ir/validate.ts @@ -1622,10 +1622,10 @@ function validateFunction( "while", ]); if (e.generatorDelegate) { - if (!fn.generator || fn.async) - err("delegation sequence outside a synchronous generator", e.loc); + if (!fn.generator) err("delegation sequence outside a generator", e.loc); allowed.add("return"); allowed.add("throw"); + allowed.add("rethrow"); allowed.add("runtimeFence"); allowed.add("tryCatch"); } diff --git a/packages/compiler/src/native/sanitizer.test.ts b/packages/compiler/src/native/sanitizer.test.ts index 7607034b2..9856dd101 100644 --- a/packages/compiler/src/native/sanitizer.test.ts +++ b/packages/compiler/src/native/sanitizer.test.ts @@ -78,6 +78,7 @@ test.skipIf(process.platform !== "linux" && process.platform !== "darwin")( wasm: false, dynTier: false, streamDyn: false, + dynForAwait: false, eventsOnceSignal: false, warningReplace: false, stdioExtended: false, diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6580-npm-dyn-for-await/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6580-npm-dyn-for-await/main.ts.json new file mode 100644 index 000000000..b2c76d5aa --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6580-npm-dyn-for-await/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6580-npm-dyn-for-await/main.ts" + ], + "diags": [] +} diff --git a/packages/runtime-pack-common/runtime-pack-matrix.mjs b/packages/runtime-pack-common/runtime-pack-matrix.mjs index 258df8dec..0aa8804d8 100644 --- a/packages/runtime-pack-common/runtime-pack-matrix.mjs +++ b/packages/runtime-pack-common/runtime-pack-matrix.mjs @@ -116,6 +116,8 @@ export const OPTIONAL_RUNTIME_SOURCES = [ ["scr_stream_ops.c", all("stream", "dynAsync")], // Checked-dynamic stream slots call JavaScript members through the dyn-invoke unit. ["scr_stream_dyn.c", "streamDyn"], + // for await over checked-dynamic values. + ["scr_dyn_for_await.c", "dynForAwait"], // fs.ReadStream / fs.WriteStream reference only the stream unit and base units. ["scr_fs_stream.c", "stream"], // FileHandle streams reference both the stream unit and FileHandle. diff --git a/packages/runtime/src/scr_dyn.h b/packages/runtime/src/scr_dyn.h index 401bf2b39..fbc3b0155 100644 --- a/packages/runtime/src/scr_dyn.h +++ b/packages/runtime/src/scr_dyn.h @@ -722,6 +722,9 @@ bool scr_dyn_arr_has_index(const ScrDyn *arr, size_t index); void scr_dyn_arr_push_hole(ScrDyn *arr); ScrDyn *scr_dyn_new_obj(void); ScrDyn *scr_dyn_iterator(const ScrDyn *value, const ScrStr *spell); +/* for await over a checked-dynamic value (scr_dyn_for_await.c). */ +ScrDyn *scr_dyn_async_iterator_open(ScrDyn *value, ScrStr *spell); +bool scr_dyn_async_iterator_was_sync(void); ScrDyn *scr_dyn_iterator_optional(const ScrDyn *value); /* from.js's probe: 2 when value[Symbol.asyncIterator] is set, 1 when * value[Symbol.iterator] is, 0 for neither, -1 with an error pending. */ diff --git a/packages/runtime/src/scr_dyn_for_await.c b/packages/runtime/src/scr_dyn_for_await.c new file mode 100644 index 000000000..a0dbe07a6 --- /dev/null +++ b/packages/runtime/src/scr_dyn_for_await.c @@ -0,0 +1,45 @@ +/* `for await` over a checked-dynamic value (the compiler's lowerForOfDyn in + * its async mode). The iterator is a compiled async generator itself, the + * value's Symbol.asyncIterator method's result, or, as for any for-await, + * its synchronous iterator, whose values the loop then awaits. Linked only + * by programs with such a loop. */ +#include "scr_runtime.h" + +#include + +static SCR_TL bool scr_for_await_sync; + +/* The loop's iterator (+1), or NULL with the exception pending: V8's + * TypeError (`spell`) for a value that is neither async nor sync iterable. + * scr_dyn_async_iterator_was_sync answers which protocol it follows. */ +ScrDyn *scr_dyn_async_iterator_open(ScrDyn *value, ScrStr *spell) { + scr_for_await_sync = false; + if (value->kind == SCR_DYN_TYPED_REF && value->v.typed_ref.type_key) { + const char *key = value->v.typed_ref.type_key; + if (!strncmp(key, "async-generator<", 16)) return scr_dyn_retain(value); + if (!strncmp(key, "generator<", 10)) { + scr_for_await_sync = true; + return scr_dyn_retain(value); + } + } + int kind = 0; + ScrDyn *iterator = scr_dyn_from_iterator(value, &kind); + if (kind < 0) return NULL; + if (!iterator) { + if (spell->len > 0) { + scr_throw_error_msg(SCR_ERR_TYPE, spell->data, spell->len); + } else { + static const char message[] = "object is not async iterable"; + scr_throw_error_msg(SCR_ERR_TYPE, message, sizeof message - 1); + } + return NULL; + } + scr_for_await_sync = kind == 1; + return iterator; +} + +/* Whether the iterator the last scr_dyn_async_iterator_open returned is a + * synchronous one. Read immediately after the open. */ +bool scr_dyn_async_iterator_was_sync(void) { + return scr_for_await_sync; +} diff --git a/tests/corpus/6580-npm-dyn-for-await/main.ts b/tests/corpus/6580-npm-dyn-for-await/main.ts new file mode 100644 index 000000000..888cad7b8 --- /dev/null +++ b/tests/corpus/6580-npm-dyn-for-await/main.ts @@ -0,0 +1,25 @@ +// for await over untyped values in a JavaScript dependency, and yield* in +// async generators: a recursive async generator the checker types `any`, +// custom async iterators (return() on break and on a throw), synchronous +// iterables of promises, delegation with its completion value, and the +// not-iterable TypeError. +import { collect, protocols } from 'async-walk'; + +const tree = { root: ['a.func', 'nested'], 'root/nested': ['b.func', 'skip.txt', 'deep'], 'root/nested/deep': ['c.func'] }; +console.log((await collect('root', tree)).join(' ')); +for (const line of await protocols()) console.log(line); + +async function* letters(): AsyncGenerator { + yield 'a'; + yield 'b'; + return 2; +} + +async function* typed(): AsyncGenerator { + const count = yield* letters(); + yield `count ${count}`; +} + +const seen: string[] = []; +for await (const value of typed()) seen.push(value); +console.log(seen.join(',')); diff --git a/tests/corpus/6580-npm-dyn-for-await/node_modules/async-walk/index.d.ts b/tests/corpus/6580-npm-dyn-for-await/node_modules/async-walk/index.d.ts new file mode 100644 index 000000000..c9f098c0c --- /dev/null +++ b/tests/corpus/6580-npm-dyn-for-await/node_modules/async-walk/index.d.ts @@ -0,0 +1,2 @@ +export declare function collect(dir: string, entries: Record): Promise; +export declare function protocols(): Promise; diff --git a/tests/corpus/6580-npm-dyn-for-await/node_modules/async-walk/index.js b/tests/corpus/6580-npm-dyn-for-await/node_modules/async-walk/index.js new file mode 100644 index 000000000..a07fe1c99 --- /dev/null +++ b/tests/corpus/6580-npm-dyn-for-await/node_modules/async-walk/index.js @@ -0,0 +1,84 @@ +// for await over untyped values in JavaScript: a recursive async generator +// (its return type is `any` to the checker, as in @vercel/build-utils' +// createFunctionsIterator), custom async iterables, synchronous iterables +// of promises, and yield* inside async generators. +'use strict'; +var SUFFIX = '.func'; +async function* walkTree(dir, root = dir, entries) { + for (let name of entries[dir] || []) { + let abs = dir + '/' + name; + name.endsWith(SUFFIX) ? yield abs.slice(root.length + 1, abs.length - SUFFIX.length) : yield* walkTree(abs, root, entries); + } +} + +async function collect(dir, entries) { + var out = []; + for await (let path of walkTree(dir, dir, entries)) out.push(path); + return out; +} + +function countdown(n, log) { + return { + [Symbol.asyncIterator]() { + var left = n; + return { + next() { + log.push('next'); + return Promise.resolve(left > 0 ? { value: left--, done: false } : { value: undefined, done: true }); + }, + return() { + log.push('return'); + return Promise.resolve({ value: undefined, done: true }); + }, + }; + }, + }; +} + +function* syncValues() { + yield Promise.resolve('g1'); + yield 'g2'; +} + +async function* delegating(items) { + var tail = yield* inner(); + yield* items; + yield 'tail:' + tail; +} + +async function* inner() { + yield 'inner-a'; + yield 'inner-b'; + return 'done'; +} + +async function protocols() { + var out = []; + var log = []; + for await (var n of countdown(3, log)) out.push('n' + n); + out.push(log.join(',')); + log.length = 0; + for await (var m of countdown(5, log)) { + if (m === 4) break; + out.push('m' + m); + } + out.push(log.join(',')); + log.length = 0; + try { + for await (var k of countdown(2, log)) throw new Error('body ' + k); + } catch (error) { + out.push(error.message, log.join(',')); + } + for await (var p of [Promise.resolve('p1'), Promise.resolve('p2')]) out.push(p); + for await (var g of syncValues()) out.push(g); + for await (var d of delegating(['x', Promise.resolve('y')])) out.push(d); + var notIterable = 42; + try { + for await (var z of notIterable) out.push(z); + } catch (error) { + out.push(error.name + ': ' + error.message); + } + return out; +} + +module.exports = { collect, protocols }; diff --git a/tests/corpus/6580-npm-dyn-for-await/node_modules/async-walk/package.json b/tests/corpus/6580-npm-dyn-for-await/node_modules/async-walk/package.json new file mode 100644 index 000000000..76cc23577 --- /dev/null +++ b/tests/corpus/6580-npm-dyn-for-await/node_modules/async-walk/package.json @@ -0,0 +1 @@ +{ "name": "async-walk", "version": "1.0.0", "main": "index.js", "types": "index.d.ts" } From 2f50141c0c15415b4287454d4c76678cd0a78c4e Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Tue, 20 Oct 2026 14:55:59 -0700 Subject: [PATCH 19/38] Compile the hot-shots StatsD client over UDP hot-shots, a StatsD client compiled from its JavaScript, failed when a program constructed a client. Each gap on its UDP path is closed generically: - Sloppy-mode functions with named parameters may read `arguments` where the aliasing of their parameters cannot show: only `arguments.length`, or element reads without parameter or element writes. - An `any` slot admits any boxable function value, including functions with optional parameters (`C.prototype.m = function (...)` through an untyped receiver). - In a JavaScript file TypeScript does not check, a prototype method takes its documented parameters as checked-dynamic values, as constructors already did. - JavaScript dgram sends run completion callbacks, buffer lists, connected sends and the six-argument offset/length form. - A dgram socket crosses into the checked-dynamic tree as a handle (SCR_DYNH_DGRAM_SOCKET) whose methods are function values, with untyped listeners that survive 'close'. The handle lives in its own runtime unit (scr_dgram_dyn.c, the dgramDyn feature), linked only when a program converts a socket to or from an untyped value. - `String.prototype.concat` on strings, and `parseInt` over a possibly missing or untyped value in JavaScript. Corpus programs 6590-6596 and the hot-shots-udp npm case (hot-shots 8.5.0 vendored) match Node in both lanes. Co-Authored-By: Claude Opus 5.5 --- .../language-features/function-values.mdx | 3 + .../limitations/udp-socket-options.mdx | 7 + .../src/backend/executable-features.ts | 1 + packages/compiler/src/backend/library-pack.ts | 1 + packages/compiler/src/backend/llvm/dyn.ts | 24 +- .../src/backend/native-link-info-core.ts | 2 + .../compiler/src/backend/native/contracts.ts | 4 + .../compiler/src/backend/native/executable.ts | 1 + .../src/backend/runtime-pack-targets.test.ts | 1 + .../compiler/src/backend/runtime-pack.test.ts | 1 + packages/compiler/src/backend/warm-cache.ts | 1 + .../src/executable/executable-cache.test.ts | 1 + .../src/executable/executable-cache.ts | 2 + .../compiler/src/frontend/arguments-usage.ts | 82 +++ .../src/frontend/lowering/call-signatures.ts | 58 +- .../lowering/containers/string-and-regexp.ts | 8 + .../lowering/expressions/narrowing.ts | 15 + .../lowering/expressions/property-access.ts | 39 + .../src/frontend/lowering/lower-calls.ts | 42 +- .../src/frontend/lowering/lower-dgram.ts | 50 +- packages/compiler/src/index.ts | 1 + .../compiler/src/ir/builtin-signatures.ts | 8 +- packages/compiler/src/ir/ir.ts | 1 + packages/compiler/src/ir/runtime-features.ts | 83 +++ packages/compiler/src/native/cli.ts | 1 + packages/compiler/src/native/prepare.ts | 1 + .../compiler/src/native/sanitizer.test.ts | 1 + ...590-sloppy-arguments-unobservable.cjs.json | 6 + .../6591-function-values-any-slot.cjs.json | 6 + .../corpus/6592-dgram-send-callbacks.cjs.json | 6 + .../6593-dgram-socket-bound-methods.cjs.json | 6 + ...jsdoc-untrusted-prototype-methods.cjs.json | 6 + .../tests/corpus/6595-string-concat.ts.json | 6 + .../6596-parseint-missing-text.cjs.json | 6 + .../npm/cases/hot-shots-udp/main.ts.json | 6 + .../runtime-pack-matrix.mjs | 2 + packages/runtime/src/scr_dgram.c | 214 ++++-- packages/runtime/src/scr_dgram.h | 14 +- packages/runtime/src/scr_dgram_dyn.c | 664 ++++++++++++++++++ packages/runtime/src/scr_dgram_private.h | 47 ++ packages/runtime/src/scr_dyn.h | 1 + .../6590-sloppy-arguments-unobservable.cjs | 42 ++ .../corpus/6591-function-values-any-slot.cjs | 39 + tests/corpus/6592-dgram-send-callbacks.cjs | 41 ++ .../6593-dgram-socket-bound-methods.cjs | 61 ++ ...6594-jsdoc-untrusted-prototype-methods.cjs | 43 ++ tests/corpus/6595-string-concat.ts | 9 + tests/corpus/6596-parseint-missing-text.cjs | 8 + .../fixtures/npm/cases/hot-shots-udp/main.ts | 101 +++ .../npm/cases/hot-shots-udp/package.json | 1 + .../npm/node_modules/hot-shots/CHANGES.md | 314 +++++++++ .../npm/node_modules/hot-shots/LICENSE | 18 + .../npm/node_modules/hot-shots/README.md | 300 ++++++++ .../npm/node_modules/hot-shots/index.js | 1 + .../node_modules/hot-shots/lib/constants.js | 35 + .../npm/node_modules/hot-shots/lib/helpers.js | 118 ++++ .../hot-shots/lib/statsFunctions.js | 349 +++++++++ .../npm/node_modules/hot-shots/lib/statsd.js | 598 ++++++++++++++++ .../node_modules/hot-shots/lib/transport.js | 177 +++++ .../npm/node_modules/hot-shots/package.json | 47 ++ .../npm/node_modules/hot-shots/types.d.ts | 147 ++++ .../self-hosting-native-toolchain.test.ts | 1 + .../harness/self-hosting-runtime-pack.test.ts | 1 + 63 files changed, 3783 insertions(+), 97 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6590-sloppy-arguments-unobservable.cjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6591-function-values-any-slot.cjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6592-dgram-send-callbacks.cjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6593-dgram-socket-bound-methods.cjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6594-jsdoc-untrusted-prototype-methods.cjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6595-string-concat.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6596-parseint-missing-text.cjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/hot-shots-udp/main.ts.json create mode 100644 packages/runtime/src/scr_dgram_dyn.c create mode 100644 packages/runtime/src/scr_dgram_private.h create mode 100644 tests/corpus/6590-sloppy-arguments-unobservable.cjs create mode 100644 tests/corpus/6591-function-values-any-slot.cjs create mode 100644 tests/corpus/6592-dgram-send-callbacks.cjs create mode 100644 tests/corpus/6593-dgram-socket-bound-methods.cjs create mode 100644 tests/corpus/6594-jsdoc-untrusted-prototype-methods.cjs create mode 100644 tests/corpus/6595-string-concat.ts create mode 100644 tests/corpus/6596-parseint-missing-text.cjs create mode 100644 tests/fixtures/npm/cases/hot-shots-udp/main.ts create mode 100644 tests/fixtures/npm/cases/hot-shots-udp/package.json create mode 100644 tests/fixtures/npm/node_modules/hot-shots/CHANGES.md create mode 100644 tests/fixtures/npm/node_modules/hot-shots/LICENSE create mode 100644 tests/fixtures/npm/node_modules/hot-shots/README.md create mode 100644 tests/fixtures/npm/node_modules/hot-shots/index.js create mode 100644 tests/fixtures/npm/node_modules/hot-shots/lib/constants.js create mode 100644 tests/fixtures/npm/node_modules/hot-shots/lib/helpers.js create mode 100644 tests/fixtures/npm/node_modules/hot-shots/lib/statsFunctions.js create mode 100644 tests/fixtures/npm/node_modules/hot-shots/lib/statsd.js create mode 100644 tests/fixtures/npm/node_modules/hot-shots/lib/transport.js create mode 100644 tests/fixtures/npm/node_modules/hot-shots/package.json create mode 100644 tests/fixtures/npm/node_modules/hot-shots/types.d.ts diff --git a/docs/content/limitations/language-features/function-values.mdx b/docs/content/limitations/language-features/function-values.mdx index 75229ff95..7083f865d 100644 --- a/docs/content/limitations/language-features/function-values.mdx +++ b/docs/content/limitations/language-features/function-values.mdx @@ -2,3 +2,6 @@ A function stored in a slot with a different signature remains the same function for equality, array searches, `length`, and own properties. Typed rest-parameter functions can be stored and called indirectly, including through a slot with fixed parameters, where the slot's arguments arrive in the rest array. Builtin function values have API-specific restrictions; see [Builtin function values](#builtin-function-values). +- **JavaScript parameters:** In a JavaScript file TypeScript does not check, a prototype method (`C.prototype.m = function (...) {}`) takes whatever values its callers pass; its JSDoc types document the usual values without constraining them. + A sloppy-mode function with named parameters may read `arguments` only where the aliasing of its parameters cannot show: it reads `arguments.length`, or reads elements and never writes a parameter or an element. + Other uses of `arguments` in such a function are refused with `SC1090`; ES modules, `'use strict'` code, classes, and functions with default, rest, or destructured parameters have no such restriction. diff --git a/docs/content/limitations/udp-socket-options.mdx b/docs/content/limitations/udp-socket-options.mdx index 1b5a368a8..cfba0559d 100644 --- a/docs/content/limitations/udp-socket-options.mdx +++ b/docs/content/limitations/udp-socket-options.mdx @@ -1,2 +1,9 @@ `socket.addMembership()` and `socket.dropMembership()` on a `dgram` socket that is not bound yet throw `SC2020`; Node binds such a socket to an ephemeral port first, so bind the socket before joining a group. The `SystemError` that the buffer size methods throw (`ERR_SOCKET_BUFFER_SIZE`, for example from `getRecvBufferSize()` before `bind()`) has Node's `message`, `code`, `errno` and `syscall`, but no `info` object. + +In JavaScript, `socket.send()` also accepts lists of buffers, sends on a connected socket, and takes a completion callback, which receives `null` and the byte count, or the send's failure instead of an `'error'` event. +The callback runs after the socket's pending events, where Node runs it on the next tick. +In TypeScript, `send()` with a callback, a list, or on a connected socket fails to compile with `SC2020`. +JavaScript can read a socket's `on`, `once`, `addListener`, `removeListener`, `off`, `emit`, `send`, `close`, `bind`, `connect`, `disconnect`, `address`, `ref`, and `unref` as function values, for example `socket.emit.bind(socket)`; reading other members as values is refused with `SC1090`. +Listeners registered that way, or with untyped parameters, take only the `message`, `error`, `close`, `listening`, and `connect` events. +When a socket closes, listeners registered from TypeScript are released, so an `emit()` from JavaScript after `'close'` reaches only listeners registered from JavaScript. diff --git a/packages/compiler/src/backend/executable-features.ts b/packages/compiler/src/backend/executable-features.ts index 0fcf86ada..5242368fd 100644 --- a/packages/compiler/src/backend/executable-features.ts +++ b/packages/compiler/src/backend/executable-features.ts @@ -43,6 +43,7 @@ export function executableLinkFeatures(mod: IrModule): NativeLinkFeatures { http2: features.http2, dgram: features.dgram, dnsPromises: features.dnsPromises, + dgramDyn: features.dgramDyn, watch: features.fsWatch, foreignFfi: hasForeignFfiCallback(mod.ffiImports ?? []), nodeTest: features.nodeTest, diff --git a/packages/compiler/src/backend/library-pack.ts b/packages/compiler/src/backend/library-pack.ts index cb929e4b0..25df30733 100644 --- a/packages/compiler/src/backend/library-pack.ts +++ b/packages/compiler/src/backend/library-pack.ts @@ -51,6 +51,7 @@ export async function compilePackedLibrary( http2: false, dgram: false, dnsPromises: false, + dgramDyn: false, watch: false, foreignFfi: false, nodeTest: false, diff --git a/packages/compiler/src/backend/llvm/dyn.ts b/packages/compiler/src/backend/llvm/dyn.ts index 731385825..b59b7e3e7 100644 --- a/packages/compiler/src/backend/llvm/dyn.ts +++ b/packages/compiler/src/backend/llvm/dyn.ts @@ -85,6 +85,7 @@ export const DYN_HANDLE_TAG_NUM: Record = { child: 16, fileHandle: 17, dir: 44, + dgramSocket: 62, cryptoHash: 52, cryptoHmac: 53, }; @@ -95,6 +96,15 @@ export const DYN_HANDLE_TAG_NUM: Record = { const DYN_HANDLE_BOX_FN: Record = { cryptoHash: "scr_crypto_hash_to_dyn", cryptoHmac: "scr_crypto_hash_to_dyn", + dgramSocket: "scr_dgram_to_dyn", +}; + +/** Handle kinds whose box holds a unit-owned wrapper rather than the + * handle itself (a dgram socket's box carries its untyped listeners): + * the unit's own function unwraps it, with the tag check and TypeError + * of scr_dyn_handle_unbox. */ +const DYN_HANDLE_UNBOX_FN: Record = { + dgramSocket: "scr_dgram_from_dyn", }; export const DYN_KIND = { @@ -2449,11 +2459,17 @@ export class LlDyn { // TypeError on any other kind or tag). const h = DYN_HANDLE_KINDS.get(t.kind); if (h) { - host.declare(`declare ptr @scr_dyn_handle_unbox(ptr, i32, ptr, ptr)`); const r = B.tmp(); - B.line( - `${r} = call ptr @scr_dyn_handle_unbox(ptr %d, i32 ${DYN_HANDLE_TAG_NUM[t.kind]}, ptr %path, ptr ${want})`, - ); + const unbox = DYN_HANDLE_UNBOX_FN[t.kind]; + if (unbox) { + host.declare(`declare ptr @${unbox}(ptr, ptr, ptr)`); + B.line(`${r} = call ptr @${unbox}(ptr %d, ptr %path, ptr ${want})`); + } else { + host.declare(`declare ptr @scr_dyn_handle_unbox(ptr, i32, ptr, ptr)`); + B.line( + `${r} = call ptr @scr_dyn_handle_unbox(ptr %d, i32 ${DYN_HANDLE_TAG_NUM[t.kind]}, ptr %path, ptr ${want})`, + ); + } B.terminate(`ret ptr ${r}`); break; } diff --git a/packages/compiler/src/backend/native-link-info-core.ts b/packages/compiler/src/backend/native-link-info-core.ts index 84f448e93..ee366fa2b 100644 --- a/packages/compiler/src/backend/native-link-info-core.ts +++ b/packages/compiler/src/backend/native-link-info-core.ts @@ -65,6 +65,8 @@ export interface NativeLinkFeatures { dgram: boolean; /** node:dns/promises: scr_dns_promises.c. */ dnsPromises: boolean; + /** dgram sockets as untyped values: scr_dgram_dyn.c. */ + dgramDyn: boolean; watch: boolean; foreignFfi: boolean; nodeTest: boolean; diff --git a/packages/compiler/src/backend/native/contracts.ts b/packages/compiler/src/backend/native/contracts.ts index a9c93d3df..635c76293 100644 --- a/packages/compiler/src/backend/native/contracts.ts +++ b/packages/compiler/src/backend/native/contracts.ts @@ -198,6 +198,10 @@ export interface CcOptions { * compiles scr_dns_promises.c over the dgram unit, so callback-only * binaries keep their exact link line. */ dnsPromises?: boolean; + /** A dgram socket converts to or from an untyped value: compiles + * scr_dgram_dyn.c over the dgram unit, so typed dgram binaries keep + * their exact link line. */ + dgramDyn?: boolean; /** The program uses fs.watch (moduleUsesFsWatch on the IR): compiles * scr_watch.c into the binary — the net gating precedent, so watch-free diff --git a/packages/compiler/src/backend/native/executable.ts b/packages/compiler/src/backend/native/executable.ts index 949f03839..53f34a784 100644 --- a/packages/compiler/src/backend/native/executable.ts +++ b/packages/compiler/src/backend/native/executable.ts @@ -281,6 +281,7 @@ const RUNTIME_UNITS: readonly RuntimeUnit[] = [ unit((u) => u.opts.http2, "scr_http2.c"), unit((u) => u.opts.dgram, "scr_dgram.c"), unit((u) => u.opts.dgram && u.opts.dnsPromises, "scr_dns_promises.c"), + unit((u) => u.opts.dgram && u.opts.dgramDyn, "scr_dgram_dyn.c"), unit((u) => u.opts.watch, "scr_watch.c"), unit((u) => u.opts.foreignFfi, "scr_ffi_queue.c"), unit((u) => u.opts.nodeTest, "scr_test.c"), diff --git a/packages/compiler/src/backend/runtime-pack-targets.test.ts b/packages/compiler/src/backend/runtime-pack-targets.test.ts index 96503e2db..7c53ba9a2 100644 --- a/packages/compiler/src/backend/runtime-pack-targets.test.ts +++ b/packages/compiler/src/backend/runtime-pack-targets.test.ts @@ -116,6 +116,7 @@ const features = { http2: false, dgram: false, dnsPromises: false, + dgramDyn: false, watch: false, foreignFfi: false, nodeTest: false, diff --git a/packages/compiler/src/backend/runtime-pack.test.ts b/packages/compiler/src/backend/runtime-pack.test.ts index cd342f85b..a18c1bf82 100644 --- a/packages/compiler/src/backend/runtime-pack.test.ts +++ b/packages/compiler/src/backend/runtime-pack.test.ts @@ -53,6 +53,7 @@ const BASE: NativeLinkFeatures = { http2: false, dgram: false, dnsPromises: false, + dgramDyn: false, watch: false, foreignFfi: false, nodeTest: false, diff --git a/packages/compiler/src/backend/warm-cache.ts b/packages/compiler/src/backend/warm-cache.ts index 389fa19e9..2d623a550 100644 --- a/packages/compiler/src/backend/warm-cache.ts +++ b/packages/compiler/src/backend/warm-cache.ts @@ -59,6 +59,7 @@ export async function warmNativeCaches( http2: false, dgram: false, dnsPromises: false, + dgramDyn: false, watch: false, foreignFfi: false, nodeTest: false, diff --git a/packages/compiler/src/executable/executable-cache.test.ts b/packages/compiler/src/executable/executable-cache.test.ts index 33da0610f..b121957e5 100644 --- a/packages/compiler/src/executable/executable-cache.test.ts +++ b/packages/compiler/src/executable/executable-cache.test.ts @@ -54,6 +54,7 @@ const native: EarlyExecutableNativeFeatures = { http2: false, dgram: false, dnsPromises: false, + dgramDyn: false, watch: false, foreignFfi: false, nodeTest: false, diff --git a/packages/compiler/src/executable/executable-cache.ts b/packages/compiler/src/executable/executable-cache.ts index 064b1f867..375c5e6aa 100644 --- a/packages/compiler/src/executable/executable-cache.ts +++ b/packages/compiler/src/executable/executable-cache.ts @@ -83,6 +83,7 @@ export interface EarlyExecutableNativeFeatures { http2: boolean; dgram: boolean; dnsPromises: boolean; + dgramDyn: boolean; watch: boolean; foreignFfi: boolean; nodeTest: boolean; @@ -204,6 +205,7 @@ const BOOLEAN_NATIVE_KEYS = [ "http2", "dgram", "dnsPromises", + "dgramDyn", "watch", "foreignFfi", "nodeTest", diff --git a/packages/compiler/src/frontend/arguments-usage.ts b/packages/compiler/src/frontend/arguments-usage.ts index 317599f4c..9b9908171 100644 --- a/packages/compiler/src/frontend/arguments-usage.ts +++ b/packages/compiler/src/frontend/arguments-usage.ts @@ -33,3 +33,85 @@ export function bodyReadsArguments(fn: ts.Node): boolean { }); return found; } + +/** Whether an expression is assigned, updated or deleted. */ +function isWritten(expr: ts.Node): boolean { + let node = expr; + while (ts.isParenthesizedExpression(node.parent)) node = node.parent; + const parent = node.parent; + if (ts.isBinaryExpression(parent)) + return ( + parent.left === node && + parent.operatorToken.kind >= ts.SyntaxKind.FirstAssignment && + parent.operatorToken.kind <= ts.SyntaxKind.LastAssignment + ); + if (ts.isPrefixUnaryExpression(parent) || ts.isPostfixUnaryExpression(parent)) + return ( + parent.operator === ts.SyntaxKind.PlusPlusToken || + parent.operator === ts.SyntaxKind.MinusMinusToken + ); + if (ts.isForInStatement(parent) || ts.isForOfStatement(parent)) + return parent.initializer === node; + if ( + ts.isArrayLiteralExpression(parent) || + ts.isSpreadElement(parent) || + ts.isShorthandPropertyAssignment(parent) || + (ts.isPropertyAssignment(parent) && parent.initializer === node) + ) + return isWritten(parent); + if (ts.isObjectLiteralExpression(parent) || ts.isSpreadAssignment(parent)) + return isWritten(parent); + return ts.isDeleteExpression(parent); +} + +/** Whether sloppy-mode `arguments` in `fn`'s own body behaves exactly like + * an unmapped copy of the call's arguments. A mapped arguments object + * aliases the named parameters, so it differs from a copy only when one + * side is written after the other is read: reading `arguments.length` + * never observes the aliasing, and element reads cannot observe it while + * nothing writes an element or a parameter. The object itself must not + * escape, since another holder could write through it. */ +export function argumentsAliasingUnobservable(fn: ts.SignatureDeclaration): boolean { + if (fn.body === undefined) return false; + let elementReads = false; + let escapes = false; + ts.walkPreorder(fn.body, (node) => { + if (ts.isTypeNode(node)) return "skip"; + if (ts.isFunctionLike(node) && !ts.isArrowFunction(node) && node !== fn) return "skip"; + if (!ts.isIdentifier(node) || node.text !== "arguments" || !isArgumentsRead(node)) + return undefined; + const parent = node.parent; + if ( + ts.isPropertyAccessExpression(parent) && + parent.expression === node && + parent.name.text === "length" && + !isWritten(parent) + ) + return undefined; + if (ts.isElementAccessExpression(parent) && parent.expression === node && !isWritten(parent)) { + elementReads = true; + return undefined; + } + escapes = true; + return "stop"; + }); + if (escapes) return false; + if (!elementReads) return true; + const names = new Set(); + for (const p of fn.parameters.nodes) { + if (!ts.isIdentifier(p.name)) return false; + names.add(p.name.text); + } + // Any write to a parameter's name, including from nested closures and + // shadowing bindings, conservatively counts as a parameter write. + let paramWritten = false; + ts.walkPreorder(fn.body, (node) => { + if (ts.isTypeNode(node)) return "skip"; + if (ts.isIdentifier(node) && names.has(node.text) && isWritten(node)) { + paramWritten = true; + return "stop"; + } + return undefined; + }); + return !paramWritten; +} diff --git a/packages/compiler/src/frontend/lowering/call-signatures.ts b/packages/compiler/src/frontend/lowering/call-signatures.ts index 47cdd834f..daeab9d44 100644 --- a/packages/compiler/src/frontend/lowering/call-signatures.ts +++ b/packages/compiler/src/frontend/lowering/call-signatures.ts @@ -5,9 +5,10 @@ import { } from "./typed-arguments.js"; import { InternalCompilerError } from "../../errors.js"; import * as ts from "../ts7/adapter.js"; -import { bodyReadsArguments } from "../arguments-usage.js"; +import { argumentsAliasingUnobservable, bodyReadsArguments } from "../arguments-usage.js"; import type { Lowerer } from "./lowerer.js"; import { + canBoxFuncIntoDyn, DYN, type IrExpr, type IrFunction, @@ -143,6 +144,26 @@ export function generatorMeta( return { yieldT: type.yieldT, nextT: type.nextT, resultType }; } +/** A function expression stored as a prototype method: `C.prototype.name = + * function (...) {}` or `C.prototype[key] = function (...) {}` + * (parentheses allowed around the function). */ +export function prototypeAssignedFunction(fn: ts.Node | undefined): boolean { + if (fn === undefined || !ts.isFunctionExpression(fn)) return false; + let node: ts.Node = fn; + while (node.parent !== undefined && ts.isParenthesizedExpression(node.parent)) node = node.parent; + const parent = node.parent; + if ( + parent === undefined || + !ts.isBinaryExpression(parent) || + parent.right !== node || + parent.operatorToken.kind !== ts.SyntaxKind.EqualsToken || + !(ts.isPropertyAccessExpression(parent.left) || ts.isElementAccessExpression(parent.left)) + ) + return false; + const owner = parent.left.expression; + return ts.isPropertyAccessExpression(owner) && owner.name.text === "prototype"; +} + /** A parameter with a type annotation or a JSDoc `@param`/`@type` tag. */ export function documentedParameter(param: ts.ParameterDeclaration): boolean { return ( @@ -199,6 +220,20 @@ export function paramShape(lowerer: Lowerer, param: ts.ParameterDeclaration): Pa ? { type: DYN, mode: "omittable", bodyType: DYN } : { type: DYN, mode: param.questionToken ? "omittable" : "required" }; } + if ( + isJsSourceFile(ts.getSourceFileOfNode(param)) && + !param.dotDotDotToken && + prototypeAssignedFunction(param.parent) && + !checkedJsDocumentedParameter(lowerer, param) + ) { + // A prototype method (`C.prototype.m = function (a) {}`) is called + // through dynamic dispatch, where callers pass whatever they hold. As + // for constructors, JSDoc in a file TypeScript does not check + // documents the usual types without constraining callers. + return param.initializer + ? { type: DYN, mode: "omittable", bodyType: DYN } + : { type: DYN, mode: param.questionToken ? "omittable" : "required" }; + } const executor = param.parent; const construction = executor?.parent; const nativeExecutor = @@ -640,6 +675,21 @@ export function requireExactArityValue( ? lowerer.mapTypeOf(lowerer.typeOf(contextual)) : null; if (mapped && typeEquals(mapped, funcType)) return; + // An 'any' slot (`C.prototype.m = function (a, b) {…}` through an + // untyped receiver) boxes the value into a checked-dynamic function: + // the box completes omitted arguments with undefined and checks every + // argument into the completed ABI, so any boxable signature passes. + if ( + target !== undefined && + (target.flags & ts.TypeFlags.Any) !== 0 && + canBoxFuncIntoDyn( + funcType, + (id) => lowerer.shapes.get(id), + (id) => lowerer.unions.get(id), + ) + ) { + return; + } // An object-literal METHOD has no contextual type of its own: the // literal's contextual type names the member slot it fills. An // overloaded member's slot holds its erased function, which spells the @@ -839,7 +889,8 @@ export function collectSignatureInner(lowerer: Lowerer, decl: ts.FunctionDeclara /** Arrows inherit arguments. Ordinary JS functions receive a hidden * array unless a declared rest parameter already owns that contract. * Class methods are strict; other parameterized forms require ES modules - * because sloppy-mode arguments alias named parameters. */ + * because sloppy-mode arguments alias named parameters, unless the body + * cannot observe that aliasing (argumentsAliasingUnobservable). */ function appendArgumentsParameter( lowerer: Lowerer, node: ts.SignatureDeclaration, @@ -864,7 +915,8 @@ function appendArgumentsParameter( if ( !classMethod && !isNodeEsmFile(ts.getSourceFileOfNode(node), lowerer.program) && - !unmappedArguments(node) + !unmappedArguments(node) && + !argumentsAliasingUnobservable(node) ) { lowerer.unsupported( "SC1090", diff --git a/packages/compiler/src/frontend/lowering/containers/string-and-regexp.ts b/packages/compiler/src/frontend/lowering/containers/string-and-regexp.ts index 50693e5e4..164ba1f50 100644 --- a/packages/compiler/src/frontend/lowering/containers/string-and-regexp.ts +++ b/packages/compiler/src/frontend/lowering/containers/string-and-regexp.ts @@ -1504,6 +1504,7 @@ function lowerLocaleCompareCall( /** String.prototype members (Annex B included) whose typed-receiver * lowering lives here. */ export const EXTRA_STRING_METHODS: ReadonlySet = new Set([ + "concat", "substr", "toLocaleLowerCase", "toLocaleUpperCase", @@ -1647,6 +1648,13 @@ function buildStringMethod( ): IrExpr | null { const loc = locOf(call); const node = (i: number): ts.Expression => call.arguments.nodes[i] ?? call; + // concat(...values): each argument's ToString, in order, after all of + // them evaluated. + if (name === "concat") { + return args + .map((arg, i) => stringArg(lowerer, arg, node(i))) + .reduce((left, right) => ({ kind: "strConcat", left, right, type: STRING, loc }), s); + } if (name === "substr") { const start = args[0] ? positionNumber(lowerer, args[0], numLit(0, loc), node(0), "substr start") diff --git a/packages/compiler/src/frontend/lowering/expressions/narrowing.ts b/packages/compiler/src/frontend/lowering/expressions/narrowing.ts index dec06fdd9..7c979b139 100644 --- a/packages/compiler/src/frontend/lowering/expressions/narrowing.ts +++ b/packages/compiler/src/frontend/lowering/expressions/narrowing.ts @@ -23,6 +23,7 @@ import { import { isJsSourceFile, locOf } from "../../program.js"; import { hasRuntimeStatics } from "../class-runtime-statics.js"; import { jsBindingHasOpenWrites } from "../lower-stmts.js"; +import { documentedParameter, prototypeAssignedFunction } from "../call-signatures.js"; import { narrowClassUnion, narrowStoredClassValue, structuralNarrowing } from "../class-unions.js"; import { checkedClassAssertion, checkedClassUnionAssertion } from "../class-assertions.js"; import { isOptionalProcessStreamProperty } from "../builtins/process.js"; @@ -480,6 +481,20 @@ export function maybeNarrow(lowerer: Lowerer, expr: IrExpr, node: ts.Node): IrEx ) ) return expr; + // A prototype method in a file TypeScript does not check takes its + // documented parameters as checked-dynamic values whatever their + // JSDoc says (paramShape); reads keep the actual value. + if ( + symbol && + !lowerer.checkedJsFile(ts.getSourceFileOfNode(node)) && + symbol.declarations.some( + (decl) => + ts.isParameter(decl) && + prototypeAssignedFunction(decl.parent) && + documentedParameter(decl), + ) + ) + return expr; if ( symbol && symbol.declarations.some( diff --git a/packages/compiler/src/frontend/lowering/expressions/property-access.ts b/packages/compiler/src/frontend/lowering/expressions/property-access.ts index 36ea0e8c2..f4e85f422 100644 --- a/packages/compiler/src/frontend/lowering/expressions/property-access.ts +++ b/packages/compiler/src/frontend/lowering/expressions/property-access.ts @@ -251,6 +251,25 @@ function isStatFsDynProperty(lowerer: Lowerer, expr: ts.PropertyAccessExpression ); } +/** The dgram.Socket methods its checked-dynamic handle answers as + * function values (scr_dgram.c's scr_dgram_dyn_method_names). */ +const DGRAM_METHOD_VALUES = new Set([ + "on", + "once", + "addListener", + "removeListener", + "off", + "emit", + "send", + "close", + "bind", + "connect", + "disconnect", + "address", + "ref", + "unref", +]); + export function lowerPropertyAccessExpr( lowerer: Lowerer, expr: ts.PropertyAccessExpression, @@ -260,6 +279,26 @@ export function lowerPropertyAccessExpr( const navigator = lowerNavigatorMember(lowerer, expr, NODE_COMPAT_VERSION); if (navigator) return navigator; } + // A dgram socket's method read as a value in JavaScript (`emit: + // socket.emit.bind(socket)`): the socket crosses as its checked-dynamic + // handle, whose methods are function values that dispatch on their + // receiver, as the prototype methods do. The receiver is a native + // socket whatever its declaration spells (the shipped types declare no + // EventEmitter members), so the name decides. + if ( + !expr.questionDotToken && + DGRAM_METHOD_VALUES.has(expr.name.text) && + isJsSourceFile(ts.getSourceFileOfNode(expr)) && + lowerer.mapTypeOf(lowerer.typeOf(expr.expression))?.kind === "dgramSocket" + ) { + return { + kind: "dynKeyGet", + value: lowerer.coerceToExpected(lowerer.lowerExpr(expr.expression), DYN), + key: { kind: "strLit", value: expr.name.text, type: STRING, loc }, + type: DYN, + loc, + }; + } // `fn.name` on a function parameter of a generic instance keyed by // its argument's static name (call-site specialization). { diff --git a/packages/compiler/src/frontend/lowering/lower-calls.ts b/packages/compiler/src/frontend/lowering/lower-calls.ts index b1b4086cd..567db5d91 100644 --- a/packages/compiler/src/frontend/lowering/lower-calls.ts +++ b/packages/compiler/src/frontend/lowering/lower-calls.ts @@ -9338,15 +9338,18 @@ function lowerObjLitGenericMethodCall( lowerer.checker.getCallSignatures(recvT).length > 0 ) { const value = tryLowerExpression(lowerer, access.expression); + // A member that lowers to a checked-dynamic value (a native + // handle's method read in JavaScript) is already the function the + // runtime binds or calls. if ( - value?.type.kind === "func" && - lowerer.dynConvertible(value.type) && + ((value?.type.kind === "func" && lowerer.dynConvertible(value.type)) || + value?.type.kind === "dyn") && !call.arguments.nodes.some(ts.isSpreadElement) ) { const loc = locOf(call); return { kind: "dynInvoke", - recv: { kind: "dynFrom", value, type: DYN, loc }, + recv: value.type.kind === "dyn" ? value : { kind: "dynFrom", value, type: DYN, loc }, method: name, calleeName: ts.getTextOfNode(access), args: call.arguments.nodes.map((arg) => lowerer.lowerExprExpecting(arg, DYN)), @@ -10218,7 +10221,33 @@ export function lowerStaticNumberGlobal( const radix: IrExpr = args[1] ? lowerer.lowerExprExpecting(args[1], F64) : { kind: "numLit", value: 0, type: F64, loc }; - const optional = optionalCallValue(lowerer, args[0]!); + let optional = optionalCallValue(lowerer, args[0]!); + let lowered: IrExpr | null = null; + // A possibly-missing or untyped text in JavaScript (`parseInt(process. + // env.PORT, 10)`): TypeScript rejects the call, Node parses the text + // of whatever arrives ("undefined"). + if (optional === null && isJsSourceFile(ts.getSourceFileOfNode(call))) { + const argT = lowerer.typeOf(args[0]!); + const parts = argT.isUnionType() ? ts.constituentTypes(argT) : [argT]; + if ( + parts.some( + (part) => + (part.flags & (ts.TypeFlags.Undefined | ts.TypeFlags.Null | ts.TypeFlags.Any)) !== 0, + ) + ) { + const raw = lowerer.lowerExpr(args[0]!); + if (raw.type.kind === "union") { + optional = raw; + lowered = raw; + } else { + const text: IrExpr = + raw.type.kind === "dyn" + ? { kind: "libCall", fn: "dyn.toStringCoerce", args: [raw], type: STRING, loc } + : lowerer.coerceInto(args[0]!, raw, STRING); + return { kind: "libCall", fn: "num.parseInt", args: [text, radix], type: F64, loc }; + } + } + } if (optional?.type.kind === "union") { const def = lowerer.unions.get(optional.type.unionId); const stringTag = def?.arms.findIndex((arm) => arm.kind === "string") ?? -1; @@ -10317,7 +10346,10 @@ export function lowerStaticNumberGlobal( return { kind: "call", callee: helper, args: [optional, radix], type: F64, loc }; } } - const s = lowerer.lowerExprExpecting(args[0]!, STRING); + const s = + lowered !== null + ? lowerer.coerceInto(args[0]!, lowered, STRING) + : lowerer.lowerExprExpecting(args[0]!, STRING); return { kind: "libCall", fn: "num.parseInt", args: [s, radix], type: F64, loc }; } // STATIC parseFloat/isFinite over exactly-typed arguments — diff --git a/packages/compiler/src/frontend/lowering/lower-dgram.ts b/packages/compiler/src/frontend/lowering/lower-dgram.ts index 4cb238352..201f02664 100644 --- a/packages/compiler/src/frontend/lowering/lower-dgram.ts +++ b/packages/compiler/src/frontend/lowering/lower-dgram.ts @@ -9,7 +9,6 @@ import { dynUndefinedExpr, boolLit } from "../../ir/build.js"; * generic rejection, never silence. */ import * as ts from "../ts7/adapter.js"; import type { Lowerer } from "./lowerer.js"; -import { ladderFenceExpr } from "./lowerer.js"; import { isJsSourceFile, locOf } from "../program.js"; import { BOOL, @@ -28,6 +27,7 @@ import { } from "../../ir/ir.js"; import { resultIsDiscarded } from "./call-position.js"; import { lowerCallbackArg as lowerCallbackArgShared } from "./callback-arg.js"; +import { documentedParameter } from "./call-signatures.js"; const DGRAM_SURFACE_HINT = "bind, connect, disconnect, send, address, close, unref/ref, the socket " + @@ -498,6 +498,17 @@ function lowerDnsModuleCall( ); } +/** A listener JavaScript leaves untyped: a function literal with a + * parameter that has neither an annotation nor JSDoc (paramShape makes it + * checked-dynamic), or any other untyped value. */ +function untypedJsListener(lowerer: Lowerer, node: ts.Expression): boolean { + let e = node; + while (ts.isParenthesizedExpression(e)) e = e.expression; + if (ts.isArrowFunction(e) || ts.isFunctionExpression(e)) + return e.parameters.nodes.some((param) => !documentedParameter(param)); + return (lowerer.typeOf(e).flags & ts.TypeFlags.Any) !== 0; +} + /** Method calls on dgram.Socket receivers — one entry in lower-calls.ts's * intrinsic chain (after lowerServerMethodCall). Null for other * receivers. */ @@ -553,9 +564,9 @@ export function lowerDgramMethodCall( // send(msg, port, address) — one datagram to an explicit destination // (the static fast path). Every OTHER shape in a JS source rides the // checked-dynamic ladder (dgram.sendChk): Node's signature shuffle, - // slice bounds, list/type contracts, port/address validation, and - // the connected-state errors — with the compiler-rendered fence as - // the post-validation tail for the callback/list/connected forms. + // slice bounds, list/type contracts, port/address validation, the + // connected-state errors, then the send itself — lists, connected + // sends and completion callbacks included. const staticShape = args.length === 3 && !args.some(ts.isSpreadElement) && @@ -579,13 +590,13 @@ export function lowerDgramMethodCall( } if ( isJsSourceFile(ts.getSourceFileOfNode(call)) && - args.length <= 5 && + args.length <= 6 && !args.some(ts.isSpreadElement) ) { const receiver = lowerer.lowerExpr(access.expression); const slots: IrExpr[] = []; let ok = true; - for (let i = 0; i < 5; i++) { + for (let i = 0; i < 6; i++) { const n = args[i]; if (!n) { slots.push(dynUndefinedExpr(loc)); @@ -613,16 +624,7 @@ export function lowerDgramMethodCall( return { kind: "libCall", fn: "dgram.sendChk", - args: [ - receiver, - ...slots, - ladderFenceExpr( - lowerer, - `send in this form`, - call, - "send(msg, port, address) — one string or Buffer datagram — is the lowered form; callback, list, and connected sends have no lowering yet", - ), - ], + args: [receiver, ...slots], type: VOID, loc, }; @@ -686,6 +688,22 @@ export function lowerDgramMethodCall( } if ((name === "on" || name === "once") && args.length === 2) { requireStatementPosition(lowerer, call, `socket.${name}(...)`); + // A JavaScript listener whose parameters are untyped registers through + // the socket's checked-dynamic handle, which hands it untyped values. + if (isJsSourceFile(ts.getSourceFileOfNode(call)) && untypedJsListener(lowerer, args[1]!)) { + return { + kind: "dynInvoke", + recv: lowerer.coerceToExpected(lowerer.lowerExpr(access.expression), DYN), + method: name, + calleeName: ts.getTextOfNode(access), + args: [ + lowerer.lowerExprExpecting(args[0]!, DYN), + lowerer.lowerExprExpecting(args[1]!, DYN), + ], + type: DYN, + loc, + }; + } const once = boolLit(name === "once", loc); const evT = lowerer.typeOf(args[0]!); const event = evT.isStringLiteralType() ? evT.value : null; diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index 0f144ca2e..845dbb3e2 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -562,6 +562,7 @@ async function compileExecutableNative( http2: features.http2, dgram: features.dgram, dnsPromises: features.dnsPromises, + dgramDyn: features.dgramDyn, watch: features.watch, foreignFfi: features.foreignFfi, ...(features.workers ? { workers: true } : {}), diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index cd3db784d..d65fa82d3 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -1507,10 +1507,10 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ "dgram.sendBytes": { argTypes: [DGRAMSOCK_T, BYTES_U8, F64, STRING], result: VOID }, /** The send argument-validation ladder over dyn arguments (Node's * signature shuffle: slice bounds, list/type contracts, port/address - * validation, connected-state errors) — a fully-validated unconnected - * single-payload send RUNS; callback/list/connected forms meet the - * trailing fence. May-throw. */ - "dgram.sendChk": { argTypes: [DGRAMSOCK_T, DYN, DYN, DYN, DYN, DYN, STRING], result: VOID }, + * validation, connected-state errors), then the send: lists, connected + * sends and completion callbacks included. Up to six arguments, the + * missing ones undefined. May-throw. */ + "dgram.sendChk": { argTypes: [DGRAMSOCK_T, DYN, DYN, DYN, DYN, DYN, DYN], result: VOID }, "dgram.address": { argTypes: [DGRAMSOCK_T], result: VOID }, "dgram.close": { argTypes: [DGRAMSOCK_T], result: VOID }, "dgram.closeCb": { diff --git a/packages/compiler/src/ir/ir.ts b/packages/compiler/src/ir/ir.ts index 4c5bbd183..b78f8f57a 100644 --- a/packages/compiler/src/ir/ir.ts +++ b/packages/compiler/src/ir/ir.ts @@ -4097,6 +4097,7 @@ export const DYN_HANDLE_KINDS: ReadonlyMap ["http2Session", { tag: "SCR_DYNH_H2_SESSION", cls: "Http2Session" }], ["http2Stream", { tag: "SCR_DYNH_H2_STREAM", cls: "Http2Stream" }], ["httpClientReq", { tag: "SCR_DYNH_HTTP_CLIENT", cls: "ClientRequest" }], + ["dgramSocket", { tag: "SCR_DYNH_DGRAM_SOCKET", cls: "Socket" }], ["dir", { tag: "SCR_DYNH_DIR", cls: "Dir" }], ["cryptoHash", { tag: "SCR_DYNH_HASH", cls: "Hash" }], ["cryptoHmac", { tag: "SCR_DYNH_HMAC", cls: "Hmac" }], diff --git a/packages/compiler/src/ir/runtime-features.ts b/packages/compiler/src/ir/runtime-features.ts index fd7c98cf6..1e88c81e8 100644 --- a/packages/compiler/src/ir/runtime-features.ts +++ b/packages/compiler/src/ir/runtime-features.ts @@ -321,6 +321,8 @@ interface TypeTables { records: Map; unions: Map; classes: Map; + /** mentionsDgramSocket's carriers, computed on first use. */ + dgramCarriers?: DgramCarriers; } /** Whether boxing a value of type `t` as dyn boxes a promise, which goes @@ -384,6 +386,78 @@ function boxesPromise(t: IrType, tables: TypeTables, seen: Set): boolean } } +/** The record shapes, unions and classes whose values carry a dgram socket + * (directly or through other members), computed once per module. */ +interface DgramCarriers { + records: Set; + unions: Set; + anyClass: boolean; +} + +/** Whether `t` carries a dgram socket, given the carrier sets so far: + * structural through functions (parameters and result), promises and + * collections; named records, unions and class instances by the sets. */ +function carriesDgram(t: IrType, carriers: DgramCarriers): boolean { + switch (t.kind) { + case "dgramSocket": + return true; + case "func": + return t.params.some((p) => carriesDgram(p, carriers)) || carriesDgram(t.ret, carriers); + case "promise": + return carriesDgram(t.inner, carriers); + case "array": + case "set": + return carriesDgram(t.elem, carriers); + case "map": + return carriesDgram(t.key, carriers) || carriesDgram(t.value, carriers); + case "record": + return carriers.records.has(t.shapeId); + case "union": + return carriers.unions.has(t.unionId); + case "object": + // Any class may be the instance's runtime class or one of its bases. + return carriers.anyClass; + default: + return false; + } +} + +/** The module's dgram carriers: a fixpoint over its records, unions and + * class fields. */ +function dgramCarriers(tables: TypeTables): DgramCarriers { + const carriers: DgramCarriers = { records: new Set(), unions: new Set(), anyClass: false }; + for (let changed = true; changed;) { + changed = false; + for (const [id, shape] of tables.records) + if (!carriers.records.has(id) && shape.fields.some((f) => carriesDgram(f.type, carriers))) { + carriers.records.add(id); + changed = true; + } + for (const [id, def] of tables.unions) + if (!carriers.unions.has(id) && def.arms.some((a) => carriesDgram(a, carriers))) { + carriers.unions.add(id); + changed = true; + } + if ( + !carriers.anyClass && + [...tables.classes.values()].some((c) => c.fields.some((f) => carriesDgram(f.type, carriers))) + ) { + carriers.anyClass = true; + changed = true; + } + } + return carriers; +} + +/** Whether converting a value of type `t` to or from dyn converts a dgram + * socket, which goes through scr_dgram_dyn.c: the socket itself, or a + * function, collection, record, union, promise or class instance that + * carries one (boxed functions convert their parameters too). */ +function mentionsDgramSocket(t: IrType, tables: TypeTables): boolean { + tables.dgramCarriers ??= dgramCarriers(tables); + return carriesDgram(t, tables.dgramCarriers); +} + /** The libCall fns served by the OPTIONAL events unit (scr_events.c): * process signal/exit listeners and the piped-stdin surface. */ const PROCESS_EVENT_LIB_FNS: ReadonlySet = new Set([ @@ -517,6 +591,14 @@ const RUNTIME_FEATURE_RULES = { dc: rule({ prefixes: ["dc."] }), /** dns.lookup lives in the dgram unit. */ dgram: rule({ prefixes: ["dgram.", "dns."], types: ["dgramSocket"] }), + /** A dgram socket converts to or from an untyped value (its handle box, + * a boxed function or record that carries one): scr_dgram_dyn.c over + * the dgram unit. */ + dgramDyn: rule({ + expr: (node, tables) => + (node.kind === "dynFrom" && mentionsDgramSocket(node.value.type, tables())) || + (node.kind === "dynCheck" && mentionsDgramSocket(node.type, tables())), + }), /** node:dns/promises: scr_dns_promises.c over the dgram unit. */ dnsPromises: rule({ fns: ["dns.promisesLookup", "dns.promisesLookupService"] }), dynAsync: rule({ @@ -727,6 +809,7 @@ function noRuntimeFeatures(): RuntimeFeatures { copying: false, dc: false, dgram: false, + dgramDyn: false, dnsPromises: false, dynAsync: false, dynBuiltinClass: false, diff --git a/packages/compiler/src/native/cli.ts b/packages/compiler/src/native/cli.ts index a9056a093..f04ccd5ef 100644 --- a/packages/compiler/src/native/cli.ts +++ b/packages/compiler/src/native/cli.ts @@ -98,6 +98,7 @@ async function main(): Promise { http2: false, dgram: false, dnsPromises: false, + dgramDyn: false, watch: false, foreignFfi: false, nodeTest: false, diff --git a/packages/compiler/src/native/prepare.ts b/packages/compiler/src/native/prepare.ts index a6b4be63c..1ec066d6c 100644 --- a/packages/compiler/src/native/prepare.ts +++ b/packages/compiler/src/native/prepare.ts @@ -152,6 +152,7 @@ function validFeatures(value: unknown): value is NativeLinkFeatures { "http2", "dgram", "dnsPromises", + "dgramDyn", "watch", "foreignFfi", "nodeTest", diff --git a/packages/compiler/src/native/sanitizer.test.ts b/packages/compiler/src/native/sanitizer.test.ts index 7607034b2..ccc10c5c1 100644 --- a/packages/compiler/src/native/sanitizer.test.ts +++ b/packages/compiler/src/native/sanitizer.test.ts @@ -107,6 +107,7 @@ test.skipIf(process.platform !== "linux" && process.platform !== "darwin")( http2: false, dgram: false, dnsPromises: false, + dgramDyn: false, watch: false, foreignFfi: false, nodeTest: false, diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6590-sloppy-arguments-unobservable.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6590-sloppy-arguments-unobservable.cjs.json new file mode 100644 index 000000000..c7d928dc9 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6590-sloppy-arguments-unobservable.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6590-sloppy-arguments-unobservable.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6591-function-values-any-slot.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6591-function-values-any-slot.cjs.json new file mode 100644 index 000000000..dea4a06d6 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6591-function-values-any-slot.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6591-function-values-any-slot.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6592-dgram-send-callbacks.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6592-dgram-send-callbacks.cjs.json new file mode 100644 index 000000000..8a5500154 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6592-dgram-send-callbacks.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6592-dgram-send-callbacks.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6593-dgram-socket-bound-methods.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6593-dgram-socket-bound-methods.cjs.json new file mode 100644 index 000000000..026270cff --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6593-dgram-socket-bound-methods.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6593-dgram-socket-bound-methods.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6594-jsdoc-untrusted-prototype-methods.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6594-jsdoc-untrusted-prototype-methods.cjs.json new file mode 100644 index 000000000..be5699183 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6594-jsdoc-untrusted-prototype-methods.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6594-jsdoc-untrusted-prototype-methods.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6595-string-concat.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6595-string-concat.ts.json new file mode 100644 index 000000000..525dedd47 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6595-string-concat.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6595-string-concat.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6596-parseint-missing-text.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6596-parseint-missing-text.cjs.json new file mode 100644 index 000000000..d1a35dce4 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6596-parseint-missing-text.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6596-parseint-missing-text.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/hot-shots-udp/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/hot-shots-udp/main.ts.json new file mode 100644 index 000000000..54baff890 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/hot-shots-udp/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/hot-shots-udp/main.ts" + ], + "diags": [] +} diff --git a/packages/runtime-pack-common/runtime-pack-matrix.mjs b/packages/runtime-pack-common/runtime-pack-matrix.mjs index 258df8dec..62bc0a198 100644 --- a/packages/runtime-pack-common/runtime-pack-matrix.mjs +++ b/packages/runtime-pack-common/runtime-pack-matrix.mjs @@ -129,6 +129,8 @@ export const OPTIONAL_RUNTIME_SOURCES = [ ["scr_dgram.c", "dgram"], // The dns promise forms run over the dgram unit's callback forms. ["scr_dns_promises.c", all("dgram", "dnsPromises")], + // Sockets as untyped values run over the dgram unit. + ["scr_dgram_dyn.c", all("dgram", "dgramDyn")], ["scr_watch.c", "watch"], ["scr_ffi_queue.c", "foreignFfi"], ["scr_test.c", "nodeTest"], diff --git a/packages/runtime/src/scr_dgram.c b/packages/runtime/src/scr_dgram.c index c34923607..00388b99c 100644 --- a/packages/runtime/src/scr_dgram.c +++ b/packages/runtime/src/scr_dgram.c @@ -62,6 +62,7 @@ #include "scr_platform.h" #include "scr_runtime.h" #include "scr_system_errors.h" +#include "scr_dgram_private.h" #include #include @@ -266,16 +267,6 @@ static const char *scr_dgram_errname(int err) { /* ── listener lists (the scr_net.c snapshot discipline, restated so this * unit links without scr_net.c) ─────────────────────────────────────── */ -typedef struct { - ScrClosure *cb; - void *fn; - bool once; -} ScrDgramL; - -typedef struct { - ScrDgramL *ls; - size_t n, cap; -} ScrDgramLs; static void scr_dgram_ls_add(ScrDgramLs *l, ScrClosure *cb, void *fn, bool once) { if (l->n == l->cap) { @@ -352,23 +343,6 @@ static void scr_dgram_fire_err(ScrDgramLs *l, ScrStr *msg) { /* ── the handle ──────────────────────────────────────────────────────── */ -struct ScrDgramSocket { - size_t rc; - int fd; /* -1 until bound/connected, -1 again once closed */ - bool reuse_addr; - bool bound; - bool connected; - bool closing; /* close() called; 'close' fires at the next sweep */ - bool close_emitted; /* settled: off the registry, listeners dropped */ - bool unrefed; /* waives loop liveness, Node's unref */ - bool read_armed; - bool emit_listening; - bool emit_connect; - ScrStr *pending_err; /* deferred bind/connect/send failure */ - ScrDgramLs msg_ls, err_ls, listening_ls, close_ls, conn_ls; - bool in_registry; - struct ScrDgramSocket *next; -}; #ifdef SCR_RC_AUDIT static SCR_TL long scr_dgram_live = 0; @@ -1905,9 +1879,10 @@ void scr_dgram_install(void) { * (ERR_SOCKET_BAD_PORT, > 0 and < 65536 with the specific-type tail and * Node's trailing period) and the address's string contract; a send with * a port or address on a connected socket answers ERR_SOCKET_DGRAM_IS_ - * CONNECTED. A fully-validated unconnected single-payload send RUNS — - * the callback form and the connected sends keep the compiler-rendered - * fence. All dyn arguments borrowed. */ + * CONNECTED. A validated send then runs: a list goes out as one + * datagram, a connected socket sends to its peer, and a completion + * callback receives (null, bytes) or the send failure after this turn + * (scr_dgram_send_out). All dyn arguments borrowed. */ static bool scr_dgram_dyn_truthy(const ScrDyn *v) { switch (v->kind) { @@ -1933,11 +1908,89 @@ static uint32_t scr_dgram_to_u32(const ScrDyn *v) { static const char SCR_DGRAM_BUF_EXPECTED[] = "of type string or an instance of Buffer, TypedArray, or DataView"; +/* A send failure as the Error Node hands a send callback: the message + * ("send ECONNREFUSED 127.0.0.1:41234", "getaddrinfo ENOTFOUND nohost") + * with the code its second word names. +1. */ +static ScrDyn *scr_dgram_send_error(const char *msg) { + ScrStr *text = scr_str_new(msg, strlen(msg)); + ScrError *e = scr_error_new(0 /* Error */, text); + scr_str_release(text); + const char *sp = strchr(msg, ' '); + if (sp != NULL) { + char code[48]; + size_t n = 0; + while (sp[1 + n] != '\0' && sp[1 + n] != ' ') n++; + if (n > 0 && n < sizeof code) { + memcpy(code, sp + 1, n); + code[n] = '\0'; + scr_error_set_code(e, code); + } + } + ScrDyn *d = scr_dyn_from_error(e); + scr_error_release(e); + return d; +} + +/* The ladder's send: Node's healthCheck, then one datagram — to the + * connected peer when host is NULL, else to host:port (an empty host is + * Node's 127.0.0.1). With a completion callback the result defers to the + * pending queue the dns deliveries share: (null, bytes) on success, the + * failure (which has no 'error' event then) otherwise; without one a + * failure is the deferred 'error', as scr_dgram_send_raw has it. */ +static void scr_dgram_send_out(ScrDgramSocket *s, const char *data, size_t len, double port, + const ScrStr *host, const ScrDyn *callback) { + if (s->closing || s->close_emitted) { + scr_dgram_throw("Not running"); + return; + } + bool has_cb = callback->kind == SCR_DYN_FUNC; + char msg[192]; + msg[0] = '\0'; + struct sockaddr_in a4; + const char *h = host == NULL ? NULL : host->len > 0 ? host->data : "127.0.0.1"; + if (h != NULL && !scr_dgram_host4(h, (uint16_t)(int)port, &a4)) { + snprintf(msg, sizeof msg, "getaddrinfo ENOTFOUND %s", h); + } else { + scr_dgram_ensure_fd(s); + if (!s->bound) { + /* Implicit ephemeral bind — Node's send-before-bind emits + * 'listening' for it. */ + s->bound = true; + s->emit_listening = true; + } + scr_dgram_register(s); /* an open sender holds the loop, like Node */ + ssize_t sent = h != NULL ? sendto(s->fd, data, len, 0, (struct sockaddr *)&a4, sizeof a4) + : send(s->fd, data, len, 0); + if (sent < 0 && errno != EAGAIN && errno != EWOULDBLOCK && errno != EINTR) { + const char *code = scr_dgram_errname(errno); + char ip[64] = ""; + if (h != NULL) inet_ntop(AF_INET, &a4.sin_addr, ip, sizeof ip); + snprintf(msg, sizeof msg, "send %s %s:%d", code, ip, (int)port); + /* A connected send names no address ("send ECONNREFUSED"): cut the + * message after the code (one format string keeps the unit's + * constant data unchanged). */ + if (h == NULL) msg[5 + strlen(code)] = '\0'; + } + scr_dgram_update_read(s); + } + if (!has_cb) { + if (msg[0] != '\0') scr_dgram_defer_err(s, msg); + return; + } + if (msg[0] != '\0') { + ScrDyn *args[1] = { scr_dgram_send_error(msg) }; + scr_dns_call_later(callback, args, 1, false); + } else { + ScrDyn *args[2] = { scr_dyn_new_null(), scr_dyn_new_num((double)len) }; + scr_dns_call_later(callback, args, 2, false); + } +} + void scr_dgram_send_chk(ScrDgramSocket *s, const ScrDyn *buffer, const ScrDyn *a1, const ScrDyn *a2, const ScrDyn *a3, const ScrDyn *a4, - const ScrStr *fence) { + const ScrDyn *a5) { const ScrDyn *offset = a1, *length = a2, *port = a3, *address = a4; - const ScrDyn *callback = scr_dyn_undefined(); + const ScrDyn *callback = a5; bool connected = s->connected; bool sliced = false; if (!connected) { @@ -1952,11 +2005,12 @@ void scr_dgram_send_chk(ScrDgramSocket *s, const ScrDyn *buffer, const ScrDyn *a } else { if (length->kind == SCR_DYN_NUM) { sliced = true; - if (port->kind == SCR_DYN_FUNC) callback = port; + if (port->kind == SCR_DYN_FUNC) { + callback = port; + port = scr_dyn_undefined(); + } } else { callback = offset; - port = a3; - address = a4; } } size_t slice_off = 0, slice_len = 0; @@ -1994,19 +2048,16 @@ void scr_dgram_send_chk(ScrDgramSocket *s, const ScrDyn *buffer, const ScrDyn *a scr_dyn_arg_type_fail("buffer", SCR_DGRAM_BUF_EXPECTED, buffer); return; } + double portnum = 0; if (connected) { if (scr_dgram_dyn_truthy(port) || scr_dgram_dyn_truthy(address)) { scr_dgram_throw("Already connected"); return; } - scr_throw_lowering_fence(fence); /* connected sends have no lowering yet */ - return; - } - /* validatePort(port, 'Port', false): integers (or numeric strings) - * strictly between 0 and 65536; everything else renders the specific - * type with Node's trailing period. */ - double portnum = -1; - { + } else { + /* validatePort(port, 'Port', false): integers (or numeric strings) + * strictly between 0 and 65536; everything else renders the specific + * type with Node's trailing period. */ bool ok = false; if (port->kind == SCR_DYN_NUM && trunc(port->v.num) == port->v.num && port->v.num > 0 && port->v.num < 65536) { @@ -2029,6 +2080,7 @@ void scr_dgram_send_chk(ScrDgramSocket *s, const ScrDyn *buffer, const ScrDyn *a return; } } + if (callback->kind != SCR_DYN_FUNC) callback = scr_dyn_undefined(); if (address->kind == SCR_DYN_FUNC) { callback = address; address = scr_dyn_undefined(); @@ -2038,26 +2090,64 @@ void scr_dgram_send_chk(ScrDgramSocket *s, const ScrDyn *buffer, const ScrDyn *a scr_dyn_arg_type_fail("address", "of type string", address); return; } - if (callback->kind == SCR_DYN_FUNC || buffer->kind == SCR_DYN_ARR) { - /* completion callbacks and list concatenation have no lowering yet — - * throw the SC2020 fence text after the full validation ladder */ - scr_throw_lowering_fence(fence); - return; - } - const ScrBytes *byte_input = buffer->kind == SCR_DYN_BYTES ? buffer->v.bytes : NULL; - SCR_BYTES_SNAPSHOT(byte_input); + /* The payload: a list concatenates into one datagram (sendmsg's + * gather), a slice narrows the single payload. */ + char *joined = NULL; char *owned = NULL; - const char *data = buffer->kind == SCR_DYN_STR ? scr_str_utf8(buffer->v.str, &owned) - : (const char *)byte_input->data; - size_t datalen = buffer->kind == SCR_DYN_STR ? buffer->v.str->len - : (size_t)scr_bytes_byte_len(buffer->v.bytes); - if (sliced) { - data += slice_off; - datalen = slice_len; + const char *data; + size_t datalen; + if (buffer->kind == SCR_DYN_ARR) { + size_t total = 0; + for (size_t i = 0; i < buffer->v.arr.len; i++) { + const ScrDyn *e = buffer->v.arr.items[i]; + total += e->kind == SCR_DYN_STR ? e->v.str->len : (size_t)scr_bytes_byte_len(e->v.bytes); + } + joined = malloc(total ? total : 1); + if (!joined) scr_dgram_oom(); + size_t at = 0; + for (size_t i = 0; i < buffer->v.arr.len; i++) { + const ScrDyn *e = buffer->v.arr.items[i]; + if (e->kind == SCR_DYN_STR) { + char *part_owned; + const char *part = scr_str_utf8(e->v.str, &part_owned); + memcpy(joined + at, part, e->v.str->len); + at += e->v.str->len; + scr_str_utf8_end(part_owned); + } else { + const ScrBytes *b = e->v.bytes; + SCR_BYTES_SNAPSHOT(b); + size_t n = (size_t)scr_bytes_byte_len(b); + if (n) memcpy(joined + at, b->data, n); + at += n; + } + } + data = joined; + datalen = total; + } else { + const ScrBytes *byte_input = buffer->kind == SCR_DYN_BYTES ? buffer->v.bytes : NULL; + SCR_BYTES_SNAPSHOT(byte_input); + data = buffer->kind == SCR_DYN_STR ? scr_str_utf8(buffer->v.str, &owned) + : (const char *)byte_input->data; + datalen = buffer->kind == SCR_DYN_STR ? buffer->v.str->len + : (size_t)scr_bytes_byte_len(buffer->v.bytes); + if (sliced) { + data += slice_off; + datalen = slice_len; + } + } + if (connected) { + scr_dgram_send_out(s, data, datalen, 0, NULL, callback); + } else { + ScrStr *host = address->kind == SCR_DYN_STR ? scr_str_retain(address->v.str) + : scr_str_new("127.0.0.1", 9); + scr_dgram_send_out(s, data, datalen, portnum, host, callback); + scr_str_release(host); } - ScrStr *host = address->kind == SCR_DYN_STR ? scr_str_retain(address->v.str) - : scr_str_new("127.0.0.1", 9); - scr_dgram_send_raw(s, data, datalen, portnum, host); scr_str_utf8_end(owned); - scr_str_release(host); + free(joined); } + +/* The read arming for the checked-dynamic unit (scr_dgram_dyn.c), after + * it removes a 'message' listener. Defined last so the functions above + * keep their emission order. */ +void scr_dgram_rearm(ScrDgramSocket *s) { scr_dgram_update_read(s); } diff --git a/packages/runtime/src/scr_dgram.h b/packages/runtime/src/scr_dgram.h index dac772c93..8396f0693 100644 --- a/packages/runtime/src/scr_dgram.h +++ b/packages/runtime/src/scr_dgram.h @@ -34,11 +34,19 @@ void scr_dgram_send_str(ScrDgramSocket *s, ScrStr *data /*borrowed*/, double por void scr_dgram_send_bytes(ScrDgramSocket *s, ScrBytes *data /*borrowed*/, double port, ScrStr *host /*borrowed*/); /* The send argument-validation ladder over dyn arguments (Node's * signature shuffle, slice bounds, list/type contracts, port/address - * validation, and the connected-state errors); a fully-validated - * unconnected single-payload send RUNS, the rest meet the fence. */ + * validation, and the connected-state errors), then the send: a list + * goes out as one datagram, a connected socket sends to its peer, and a + * completion callback receives (null, bytes) or the failure after this + * turn. */ void scr_dgram_send_chk(ScrDgramSocket *s, const ScrDyn *buffer, const ScrDyn *a1, const ScrDyn *a2, const ScrDyn *a3, const ScrDyn *a4, - const ScrStr *fence); + const ScrDyn *a5); +/* The socket as an untyped value (SCR_DYNH_DGRAM_SOCKET): boxes of one + * socket share its untyped listeners; the first box installs the tag's + * ops. Borrows s; +1. The unbox checks the tag and answers the socket + * (+1), or NULL with the path-annotated TypeError pending. */ +ScrDyn *scr_dgram_to_dyn(ScrDgramSocket *s); +ScrDgramSocket *scr_dgram_from_dyn(const ScrDyn *d, const ScrDynPath *path, const char *want); /* address() parts: ip THROWS "Not running" before bind/connect (+1 * otherwise); family/port are only called after ip succeeded. */ ScrStr *scr_dgram_addr_ip(ScrDgramSocket *s); /* +1, may throw */ diff --git a/packages/runtime/src/scr_dgram_dyn.c b/packages/runtime/src/scr_dgram_dyn.c new file mode 100644 index 000000000..b779ffccd --- /dev/null +++ b/packages/runtime/src/scr_dgram_dyn.c @@ -0,0 +1,664 @@ +/* node:dgram sockets as checked-dynamic values (SCR_DYNH_DGRAM_SOCKET): + * the handle ops, the untyped listener mirror and the method values. A + * unit of its own so programs that never box a socket link none of it + * (native-toolchain gating: the dgramDyn runtime feature). */ +#include "scr_platform.h" +#include "scr_runtime.h" +#include "scr_dgram_private.h" + +#include +#include +#include + +static void scr_dgram_dyn_oom(void) { + fputs("scriptc: out of memory\n", stderr); + abort(); +} + +/* The native lists' firing snapshot (scr_dgram.c's discipline): entries + * retained; once entries leave the live list before their callbacks run. */ +static size_t scr_dgram_dyn_snapshot(ScrDgramLs *l, ScrDgramL **out) { + size_t n = l->n; + if (n == 0) { + *out = NULL; + return 0; + } + ScrDgramL *snap = malloc(n * sizeof *snap); + if (!snap) scr_dgram_dyn_oom(); + for (size_t i = 0; i < n; i++) { + snap[i] = l->ls[i]; + scr_closure_retain(snap[i].cb); + } + size_t w = 0; + for (size_t i = 0; i < l->n; i++) { + if (l->ls[i].once) { + scr_closure_release(l->ls[i].cb); + } else { + l->ls[w++] = l->ls[i]; + } + } + l->n = w; + *out = snap; + return n; +} + +static void scr_dgram_dyn_ls_drop(ScrDgramLs *l) { + for (size_t i = 0; i < l->n; i++) scr_closure_release(l->ls[i].cb); + free(l->ls); + l->ls = NULL; + l->n = l->cap = 0; +} + +/* ── checked-dynamic dgram sockets (SCR_DYNH_DGRAM_SOCKET) ───────────── + * A socket crosses into untyped code (`socket.on.bind(socket)` in a + * package, an untyped binding) as a handle box over a refcounted entry + * that holds the socket and the untyped listeners, so the boxes of one + * socket share their identity and listener state. This unit links only + * when a program converts a socket to or from an untyped value. + * + * Listeners. An untyped listener lives in the entry's per-event list + * and as a proxy closure in the socket's own native list, so runtime + * events reach it in registration order beside typed listeners. The + * entry's list outlives the socket's settlement ('close' drops the + * native lists to break cycles): Node keeps listeners after close, and + * an `emit('error', e)` afterwards still reaches them. removeListener + * removes the most recent registration of the function, as Node does. + * + * emit(event, ...args) calls each listener with the given values; a + * typed 'error' listener receives the emitted Error (the payload rides + * scr_event_error_payload), and a typed 'message' listener cannot take + * untyped values, so that emit throws the not-supported Error. An 'error' + * nobody listens to throws the emitted value, as EventEmitter does. */ + +enum { + SCR_DGRAM_EV_MESSAGE, + SCR_DGRAM_EV_ERROR, + SCR_DGRAM_EV_CLOSE, + SCR_DGRAM_EV_LISTENING, + SCR_DGRAM_EV_CONNECT, + SCR_DGRAM_EV_COUNT +}; + +static const char *const scr_dgram_ev_names[SCR_DGRAM_EV_COUNT] = { + "message", "error", "close", "listening", "connect", +}; + +typedef struct { + ScrDyn *fn; /* +1 */ + bool once; + double id; /* the proxy closure's key */ +} ScrDgramDynL; + +typedef struct ScrDgramDyn { + size_t rc; + ScrDgramSocket *s; /* +1 */ + struct { + ScrDgramDynL *ls; + size_t n, cap; + } ev[SCR_DGRAM_EV_COUNT]; + bool cleaned; /* exit cleanup has released its listeners */ + struct ScrDgramDyn *next; /* the live-entry table (weak) */ +} ScrDgramDyn; + +static SCR_TL ScrDgramDyn *scr_dgram_dyns = NULL; +static SCR_TL double scr_dgram_dyn_next_id = 0; +static SCR_TL ScrDyn *scr_dgram_dyn_methods = NULL; /* method name -> function value */ + +static int scr_dgram_ev_of(const ScrDyn *name) { + if (name->kind != SCR_DYN_STR) return -1; + for (int i = 0; i < SCR_DGRAM_EV_COUNT; i++) { + size_t n = strlen(scr_dgram_ev_names[i]); + if (name->v.str->len == n && memcmp(name->v.str->data, scr_dgram_ev_names[i], n) == 0) return i; + } + return -1; +} + +static void scr_dgram_dyn_clear(ScrDgramDyn *e) { + for (int i = 0; i < SCR_DGRAM_EV_COUNT; i++) { + for (size_t j = 0; j < e->ev[i].n; j++) scr_dyn_release(e->ev[i].ls[j].fn); + free(e->ev[i].ls); + e->ev[i].ls = NULL; + e->ev[i].n = e->ev[i].cap = 0; + } +} + +static void *scr_dgram_dyn_retain_v(void *p) { + ScrDgramDyn *e = (ScrDgramDyn *)p; + e->rc++; + return e; +} + +static void scr_dgram_dyn_release_v(void *p) { + ScrDgramDyn *e = (ScrDgramDyn *)p; + if (--e->rc != 0) return; + ScrDgramDyn **link = &scr_dgram_dyns; + while (*link && *link != e) link = &(*link)->next; + if (*link) *link = e->next; + scr_dgram_dyn_clear(e); + scr_dgram_release(e->s); + free(e); +} + +/* The entry's listener at the proxy key, or -1 (removed meanwhile). */ +static long scr_dgram_dyn_find(ScrDgramDyn *e, int ev, double id) { + for (size_t i = 0; i < e->ev[ev].n; i++) { + if (e->ev[ev].ls[i].id == id) return (long)i; + } + return -1; +} + +/* Takes the listener out of the entry's list; its function moves to + * the caller (+1). */ +static ScrDyn *scr_dgram_dyn_take(ScrDgramDyn *e, int ev, size_t i) { + ScrDyn *fn = e->ev[ev].ls[i].fn; + memmove(&e->ev[ev].ls[i], &e->ev[ev].ls[i + 1], (e->ev[ev].n - i - 1) * sizeof *e->ev[ev].ls); + e->ev[ev].n--; + return fn; +} + +/* A proxy closure fired from the native list: its entry and key. The + * listener is +1 for the call (taken out first when it was once). */ +static ScrDyn *scr_dgram_dyn_proxy_fn(ScrClosure *cb, int ev) { + ScrDgramDyn *e = (ScrDgramDyn *)scr_box_get_ref(cb->caps[0]); + double id = scr_box_get_f64(cb->caps[1]); + long i = scr_dgram_dyn_find(e, ev, id); + ScrDyn *fn = NULL; + if (i >= 0) { + fn = e->ev[ev].ls[i].once ? scr_dgram_dyn_take(e, ev, (size_t)i) + : scr_dyn_retain(e->ev[ev].ls[i].fn); + } + scr_dgram_dyn_release_v(e); + return fn; +} + +static void scr_dgram_dyn_call(ScrDyn *fn, ScrDyn *const *args, size_t argc) { + if (fn == NULL) return; + ScrDyn *r = scr_dyn_call(fn, args, argc, "listener"); + scr_dyn_release(r); + scr_dyn_release(fn); +} + +static void scr_dgram_dyn_fire_message(ScrClosure *cb, ScrBytes *msg, ScrStr *addr, + ScrStr *family, double port, double size) { + ScrDyn *fn = scr_dgram_dyn_proxy_fn(cb, SCR_DGRAM_EV_MESSAGE); + if (fn == NULL) return; + ScrDyn *rinfo = scr_dyn_new_obj(); + scr_dyn_obj_set(rinfo, "address", 7, scr_dyn_new_str(addr)); + scr_dyn_obj_set(rinfo, "family", 6, scr_dyn_new_str(family)); + scr_dyn_obj_set(rinfo, "port", 4, scr_dyn_new_num(port)); + scr_dyn_obj_set(rinfo, "size", 4, scr_dyn_new_num(size)); + ScrDyn *args[2] = { scr_dyn_new_bytes(msg), rinfo }; + scr_dgram_dyn_call(fn, args, 2); + scr_dyn_release(args[0]); + scr_dyn_release(rinfo); +} + +/* A runtime 'error' (msg) or an emitted one (scr_event_error_payload). */ +static void scr_dgram_dyn_fire_error(ScrClosure *cb, ScrStr *msg) { + ScrDyn *fn = scr_dgram_dyn_proxy_fn(cb, SCR_DGRAM_EV_ERROR); + if (fn == NULL) return; + ScrDyn *err; + if (scr_event_error_payload != NULL) { + err = scr_dyn_from_error(scr_event_error_payload); + } else { + ScrError *e = scr_error_new(0 /* Error */, msg); + err = scr_dyn_from_error(e); + scr_error_release(e); + } + ScrDyn *args[1] = { err }; + scr_dgram_dyn_call(fn, args, 1); + scr_dyn_release(err); +} + +static void scr_dgram_dyn_fire_close(ScrClosure *cb) { + scr_dgram_dyn_call(scr_dgram_dyn_proxy_fn(cb, SCR_DGRAM_EV_CLOSE), NULL, 0); +} +static void scr_dgram_dyn_fire_listening(ScrClosure *cb) { + scr_dgram_dyn_call(scr_dgram_dyn_proxy_fn(cb, SCR_DGRAM_EV_LISTENING), NULL, 0); +} +static void scr_dgram_dyn_fire_connect(ScrClosure *cb) { + scr_dgram_dyn_call(scr_dgram_dyn_proxy_fn(cb, SCR_DGRAM_EV_CONNECT), NULL, 0); +} + +static ScrDgramLs *scr_dgram_native_list(ScrDgramSocket *s, int ev) { + switch (ev) { + case SCR_DGRAM_EV_MESSAGE: return &s->msg_ls; + case SCR_DGRAM_EV_ERROR: return &s->err_ls; + case SCR_DGRAM_EV_CLOSE: return &s->close_ls; + case SCR_DGRAM_EV_LISTENING: return &s->listening_ls; + default: return &s->conn_ls; + } +} + +/* Whether a native list entry is an untyped listener's proxy. */ +static bool scr_dgram_is_proxy(const ScrDgramL *l) { + void *fn = l->cb->fn; + return fn == (void *)&scr_dgram_dyn_fire_message || fn == (void *)&scr_dgram_dyn_fire_error || + fn == (void *)&scr_dgram_dyn_fire_close || fn == (void *)&scr_dgram_dyn_fire_listening || + fn == (void *)&scr_dgram_dyn_fire_connect; +} + +static void scr_dgram_dyn_on(ScrDgramDyn *e, int ev, const ScrDyn *fn, bool once) { + if (e->ev[ev].n == e->ev[ev].cap) { + e->ev[ev].cap = e->ev[ev].cap ? e->ev[ev].cap * 2 : 2; + e->ev[ev].ls = realloc(e->ev[ev].ls, e->ev[ev].cap * sizeof *e->ev[ev].ls); + if (!e->ev[ev].ls) scr_dgram_dyn_oom(); + } + double id = ++scr_dgram_dyn_next_id; + e->ev[ev].ls[e->ev[ev].n].fn = scr_dyn_retain((ScrDyn *)fn); + e->ev[ev].ls[e->ev[ev].n].once = once; + e->ev[ev].ls[e->ev[ev].n].id = id; + e->ev[ev].n++; + ScrDgramSocket *s = e->s; + if (s->close_emitted) return; /* settled: nothing fires natively any more */ + static void *const fires[SCR_DGRAM_EV_COUNT] = { + (void *)&scr_dgram_dyn_fire_message, (void *)&scr_dgram_dyn_fire_error, + (void *)&scr_dgram_dyn_fire_close, (void *)&scr_dgram_dyn_fire_listening, + (void *)&scr_dgram_dyn_fire_connect, + }; + ScrClosure *proxy = scr_closure_new(fires[ev], 2); + proxy->caps[0] = scr_box_new_obj(&scr_dgram_dyn_retain_v, &scr_dgram_dyn_release_v, NULL); + scr_box_set_ref(proxy->caps[0], scr_dgram_dyn_retain_v(e)); + proxy->caps[1] = scr_box_new(SCR_BOX_F64); + scr_box_set_f64(proxy->caps[1], id); + switch (ev) { + case SCR_DGRAM_EV_MESSAGE: + scr_dgram_on_message(s, proxy, &scr_dgram_dyn_fire_message, once); + break; + case SCR_DGRAM_EV_ERROR: + scr_dgram_on_error(s, proxy, &scr_dgram_dyn_fire_error, once); + break; + case SCR_DGRAM_EV_CLOSE: + scr_dgram_on_close(s, proxy, once); + break; + case SCR_DGRAM_EV_LISTENING: + scr_dgram_on_listening(s, proxy, once); + break; + default: + scr_dgram_on_connect(s, proxy, once); + break; + } +} + +/* removeListener(event, fn): the most recent registration of fn leaves + * the entry's list and its proxy leaves the native one. */ +static void scr_dgram_dyn_off(ScrDgramDyn *e, int ev, const ScrDyn *fn) { + if (fn->kind != SCR_DYN_FUNC) return; + for (size_t k = e->ev[ev].n; k-- > 0;) { + const ScrDyn *have = e->ev[ev].ls[k].fn; + if (have->kind != SCR_DYN_FUNC || + !scr_closure_identity_equal(have->v.fn.clo, fn->v.fn.clo)) { + continue; + } + double id = e->ev[ev].ls[k].id; + scr_dyn_release(scr_dgram_dyn_take(e, ev, k)); + ScrDgramLs *l = scr_dgram_native_list(e->s, ev); + for (size_t i = 0; i < l->n; i++) { + ScrClosure *c = l->ls[i].cb; + if (!scr_dgram_is_proxy(&l->ls[i]) || scr_box_get_f64(c->caps[1]) != id) continue; + scr_closure_release(c); + memmove(&l->ls[i], &l->ls[i + 1], (l->n - i - 1) * sizeof *l->ls); + l->n--; + break; + } + if (ev == SCR_DGRAM_EV_MESSAGE) scr_dgram_rearm(e->s); + return; + } +} + +/* Node's ERR_INVALID_ARG_TYPE listener gate (scr_dyn_check_listener's, + * restated so this unit links without the handle support unit). */ +static bool scr_dgram_dyn_listener_ok(const ScrDyn *cb) { + if (cb->kind == SCR_DYN_FUNC) return true; + scr_dyn_arg_type_fail("listener", "of type function", cb); + return false; +} + +static void scr_dgram_dynh_unsupported(const char *member, const char *why) { + char msg[200]; + int n = snprintf(msg, sizeof msg, + "'dgram.Socket.prototype.%s' on a dynamic value is not supported yet%s%s", member, + why ? " — " : "", why ? why : ""); + scr_throw_error_msg(SCR_ERR_ERROR, msg, (size_t)n); +} + +/* The message text a typed 'error' listener receives for an emitted + * value: an Error's message, any other value's string. +1, or NULL with + * the conversion's exception pending. */ +static ScrStr *scr_dgram_emit_text(const ScrDyn *v) { + ScrError *err = scr_errdyn_err_of(v); + if (err != NULL) { + ScrStr *m = scr_str_retain(err->message); + scr_error_release(err); + return m; + } + return scr_dyn_to_string(v, NULL); +} + +/* emit(event, ...args): true when the event had listeners. */ +static ScrDyn *scr_dgram_dyn_emit(ScrDgramDyn *e, ScrDyn *const *args, size_t argc) { + const ScrDyn *name = argc > 0 ? args[0] : scr_dyn_undefined(); + ScrDyn *const *rest = argc > 1 ? args + 1 : NULL; + size_t nrest = argc > 1 ? argc - 1 : 0; + int ev = scr_dgram_ev_of(name); + ScrDgramSocket *s = e->s; + bool live = ev >= 0 && !s->close_emitted; + size_t count = ev < 0 ? 0 : live ? scr_dgram_native_list(s, ev)->n : e->ev[ev].n; + if (count == 0) { + bool is_error = name->kind == SCR_DYN_STR && name->v.str->len == 5 && + memcmp(name->v.str->data, "error", 5) == 0; + if (!is_error) return scr_dyn_new_bool(false); + const ScrDyn *er = nrest > 0 ? rest[0] : scr_dyn_undefined(); + ScrError *err = scr_errdyn_err_of(er); + if (err != NULL) { + scr_error_release(err); + scr_dyn_throw(scr_dyn_retain((ScrDyn *)er)); + return NULL; + } + char shown[120], msg[200]; + int n = snprintf(msg, sizeof msg, "Unhandled error. (%s)", + scr_dyn_inspect_lite(er, shown, sizeof shown)); + scr_throw_error_msg_code(SCR_ERR_ERROR, msg, (size_t)n, "ERR_UNHANDLED_ERROR"); + return NULL; + } + if (!live) { + /* Settled: only the untyped listeners remain (the native lists were + * dropped with the typed ones). */ + size_t n = e->ev[ev].n; + ScrDgramDynL *snap = malloc(n * sizeof *snap); + if (!snap) scr_dgram_dyn_oom(); + for (size_t i = 0; i < n; i++) { + snap[i] = e->ev[ev].ls[i]; + scr_dyn_retain(snap[i].fn); + } + for (size_t i = e->ev[ev].n; i-- > 0;) { + if (e->ev[ev].ls[i].once) scr_dyn_release(scr_dgram_dyn_take(e, ev, i)); + } + for (size_t i = 0; i < n; i++) { + if (!scr_exc_pending()) { + ScrDyn *r = scr_dyn_call(snap[i].fn, rest, nrest, "listener"); + scr_dyn_release(r); + } + scr_dyn_release(snap[i].fn); + } + free(snap); + return scr_exc_pending() ? NULL : scr_dyn_new_bool(true); + } + ScrDgramL *snap; + size_t n = scr_dgram_dyn_snapshot(scr_dgram_native_list(s, ev), &snap); + ScrError *payload = NULL; + ScrStr *text = NULL; + for (size_t i = 0; i < n; i++) { + ScrClosure *cb = snap[i].cb; + if (scr_exc_pending()) { + scr_closure_release(cb); + continue; + } + if (scr_dgram_is_proxy(&snap[i])) { + scr_dgram_dyn_call(scr_dgram_dyn_proxy_fn(cb, ev), rest, nrest); + } else if (ev == SCR_DGRAM_EV_ERROR) { + const ScrDyn *er = nrest > 0 ? rest[0] : scr_dyn_undefined(); + if (text == NULL) { + payload = scr_errdyn_err_of(er); + text = scr_dgram_emit_text(er); + } + if (text != NULL) { + ScrError *saved = scr_event_error_payload; + scr_event_error_payload = payload; + ((ScrChildErrFn)snap[i].fn)(cb, text); + scr_event_error_payload = saved; + } + } else if (ev == SCR_DGRAM_EV_MESSAGE) { + scr_dgram_dynh_unsupported("emit", "a typed 'message' listener cannot receive untyped values"); + } else { + ((void (*)(ScrClosure *))cb->fn)(cb); + } + scr_closure_release(cb); + } + free(snap); + if (payload != NULL) scr_error_release(payload); + scr_str_release(text); + if (ev == SCR_DGRAM_EV_MESSAGE) scr_dgram_rearm(s); + return scr_exc_pending() ? NULL : scr_dyn_new_bool(true); +} + +/* address() as Node's { address, family, port } object. */ +static ScrDyn *scr_dgram_dyn_address(ScrDgramSocket *s) { + ScrStr *ip = scr_dgram_addr_ip(s); + if (ip == NULL) return NULL; + ScrDyn *o = scr_dyn_new_obj(); + scr_dyn_obj_set(o, "address", 7, scr_dyn_new_str(ip)); + scr_str_release(ip); + ScrStr *family = scr_dgram_addr_family(s); + scr_dyn_obj_set(o, "family", 6, scr_dyn_new_str(family)); + scr_str_release(family); + scr_dyn_obj_set(o, "port", 4, scr_dyn_new_num(scr_dgram_addr_port(s))); + return o; +} + +/* bind/connect over positional untyped arguments: (port[, address][, + * callback]); the options-object forms meet the not-supported Error. */ +static bool scr_dgram_dyn_bind_args(const char *member, ScrDyn *const *args, size_t argc, + double *port, ScrStr **host, const ScrDyn **cb) { + size_t i = 0; + *port = 0; + *host = NULL; + *cb = NULL; + if (i < argc && args[i]->kind == SCR_DYN_NUM) *port = args[i++]->v.num; + else if (i < argc && args[i]->kind == SCR_DYN_OBJ) { + scr_dgram_dynh_unsupported(member, "the options-object form is not modeled"); + return false; + } else if (i < argc && (args[i]->kind == SCR_DYN_UNDEF || args[i]->kind == SCR_DYN_NULL)) { + i++; + } + if (i < argc && args[i]->kind == SCR_DYN_STR) *host = args[i++]->v.str; + if (i < argc && args[i]->kind == SCR_DYN_FUNC) *cb = args[i++]; + if (i < argc && args[i]->kind != SCR_DYN_UNDEF) { + scr_dgram_dynh_unsupported(member, "the modeled shape is (port[, address][, callback])"); + return false; + } + return true; +} + +static ScrDyn *scr_dgram_dynh_invoke(void *h, ScrDyn *self, const char *method, + ScrDyn *const *args, size_t argc, const char *what) { + ScrDgramDyn *e = (ScrDgramDyn *)h; + ScrDgramSocket *s = e->s; + const ScrDyn *a0 = argc > 0 ? args[0] : scr_dyn_undefined(); + const ScrDyn *a1 = argc > 1 ? args[1] : scr_dyn_undefined(); + bool once = strcmp(method, "once") == 0; + if (once || strcmp(method, "on") == 0 || strcmp(method, "addListener") == 0) { + if (!scr_dgram_dyn_listener_ok(a1)) return NULL; + int ev = scr_dgram_ev_of(a0); + if (ev < 0) { + ScrJsonBuf b; + scr_jb_init(&b); + scr_jb_puts(&b, "listening for '"); + if (a0->kind == SCR_DYN_STR) scr_jb_write(&b, a0->v.str->data, a0->v.str->len); + scr_jb_puts(&b, "' on a dynamic dgram Socket is not supported yet"); + scr_throw_error(SCR_ERR_ERROR, scr_jb_finish(&b)); + return NULL; + } + scr_dgram_dyn_on(e, ev, a1, once); + return scr_dyn_retain(self); + } + if (strcmp(method, "removeListener") == 0 || strcmp(method, "off") == 0) { + if (!scr_dgram_dyn_listener_ok(a1)) return NULL; + int ev = scr_dgram_ev_of(a0); + if (ev >= 0) scr_dgram_dyn_off(e, ev, a1); + return scr_dyn_retain(self); + } + if (strcmp(method, "emit") == 0) return scr_dgram_dyn_emit(e, args, argc); + if (strcmp(method, "send") == 0) { + const ScrDyn *slot[6]; + for (size_t i = 0; i < 6; i++) slot[i] = i < argc ? args[i] : scr_dyn_undefined(); + scr_dgram_send_chk(s, slot[0], slot[1], slot[2], slot[3], slot[4], slot[5]); + return scr_exc_pending() ? NULL : scr_dyn_retain(scr_dyn_undefined()); + } + if (strcmp(method, "close") == 0) { + /* Node registers a function callback with on('close') before its + * state check, so a second close still registers it, then throws. */ + if (a0->kind == SCR_DYN_FUNC) scr_dgram_dyn_on(e, SCR_DGRAM_EV_CLOSE, a0, false); + scr_dgram_close(s, NULL); + return scr_exc_pending() ? NULL : scr_dyn_retain(self); + } + if (strcmp(method, "bind") == 0 || strcmp(method, "connect") == 0) { + double port; + ScrStr *host; + const ScrDyn *cb; + if (!scr_dgram_dyn_bind_args(method, args, argc, &port, &host, &cb)) return NULL; + bool bind = method[0] == 'b'; + ScrStr *h2 = host ? scr_str_retain(host) : bind ? scr_str_new("", 0) : scr_str_new("127.0.0.1", 9); + /* the callback is once('listening') / once('connect'), registered + * before the state checks as Node registers it */ + if (cb) scr_dgram_dyn_on(e, bind ? SCR_DGRAM_EV_LISTENING : SCR_DGRAM_EV_CONNECT, cb, true); + if (bind) scr_dgram_bind(s, port, h2, NULL); + else scr_dgram_connect(s, port, h2, NULL); + scr_str_release(h2); + if (scr_exc_pending()) return NULL; + return bind ? scr_dyn_retain(self) : scr_dyn_retain(scr_dyn_undefined()); + } + if (strcmp(method, "disconnect") == 0) { + scr_dgram_disconnect(s); + return scr_exc_pending() ? NULL : scr_dyn_retain(scr_dyn_undefined()); + } + if (strcmp(method, "address") == 0) return scr_dgram_dyn_address(s); + if (strcmp(method, "ref") == 0 || strcmp(method, "unref") == 0) { + if (method[0] == 'r') scr_dgram_ref(s); + else scr_dgram_unref(s); + return scr_dyn_retain(self); + } + { + static const char *const known[] = { + "remoteAddress", "setBroadcast", "setTTL", "setMulticastTTL", "setMulticastLoopback", + "setMulticastInterface", "addMembership", "dropMembership", "addSourceSpecificMembership", + "dropSourceSpecificMembership", "getRecvBufferSize", "getSendBufferSize", + "setRecvBufferSize", "setSendBufferSize", "getSendQueueSize", "getSendQueueCount", + "prependListener", "prependOnceListener", "removeAllListeners", "listeners", + "rawListeners", "listenerCount", "eventNames", "setMaxListeners", "getMaxListeners", NULL, + }; + for (size_t i = 0; known[i]; i++) { + if (strcmp(method, known[i]) == 0) { + scr_dgram_dynh_unsupported(method, NULL); + return NULL; + } + } + } + char msg[160]; + int n = snprintf(msg, sizeof msg, "%s is not a function", what); + scr_throw_error_msg(SCR_ERR_TYPE, msg, (size_t)n); + return NULL; +} + +/* A method read as a value (`socket.emit`, then `.bind(socket)`): one + * function per name, which dispatches on its receiver like the + * prototype method it stands for. */ +static ScrDyn *scr_dgram_dyn_method_call(ScrClosure *closure, ScrDyn *const *args, size_t argc) { + ScrStr *method = scr_box_get_ref(closure->caps[0]); + ScrDyn *receiver = scr_dyn_this_get(); + ScrDyn *result = NULL; + if (receiver->kind == SCR_DYN_HANDLE && receiver->v.handle.tag == SCR_DYNH_DGRAM_SOCKET) { + result = scr_dgram_dynh_invoke(receiver->v.handle.ptr, receiver, method->data, args, argc, + method->data); + } else { + char msg[160]; + int n = snprintf(msg, sizeof msg, + "dgram.Socket.prototype.%s called on a value that is not a dgram Socket", + method->data); + scr_throw_error_msg(SCR_ERR_TYPE, msg, (size_t)n); + } + scr_dyn_release(receiver); + scr_str_release(method); + return result; +} + +static const char *const scr_dgram_dyn_method_names[] = { + "on", "once", "addListener", "removeListener", "off", "emit", "send", "close", "bind", + "connect", "disconnect", "address", "ref", "unref", NULL, +}; + +static void scr_dgram_dyn_cleanup(void) { + scr_dyn_release(scr_dgram_dyn_methods); + scr_dgram_dyn_methods = NULL; + /* Untyped listeners can close cycles through the boxes they capture, + * and the proxies in a socket's lists hold its entry: exit releases + * both, socket by socket. */ + /* Releasing one entry's listeners can free other entries (they unlink + * themselves), so each pass restarts from the head. */ + for (;;) { + ScrDgramDyn *e = scr_dgram_dyns; + while (e && e->cleaned) e = e->next; + if (e == NULL) break; + e->cleaned = true; + e->rc++; + scr_dgram_dyn_clear(e); + ScrDgramSocket *s = e->s; + scr_dgram_dyn_ls_drop(&s->msg_ls); + scr_dgram_dyn_ls_drop(&s->err_ls); + scr_dgram_dyn_ls_drop(&s->listening_ls); + scr_dgram_dyn_ls_drop(&s->close_ls); + scr_dgram_dyn_ls_drop(&s->conn_ls); + scr_dgram_dyn_release_v(e); + } +} + +static ScrDyn *scr_dgram_dynh_get(void *h, const char *key, size_t key_len) { + (void)h; + for (size_t i = 0; scr_dgram_dyn_method_names[i]; i++) { + if (strcmp(key, scr_dgram_dyn_method_names[i]) != 0) continue; + if (scr_dgram_dyn_methods == NULL) scr_dgram_dyn_methods = scr_dyn_new_obj(); + ScrDyn *have = scr_dyn_obj_get(scr_dgram_dyn_methods, key, key_len); + if (have != NULL) return scr_dyn_retain(have); + ScrStr *name = scr_str_new(key, key_len); + ScrClosure *closure = scr_closure_new(NULL, 1); + closure->caps[0] = scr_box_new(SCR_BOX_STR); + scr_box_set_ref(closure->caps[0], name); + ScrDyn *fn = scr_dyn_new_func(closure, scr_dgram_dyn_method_call, 0, "native:dgram.Socket", + scr_dgram_dyn_method_names[i]); + scr_dyn_obj_set(scr_dgram_dyn_methods, key, key_len, scr_dyn_retain(fn)); + return fn; + } + return NULL; +} + +static bool scr_dgram_dynh_set(void *h, const char *key, size_t key_len, const ScrDyn *value) { + (void)h; (void)key; (void)key_len; (void)value; + return false; +} + +static const ScrDynHandleOps scr_dgram_dynh_ops = { + .cls = "Socket", + .retain = &scr_dgram_dyn_retain_v, + .release = &scr_dgram_dyn_release_v, + .invoke = &scr_dgram_dynh_invoke, + .get = &scr_dgram_dynh_get, + .set = &scr_dgram_dynh_set, +}; + +ScrDyn *scr_dgram_to_dyn(ScrDgramSocket *s) { + static SCR_TL bool installed = false; + if (!installed) { + installed = true; + scr_dyn_handle_install(SCR_DYNH_DGRAM_SOCKET, &scr_dgram_dynh_ops); + scr_atexit(scr_dgram_dyn_cleanup); + } + ScrDgramDyn *e = scr_dgram_dyns; + while (e && e->s != s) e = e->next; + if (e == NULL) { + /* rc 0: the box's retain takes the first reference */ + e = calloc(1, sizeof *e); + if (!e) scr_dgram_dyn_oom(); + e->s = scr_dgram_retain(s); + e->next = scr_dgram_dyns; + scr_dgram_dyns = e; + } + return scr_dyn_new_handle(e, SCR_DYNH_DGRAM_SOCKET); +} + +ScrDgramSocket *scr_dgram_from_dyn(const ScrDyn *d, const ScrDynPath *path, const char *want) { + if (d->kind != SCR_DYN_HANDLE || d->v.handle.tag != SCR_DYNH_DGRAM_SOCKET) { + scr_dyn_check_fail(path, want, d); + return NULL; + } + return scr_dgram_retain(((ScrDgramDyn *)d->v.handle.ptr)->s); +} diff --git a/packages/runtime/src/scr_dgram_private.h b/packages/runtime/src/scr_dgram_private.h new file mode 100644 index 000000000..95f418da9 --- /dev/null +++ b/packages/runtime/src/scr_dgram_private.h @@ -0,0 +1,47 @@ +/* scriptc runtime — the dgram socket's layout, shared by scr_dgram.c and + * the checked-dynamic unit scr_dgram_dyn.c (runtime-internal; not part of + * scr_runtime.h). */ +#ifndef SCR_DGRAM_PRIVATE_H +#define SCR_DGRAM_PRIVATE_H + +#include "scr_runtime.h" + +/* ── listener lists (the scr_net.c snapshot discipline, restated so this + * unit links without scr_net.c) ─────────────────────────────────────── */ + +typedef struct { + ScrClosure *cb; + void *fn; + bool once; +} ScrDgramL; + +typedef struct { + ScrDgramL *ls; + size_t n, cap; +} ScrDgramLs; + +/* ── the handle ──────────────────────────────────────────────────────── */ + +struct ScrDgramSocket { + size_t rc; + int fd; /* -1 until bound/connected, -1 again once closed */ + bool reuse_addr; + bool bound; + bool connected; + bool closing; /* close() called; 'close' fires at the next sweep */ + bool close_emitted; /* settled: off the registry, listeners dropped */ + bool unrefed; /* waives loop liveness, Node's unref */ + bool read_armed; + bool emit_listening; + bool emit_connect; + ScrStr *pending_err; /* deferred bind/connect/send failure */ + ScrDgramLs msg_ls, err_ls, listening_ls, close_ls, conn_ls; + bool in_registry; + struct ScrDgramSocket *next; +}; + +/* Re-evaluates the socket's read arming after its 'message' listeners + * changed (scr_dgram.c). */ +void scr_dgram_rearm(ScrDgramSocket *s); + +#endif /* SCR_DGRAM_PRIVATE_H */ diff --git a/packages/runtime/src/scr_dyn.h b/packages/runtime/src/scr_dyn.h index 401bf2b39..b75806136 100644 --- a/packages/runtime/src/scr_dyn.h +++ b/packages/runtime/src/scr_dyn.h @@ -154,6 +154,7 @@ typedef enum { X(CUSTOM_EVENT, 40) /* ScrEvent with detail — new CustomEvent() */ \ X(DATE, 30) /* JavaScript Date with stable native identity */ \ X(DECIPHER, 46) /* node:crypto Decipheriv (scr_cipher.c) */ \ + X(DGRAM_SOCKET, 62) /* dgram.Socket (scr_dgram.c) */ \ X(DIR, 44) /* fs.Dir (scr_lib.c) */ \ X(DISPOSABLE_STACK, 49) /* DisposableStack (scr_symbol.c) */ \ X(EVENT, 14) /* ScrEvent — new Event() and AbortSignal's abort events */ \ diff --git a/tests/corpus/6590-sloppy-arguments-unobservable.cjs b/tests/corpus/6590-sloppy-arguments-unobservable.cjs new file mode 100644 index 000000000..d65470d6e --- /dev/null +++ b/tests/corpus/6590-sloppy-arguments-unobservable.cjs @@ -0,0 +1,42 @@ +// Sloppy-mode functions with named parameters that read `arguments` only +// where its aliasing of the parameters cannot show: the count, and +// element reads in functions that never write a parameter or an element. +function count(a, b, c) { + if (arguments.length < 3) { + if (typeof b !== 'number') { + c = b; + b = undefined; + } + } + return [arguments.length, a, b, c]; +} +console.log(JSON.stringify(count('x'))); +console.log(JSON.stringify(count('x', ['t']))); +console.log(JSON.stringify(count('x', 2, ['t']))); + +const Options = function (host, port) { + let options = host || {}; + if (arguments.length > 1 || typeof host === 'string') { + options = { host: host, port: port }; + } + this.host = options.host || 'localhost'; + this.port = options.port || 8125; +}; +const a = new Options({ host: 'h1', port: 1 }); +const b = new Options('h2', 2); +const c = new Options('h3'); +const d = new Options(); +console.log(a.host, a.port, b.host, b.port, c.host, c.port, d.host, d.port); + +function pick(first, second) { + const out = []; + for (let i = 0; i < arguments.length; i++) out.push(arguments[i]); + return out.join(',') + '|' + first + '|' + second; +} +console.log(pick(1), pick(1, 2), pick(1, 2, 3, 4)); + +const proto = {}; +proto.step = function (value, by) { + return arguments.length === 1 ? value + 1 : value + by; +}; +console.log(proto.step(1), proto.step(1, 10)); diff --git a/tests/corpus/6591-function-values-any-slot.cjs b/tests/corpus/6591-function-values-any-slot.cjs new file mode 100644 index 000000000..171b3a4f7 --- /dev/null +++ b/tests/corpus/6591-function-values-any-slot.cjs @@ -0,0 +1,39 @@ +// Functions whose JSDoc makes parameters optional, stored through an +// untyped receiver (`C.prototype.m = function (...)` inside a helper +// that takes the constructor): the stored value takes any arity, and +// omitted arguments are undefined. +function install(C) { + /** + * @param {string} name + * @param {number=} amount + * @param {string[]=} tags + */ + C.prototype.add = function (name, amount, tags) { + const n = amount === undefined ? 1 : amount; + this.log.push(`${name}:${n}${tags ? '|#' + tags.join(',') : ''}`); + return this; + }; + /** + * @param {(x: number) => number} fn + * @param {string=} label + */ + C.prototype.time = function (fn, label) { + const self = this; + return (x) => { + const result = fn(x); + self.log.push(`${label || 'anon'}=${result}`); + return result; + }; + }; +} + +const Recorder = function () { + this.log = []; +}; +install(Recorder); +const r = new Recorder(); +r.add('a').add('b', 2).add('c', 3, ['x:y']); +const timed = r.time((x) => x * 2, 'double'); +console.log(timed(21)); +console.log(r.time((x) => x + 1)(6)); +console.log(r.log.join(' ')); diff --git a/tests/corpus/6592-dgram-send-callbacks.cjs b/tests/corpus/6592-dgram-send-callbacks.cjs new file mode 100644 index 000000000..66fb64a8d --- /dev/null +++ b/tests/corpus/6592-dgram-send-callbacks.cjs @@ -0,0 +1,41 @@ +// dgram.Socket.send's callback forms from JavaScript: the offset/length +// form with a trailing callback, the port/address form with one, a buffer +// list sent as one datagram, and a connected socket's send. Each callback +// receives (null, bytes) after the send; a closed socket throws first. +'use strict'; +const dgram = require('dgram'); + +const server = dgram.createSocket('udp4'); +const got = []; +server.on('message', (msg) => { + got.push(msg.toString()); +}); + +function run(port) { + const client = dgram.createSocket('udp4'); + const steps = []; + const done = (label) => (err, bytes) => steps.push(`${label} ${err} ${bytes}`); + const buf = Buffer.from('xxhello.sliceyy'); + client.send(buf, 2, 11, port, '127.0.0.1', done('slice')); + client.send('plain', port, '127.0.0.1', done('plain')); + client.send(['li', Buffer.from('st'), 'ed'], port, '127.0.0.1', done('list')); + client.send(Buffer.from('nocb'), 0, 4, port, '127.0.0.1'); + const connected = dgram.createSocket('udp4'); + connected.connect(port, '127.0.0.1', () => { + connected.send('connected', done('connected')); + setTimeout(() => { + console.log(steps.join('\n')); + console.log(got.sort().join(',')); + client.close(); + connected.close(); + try { + client.send('late', port, '127.0.0.1', done('late')); + } catch (e) { + console.log(`${e.code}: ${e.message}`); + } + server.close(); + }, 100); + }); +} + +server.bind(0, '127.0.0.1', () => run(server.address().port)); diff --git a/tests/corpus/6593-dgram-socket-bound-methods.cjs b/tests/corpus/6593-dgram-socket-bound-methods.cjs new file mode 100644 index 000000000..8fce941e2 --- /dev/null +++ b/tests/corpus/6593-dgram-socket-bound-methods.cjs @@ -0,0 +1,61 @@ +// A dgram socket's methods read as values and bound to it (the transport +// wrapper shape): listeners registered, removed and emitted through the +// bound functions, the socket closed through its bound close, and an +// 'error' emitted after close still reaching the listener left on it. +'use strict'; +const dgram = require('dgram'); + +function wrap(socket) { + return { + emit: socket.emit.bind(socket), + on: socket.on.bind(socket), + once: socket.once.bind(socket), + removeListener: socket.removeListener.bind(socket), + send: function (buf, callback) { + socket.send(buf, 0, buf.length, this.port, '127.0.0.1', callback); + }, + close: socket.close.bind(socket), + unref: socket.unref.bind(socket), + port: 0, + }; +} + +const server = dgram.createSocket('udp4'); +let t; + +function finish() { + t.on('close', () => { + console.log('closed'); + console.log('emit after close', t.emit('error', new Error('late'))); + t.removeListener('error', second); + try { + t.emit('error', new Error('unheard')); + } catch (e) { + console.log('threw', e.message); + } + server.close(); + }); + t.close(); +} + +server.on('message', (msg, rinfo) => { + console.log('server got', msg.toString(), rinfo.family, rinfo.size); + finish(); +}); + +const first = (err) => console.log('first', err.message); +const second = (err) => console.log('second', err.message); + +server.bind(0, '127.0.0.1', () => { + t = wrap(dgram.createSocket('udp4')); + t.port = server.address().port; + t.unref(); + t.on('error', first); + t.on('error', second); + t.once('error', (err) => console.log('once', err.message)); + console.log('emit', t.emit('error', new Error('boom'))); + t.removeListener('error', first); + console.log('emit', t.emit('error', new Error('again'))); + console.log('emit custom', t.emit('nobody')); + t.send(Buffer.from('ping'), (err, bytes) => console.log('sent', err, bytes)); +}); diff --git a/tests/corpus/6594-jsdoc-untrusted-prototype-methods.cjs b/tests/corpus/6594-jsdoc-untrusted-prototype-methods.cjs new file mode 100644 index 000000000..356c3ed9d --- /dev/null +++ b/tests/corpus/6594-jsdoc-untrusted-prototype-methods.cjs @@ -0,0 +1,43 @@ +// @ts-nocheck +// JSDoc in a file TypeScript does not check documents the usual types of +// functions stored on a prototype without constraining their callers: +// undefined, objects and numbers reach parameters documented otherwise. +const Client = function (rate) { + this.sampleRate = rate || 1; + this.prefix = 'p.'; +}; + +/** + * Sends a stat. + * @param stat {String} The stat + * @param value The value + * @param type {String} The type + * @param sampleRate {Number} The rate + * @param tags {Array} The tags + * @param callback {Function=} Called after sending + */ +Client.prototype.sendStat = function (stat, value, type, sampleRate, tags, callback) { + let message = `${this.prefix + stat}:${value}|${type}`; + sampleRate = sampleRate || this.sampleRate; + if (sampleRate && sampleRate < 1) { + message += `|@${sampleRate}`; + } + console.log(message, Array.isArray(tags) ? tags.join(',') : String(tags)); + if (callback) callback(); +}; + +/** + * @param stat {String} The stat + * @param value {Number} The value + */ +Client.prototype.count = function (stat, value) { + this.sendStat(stat, value === undefined ? 1 : value, 'c'); +}; + +const c = new Client(); +c.sendStat('a', 1, 'c', undefined, undefined); +c.sendStat('b', 2, 'g', 0.5, { k: 'v' }, () => console.log('callback')); +c.sendStat(7, '3', 'ms', '0.25', ['x', 'y']); +new Client(0.25).sendStat('c', 3, 'ms'); +c.count('hits'); +c.count('more', 'five'); diff --git a/tests/corpus/6595-string-concat.ts b/tests/corpus/6595-string-concat.ts new file mode 100644 index 000000000..17498ff59 --- /dev/null +++ b/tests/corpus/6595-string-concat.ts @@ -0,0 +1,9 @@ +// String.prototype.concat converts each argument with ToString, in order. +const base = "dd.internal.entity_id:"; +console.log(base.concat("pod-7")); +console.log("a".concat()); +console.log("n=".concat(String(1), "+", (2).toString())); +const parts: string[] = ["x", "y"]; +console.log("list:".concat(parts[0]!, parts[1]!)); +const value: unknown = { toString: () => "custom" }; +console.log("obj:".concat(String(value))); diff --git a/tests/corpus/6596-parseint-missing-text.cjs b/tests/corpus/6596-parseint-missing-text.cjs new file mode 100644 index 000000000..89b4f4b43 --- /dev/null +++ b/tests/corpus/6596-parseint-missing-text.cjs @@ -0,0 +1,8 @@ +// parseInt over a value that may be missing, as JavaScript writes it with +// an environment variable fallback: Node parses the text "undefined". +'use strict'; +const port = parseInt(process.env.SCRIPTC_UNSET_PORT_VARIABLE, 10) || 8125; +console.log(port); +console.log(parseInt(process.env.SCRIPTC_TEST_ENV, 36)); +const options = {}; +console.log(parseInt(options.missing, 10), parseInt(options.missing, 36)); diff --git a/tests/fixtures/npm/cases/hot-shots-udp/main.ts b/tests/fixtures/npm/cases/hot-shots-udp/main.ts new file mode 100644 index 000000000..578ed87cc --- /dev/null +++ b/tests/fixtures/npm/cases/hot-shots-udp/main.ts @@ -0,0 +1,101 @@ +// @stderr +// The hot-shots StatsD client (its JavaScript, vendored at 8.5.0) sends +// metrics over UDP to a local dgram server, which prints every datagram in +// arrival order: tags and explicit sample rates, the callback, child +// clients, the error handler, close, and a send after close. Its mock and +// Unix-socket modes format and refuse without the network. +import * as dgram from "node:dgram"; +import { StatsD } from "hot-shots"; + +const server = dgram.createSocket("udp4"); +const received: string[] = []; +let expected = 0; +let arrived: () => void = () => {}; + +server.on("message", (msg) => { + received.push(msg.toString()); + if (received.length === expected) arrived(); +}); + +function datagrams(count: number): Promise { + expected += count; + return new Promise((resolve) => { + arrived = resolve; + if (received.length >= expected) resolve(); + }); +} + +function flush(): void { + for (const line of received.splice(0)) console.log(line); + expected = 0; +} + +async function udp(port: number): Promise { + const client = new StatsD({ + host: "127.0.0.1", + port, + prefix: "app.", + globalTags: { service: "svc", env: "test" }, + errorHandler: (err) => console.log("error handler", (err as Error & { code?: string }).code), + }); + client.increment("boot"); + client.increment("requests", 3, ["route:/a"]); + client.increment("poll", 1, { result: "none", env: "prod" }); + client.decrement("inflight"); + client.decrement("slots", 2, 1, ["pool:a"]); + client.gauge("queue.depth", 12, 1, { queue: "builds" }); + client.timing("startup", 250); + client.timing("phase", 12.5, ["phase:load"]); + client.histogram("payload.bytes", 1024, ["kind:json"]); + client.distribution("latency", 42.5, 1, { op: "read" }); + client.distribution("memory", 1048576, { node: "v24" }); + client.distribution("polltime", 17); + client.set("users", "alice"); + client.increment(["a.multi", "b.multi"], 2); + await datagrams(15); + flush(); + + await new Promise((resolve) => { + client.increment("acked", 1, (err: unknown, bytes: unknown) => { + console.log("sent", err, bytes); + resolve(); + }); + }); + await datagrams(1); + flush(); + + const child = client.childClient({ prefix: "child.", globalTags: { component: "c" } }); + child.increment("hit"); + child.distribution("size", 3, { env: "dev" }); + await datagrams(2); + flush(); + + await new Promise((resolve) => { + client.close((err) => { + console.log("closed", err); + resolve(); + }); + }); + client.increment("after.close"); +} + +function offline(): void { + process.env.DD_ENTITY_ID = "pod|7"; + const mock = new StatsD({ mock: true, prefix: "m.", globalTags: ["a:b"] }); + mock.increment("hits", 2); + mock.timing("t", 5, ["x:y"]); + mock.check("svc", mock.CHECKS.OK, { hostname: "h", message: "fine" }); + mock.event("title", "text\nmore", { priority: "low" }, ["e:1"]); + console.log(mock.mockBuffer); + const errors: string[] = []; + const uds = new StatsD({ protocol: "uds", errorHandler: (e) => errors.push(e.message) }); + uds.increment("x"); + console.log(errors); +} + +server.bind(0, "127.0.0.1", () => { + void udp(server.address().port).then(() => { + offline(); + server.close(() => console.log("server closed")); + }); +}); diff --git a/tests/fixtures/npm/cases/hot-shots-udp/package.json b/tests/fixtures/npm/cases/hot-shots-udp/package.json new file mode 100644 index 000000000..5ffd9800b --- /dev/null +++ b/tests/fixtures/npm/cases/hot-shots-udp/package.json @@ -0,0 +1 @@ +{ "type": "module" } diff --git a/tests/fixtures/npm/node_modules/hot-shots/CHANGES.md b/tests/fixtures/npm/node_modules/hot-shots/CHANGES.md new file mode 100644 index 000000000..67855c4dc --- /dev/null +++ b/tests/fixtures/npm/node_modules/hot-shots/CHANGES.md @@ -0,0 +1,314 @@ +CHANGELOG +========= + +## 8.5.0 (2021-7-16) +@maxday Add a closingFlushInterval option which allows stopping quicker + +## 8.4.0 (2021-7-3) +@roim Use errorHandler when possible on UDS socket replace error + +## 8.3.2 (2021-5-29) +@cmaddalozzo Close unix domain socket after unsuccessful attempts to connect + +## 8.3.1 (2021-4-1) +* @dvd-z Fix date_happened to allow usage of numbers + +## 8.3.0 (2020-12-16) +* @chotiwat Handle UDS errors occurring when sending metrics + +## 8.2.1 (2020-12-1) +* @stephenmathieson Make close callback optional in TypeScript definition + +## 8.2.0 (2020-9-30) +* @dhermes Making UDS error handling and recovery more robust. Note these look to be ok in a minor release but are signficant upgrades to how UDS works. Thanks as well to @prognant for an overlapping PR. + +## 8.1.0 (2020-9-25) +* @maleblond Support multiple values for the same tag key + +## 8.0.0 (2020-9-23) +* @naseemkullah Change default value for 'host' from 'localhost' to + undefined. This means the default host will now be 127.0.0.1 or ::1, + which has cases where it will speed up sending metrics. This should be a + non-breaking change, but bumping to a major version for it given + it's a very base change to the library. +* @naseemkullah Switch from equals to strictEquals in tests + +## 7.8.0 (2020-8-28) +* @bdeitte Fix some flaky tests +* @ralphiech Add missing error handler when socket is not created +* @ralphiech Add missing socket checks +* @dependabot Bump lodash from 4.17.15 to 4.17.19 +* @DerGut Add "Congestion error" section to README + +## 7.7.1 (2020-8-4) +* @DerGut Fix udsGracefulErrorHandling default value + +## 7.7.0 (2020-7-29) +* @tebriel Add asyncDistTimer function + +## 7.6.0 (2020-6-16) +* @Impeekay Add date type to timing function + +## 7.5.0 (2020-6-5) +* @benblack86 Unreference underlying socket/interval to prevent process hangs + +## 7.4.2 (2020-5-5) +* @kazk Fix types for set/unique + +## 7.4.1 (2020-4-28) +* @lbeschastny Sanitize ',' tags characters for telegraf + +## 7.4.0 (2020-4-3) +* @MichaelSitter add tagPrefix and tagSeparator options + +## 7.3.0 (2020-4-1) +* @marciopd Use Date.now() instead of new Date() +* @chotiwat Add UDS graceful error handling options to typescript +* @bdeitte Update packages, most notably getting node-unix-dgram 2.0.4 + +## 7.2.0 (2020-3-19) +* @marciopd Add cacheDnsTtl +* @dependabot Bump acorn from 6.3.0 to 6.4.1 + +## 7.1.0 (2020-3-4) +* @wision Actually fix cachedDns with udp +* @casey-chow TypeScript: parameterize function types in timer and asyncTimer + +## 7.0.0 (2020-2-13) +* @tomruggs Remove support for Node 6- now supporting Node 8.x or higher +* @tomruggs Update to the latest mocha version to get rid of a security warning + +## 6.8.7 (2020-2-10) +* @mrknmc Fix TypeError when increment called without a callback argument + +## 6.8.6 (2020-1-28) +* @ericmustin callback is not properly passed bytes argument + +## 6.8.5 (2019-12-19) +* @bdeitte Fix for socket on reading when cacheDns and udp in use + +## 6.8.4 (2019-12-18) +* @bdeitte Fix cacheDns with udp + +## 6.8.3 (2019-12-15) +* @gleb-rudenko Fix StatsD constructor typing + +## 6.8.2 (2019-11-12) +* @almandsky Fix useDefaultRoute to work again after abstract transports + +## 6.8.1 (2019-10-16) +* @hayes Add unref method to transport interface + +## 6.8.0 (2019-10-14) +* @runk Add new protocol, stream, and a stream parameter for + specifying it. + +## 6.7.0 (2019-10-9) +* @runk Code refactoring to have abstract transports + +## 6.6.0 (2019-10-7) +* @NinjaBanjo @msiebuhr Add udsGracefulErrorHandling, ensuring uds + handles socket errors gracefully + +## 6.5.1 (2019-9-28) +* @msiebuhr Fix crasher when closing Unix Datagram Sockets without callback + +## 6.5.0 (2019-9-22) +* @bdeitte Update decrement to handle missing arguments the same way +that increment does +* @bdeitte Document that memory may grow unbounded in mock mode +* @bdeitte Only load in unix-dgram library when uds protocol in use + +## 6.4.1 (2019-9-19) +* @jfirebaugh Fix cacheDns option when obtaining host from DD_AGENT_HOST + +## 6.4.0 (2019-6-28) +* @tghaas Add Node 12 support to uds protocol support +* @jhoch README clarifications + +## 6.3.0 (2019-5-18) +* @paguillama Fix user defined tag example on README optional parameters +* @gabsn Initial support for uds protocol +* @bdeitte Updated and fixed up uds protocol support + +## 6.2.0 (2019-4-10) +* @ahmed-mez Add support for env variables DD_AGENT_HOST, +DD_DOGSTATSD_PORT, and DD_ENTITY_ID +* @JamesMGreene Fix syntax in README example + +## 6.1.1 (2019-1-8) +* @bdeitte Fix errorHandler to only happen again on errors +* @Ithildir Readme fixes + +## 6.1.0 (2019-1-5) +* @bdeitte Ensure close() call always sends data before closing +* @bdeitte Recommend errorHandler over client.socket.on() for handling +errors +* @mbellerose Fix the timer function type definition + +## 6.0.1 (2018-12-17) +* @msmnc Fix regression when tag value is a number +* @bdeitte Make non-options in constructor more deprecated + +## 6.0.0 (2018-12-15) +@bdeitte Major upgrade to the codebase to be more modern, +overhaul tests, and many small tweaks. Most of this is internal to +the project, but there are a few changes to note for everyone: +* Now requires Node 6 or above +* Update close() to handle errors better, not doubling up in error +messages and not leaving uncaught errors + +Everything else done here should be internal facing. Those changes +include: +* Use "lebab" to ES6-ify the project +* Switch from jshint and eslint and make syntax updates based on this +* Remove a lot of duplication in tests and many small fixups in tests +* Start using Mocha 4 +* Stop using index.js for testing +* Start using the code coverage report as part of the build +* Remove the ignoring of errors on close of tests, and tear down tests in general better +* Stop using "new Buffer", that is deprecated, and use Buffer.from() instead + +## 5.9.2 (2018-11-10) +* @stieg Add mockBuffer to types + +## 5.9.1 (2018-9-18) +* @etaoins Add asyncTimer types +* @blimmer: Add increment doc snippet + +## 5.9.0 (2018-7-27) +* @chrismatheson: Fix timer to have duration in microseconds (was nanoseconds) +* @chrismatheson: Add asyncTimer functionality + +## 5.8.0 (2018-7-17) +* @michalholasek Clean up code formatting and split up tests +* @michalholasek Add tcp protocol support +* @remie Add tcp protocol support + +## 5.7.0 (2018-7-4) +* @Willyham Add support for recording buffers in mock mode + +## 5.6.3 (2018-6-20) +* @singerb correct close() type definition + +## 5.6.2 (2018-6-15) +* @mjesuele Fix time in timer + +## 5.6.1 (2018-6-4) +* @MattySheikh Typescript: add socket type for StatsD class + +## 5.6.0 (2018-6-3) +* @drewen TypeScript: add overload types for stats functions + +## 5.5.1 (2018-5-30) +* @emou Typescript declaration for the 'timer' method + +## 5.5.0 (2018-5-30) +* @drewen Split up single file, add code coverage capabilities + +## 5.4.1 (2018-5-12) +* @jasonsack Fixups for new useDefaultRoute option +* @bdeitte Test against more modern set of Node versions in Travis + +## 5.4.0 (2018-4-26) +* @RobGraham Added `distribution()` support for DataDog v6 + +## 5.3.0 (2018-4-3) +* @tanelso2 Added support for using default route on Linux + +## 5.2.0 (2018-2-28) +* @ericapisani Add timer decorator function + +## 5.1.0 (2018-2-14) +* @lautis Pass key-value tags as objects + +## 5.0.1 (2018-2-2) +* @punya-asapp Add childClient to TypeScript types + +## 5.0.0 (2017-11-9) +* @jgwmaxwell TypeScript typings, resolving the default export issue and missing options from last time. This is being marked as a major release, in caution given the revert last time, but it is not actually known to cause any backwards-compatible issues. + +## 4.8.0 (2017-10-31) +* @Jiggmin concat prefix and suffix in check function +* @Jiggmin commit package-lock.json + +## 4.7.1 (2017-10-31) +* @Jiggmin Add backwards compatibility for global_tags + +## 4.7.0 (2017-9-21) +* @bdeitte Revert TypeScript typings, which ended up not being semver minor + +## 4.6.0 (2017-9-19) +* @jgwmaxwell TypeScript typings + +## 4.5.0 (2017-5-4) +* @jsocol Support default value with tags in increment + +## 4.4.0 (2017-3-23) +* @RijulB Global sample rate + +## 4.3.1 (2016-11-7) +* @RandomSeeded Fix callbacks not being triggered when using buffers + +## 4.3.0 (2016-9-30) +* @ggoodman Allow socket errors to be handled with errorHandler + +## 4.2.0 (2016-8-3) +* @mhahn Add support for DataDog service checks + +## 4.1.1 (2016-5-22) +* @ash2k date_happened should be seconds, not milliseconds + +## 4.1.0 (2016-5-8) +* @ash2k Support multiline text in DataDog events + +## 4.0.0 (2016-5-7) +* @ash2k Provided tags, including `childClient()` tags, override global tags with same names. + +## 3.1.0 (2016-5-3) +* @ash2k Support a client-wide error handler used in case no callback is provided and to handle various exceptions. + +## 3.0.1 (2016-4-28) +* @bdeitte Add 'use strict' to files and make changes needed for this. + +## 3.0.0 (2016-4-27) +* @ash2k Method to create child clients. (This is not a backwards-incompatible change but is rather large.) +* @ash2k Shrink npm package a bit more + +## 2.4.0 (2016-2-26) +* @arlolra Shrink npm package +* @arlolra/@bdeitte Move DNS errors when caching them to send() and use callback when possible +* @bdeitte Use callback for Telegraf error when possible + +## 2.3.1 (2016-2-3) +* @Pchelolo Ensure messages not larger then maxBufferSize + +## 2.3.0 (2016-1-17) +* @bdeitte Fix increment(name, 0) to send a 0 count instead of 1 +* @bdeitte Flush the queue when needed on close() + +## 2.2.0 (2016-1-10) +* @bdeitte Document and expand on close API +* @bdeitte Catch more error cases for callbacks + +## 2.1.2 (2015-12-9) +* @bdeitte Even more doc updates +* @mmoulton Fix multiple tags with Telegraf + +## 2.1.1 (2015-12-9) +* @bdeitte Doc updates + +## 2.1.0 (2015-12-9) +* @mmoulton Add options.telegraf to enable support for Telegraf's StatsD line protocol format +* @mmoulton Ensure message callback is sent in buffered case, even when we just buffer. + +## 2.0.0 (2015-10-22) +* @jjofseattle Add options.maxBufferSize and options.bufferFlushInterval +* @bdeitte Change options.global_tags to options.globalTags for consistency + +## 1.0.2 (2015-09-25) +* @ainsleyc Thrown error when cacheDNS flag fails to resolve DNS name + +## 1.0.1 (2015-09-24) +* @bdeitte Add the event API used by DogStatsD +* @sivy Start from the base of https://github.com/sivy/node-statsd diff --git a/tests/fixtures/npm/node_modules/hot-shots/LICENSE b/tests/fixtures/npm/node_modules/hot-shots/LICENSE new file mode 100644 index 000000000..176fc2b2f --- /dev/null +++ b/tests/fixtures/npm/node_modules/hot-shots/LICENSE @@ -0,0 +1,18 @@ +Copyright 2011 Steve Ivy. All rights reserved. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to +deal in the Software without restriction, including without limitation the +rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +sell copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. diff --git a/tests/fixtures/npm/node_modules/hot-shots/README.md b/tests/fixtures/npm/node_modules/hot-shots/README.md new file mode 100644 index 000000000..29f929de3 --- /dev/null +++ b/tests/fixtures/npm/node_modules/hot-shots/README.md @@ -0,0 +1,300 @@ +# hot-shots + +A Node.js client for [Etsy](http://etsy.com)'s [StatsD](https://github.com/etsy/statsd) server, Datadog's [DogStatsD](http://docs.datadoghq.com/guides/dogstatsd/) server, and [InfluxDB's](http://influxdb.com) [Telegraf](https://github.com/influxdb/telegraf) StatsD server. + +This project was originally a fork off of [node-statsd](https://github.com/sivy/node-statsd). This project +includes all changes in the latest node-statsd and many additional changes, including: +* TypeScript types +* Telegraf support +* events +* child clients +* tcp protocol support +* uds (Unix domain socket) protocol support +* raw stream protocol support +* mock mode +* asyncTimer +* asyncDistTimer +* much more, including many bug fixes + +hot-shots supports Node 8.x and higher. + +[![Build Status](https://secure.travis-ci.org/brightcove/hot-shots.png?branch=master)](http://travis-ci.org/brightcove/hot-shots) + +## Migrating from node-statsd + +You should only need to do one thing: change node-statsd to hot-shots in all requires. + +You can check the detailed [change log](https://github.com/brightcove/hot-shots/blob/master/CHANGES.md) for what has changed since the last release of node-statsd. + +## Usage + +All initialization parameters are optional. + +Parameters (specified as one object passed into hot-shots): + +* `host`: The host to send stats to, if not set, the constructor tries to + retrieve it from the `DD_AGENT_HOST` environment variable, `default: 'undefined'` which as per [UDP/datagram socket docs](https://nodejs.org/api/dgram.html#dgram_socket_send_msg_offset_length_port_address_callback) results in `127.0.0.1` or `::1` being used. +* `port`: The port to send stats to, if not set, the constructor tries to retrieve it from the `DD_DOGSTATSD_PORT` environment variable, `default: 8125` +* `prefix`: What to prefix each stat name with `default: ''` +* `suffix`: What to suffix each stat name with `default: ''` +* `tagPrefix`: Prefix tag list with character `default: '#'`. Note does not work with `telegraf` option. +* `tagSeparator`: Separate tags with character `default: ','`. Note does not work with `telegraf` option. +* `globalize`: Expose this StatsD instance globally. `default: false` +* `cacheDns`: Caches dns lookup to *host* for *cacheDnsTtl*, only used + when protocol is `udp`, `default: false` +* `cacheDnsTtl`: time-to-live of dns lookups in milliseconds, when *cacheDns* is enabled. `default: 60000` +* `mock`: Create a mock StatsD instance, sending no stats to + the server and allowing data to be read from mockBuffer. Note that + mockBuffer will keep growing, so only use for testing or clear out periodically. `default: false` +* `globalTags`: Tags that will be added to every metric. Can be either an object or list of tags. The *Datadog* `dd.internal.entity_id` tag is appended to `globalTags` from the `DD_ENTITY_ID` environment variable if the latter is set. `default: {}` +* `maxBufferSize`: If larger than 0, metrics will be buffered and only sent when the string length is greater than the size. `default: 0` +* `bufferFlushInterval`: If buffering is in use, this is the time in ms to always flush any buffered metrics. `default: 1000` +* `telegraf`: Use Telegraf's StatsD line protocol, which is slightly different than the rest `default: false` +* `sampleRate`: Sends only a sample of data to StatsD for all StatsD methods. Can be overridden at the method level. `default: 1` +* `errorHandler`: A function with one argument. It is called to handle various errors. `default: none`, errors are thrown/logger to console +* `useDefaultRoute`: Use the default interface on a Linux system. Useful when running in containers +* `protocol`: Use `tcp` option for TCP protocol, or `uds` for the Unix Domain Socket protocol or `stream` for the raw stream. Defaults to `udp` otherwise. +* `path`: Used only when the protocol is `uds`. Defaults to `/var/run/datadog/dsd.socket`. +* `stream`: Reference to a stream instance. Used only when the protocol is `stream`. +* `udsGracefulErrorHandling`: Used only when the protocol is `uds`. Boolean indicating whether to handle socket errors gracefully. Defaults to true. +* `udsGracefulRestartRateLimit`: Used only when the protocol is `uds`. Time (ms) between re-creating the socket. Defaults to `1000`. +* `closingFlushInterval`: Before closing, StatsD will check for inflight messages. Time (ms) between each check. Defaults to `50`. + +### StatsD methods +All StatsD methods other than `event`, `close`, and `check` have the same API: +* `name`: Stat name `required` +* `value`: Stat value `required except in increment/decrement where it defaults to 1/-1 respectively` +* `sampleRate`: Sends only a sample of data to StatsD `default: 1` +* `tags`: The tags to add to metrics. Can be either an object `{ tag: "value"}` or an array of tags. `default: []` +* `callback`: The callback to execute once the metric has been sent or buffered + +If an array is specified as the `name` parameter each item in that array will be sent along with the specified value. + +#### `close` +The close method has the following API: + +* `callback`: The callback to execute once close is complete. All other calls to statsd will fail once this is called. + +#### `event` +The event method has the following API: + +* `title`: Event title `required` +* `text`: Event description `default is title` +* `options`: Options for the event + * `date_happened` Assign a timestamp to the event `default is now` + * `hostname` Assign a hostname to the event. + * `aggregation_key` Assign an aggregation key to the event, to group it with some others. + * `priority` Can be ‘normal’ or ‘low’ `default: normal` + * `source_type_name` Assign a source type to the event. + * `alert_type` Can be ‘error’, ‘warning’, ‘info’ or ‘success’ `default: info` +* `tags`: The tags to add to metrics. Can be either an object `{ tag: "value"}` or an array of tags. `default: []` +* `callback`: The callback to execute once the metric has been sent. + +#### `check` +The check method has the following API: + +* `name`: Check name `required` +* `status`: Check status `required` +* `options`: Options for the check + * `date_happened` Assign a timestamp to the check `default is now` + * `hostname` Assign a hostname to the check. + * `message` Assign a message to the check. +* `tags`: The tags to add to metrics. Can be either an object `{ tag: "value"}` or an array of tags. `default: []` +* `callback`: The callback to execute once the metric has been sent. + +```javascript + var StatsD = require('hot-shots'), + client = new StatsD({ + port: 8020, + globalTags: { env: process.env.NODE_ENV }, + errorHandler: errorHandler, + }); + + // Increment: Increments a stat by a value (default is 1) + client.increment('my_counter'); + + // Decrement: Decrements a stat by a value (default is -1) + client.decrement('my_counter'); + + // Histogram: send data for histogram stat (DataDog and Telegraf only) + client.histogram('my_histogram', 42); + + // Distribution: Tracks the statistical distribution of a set of values across your infrastructure. + // (DataDog v6) + client.distribution('my_distribution', 42); + + // Gauge: Gauge a stat by a specified amount + client.gauge('my_gauge', 123.45); + + // Set: Counts unique occurrences of a stat (alias of unique) + client.set('my_unique', 'foobar'); + client.unique('my_unique', 'foobarbaz'); + + // Event: sends the titled event (DataDog only) + client.event('my_title', 'description'); + + // Check: sends a service check (DataDog only) + client.check('service.up', client.CHECKS.OK, { hostname: 'host-1' }, ['foo', 'bar']) + + // Incrementing multiple items + client.increment(['these', 'are', 'different', 'stats']); + + // Incrementing with tags + client.increment('my_counter', ['foo', 'bar']); + + // Sampling, this will sample 25% of the time the StatsD Daemon will compensate for sampling + client.increment('my_counter', 1, 0.25); + + // Tags, this will add user-defined tags to the data + // (DataDog and Telegraf only) + client.histogram('my_histogram', 42, ['foo', 'bar']); + + // Using the callback. This is the same format for the callback + // with all non-close calls + client.set(['foo', 'bar'], 42, function(error, bytes){ + //this only gets called once after all messages have been sent + if(error){ + console.error('Oh noes! There was an error:', error); + } else { + console.log('Successfully sent', bytes, 'bytes'); + } + }); + + // Timing: sends a timing command with the specified milliseconds + client.timing('response_time', 42); + + // Timing: also accepts a Date object of which the difference is calculated + client.timing('response_time', new Date()); + + // Timer: Returns a function that you call to record how long the first + // parameter takes to execute (in milliseconds) and then sends that value + // using 'client.timing'. + // The parameters after the first one (in this case 'fn') + // match those in 'client.timing'. + var fn = function(a, b) { return a + b }; + client.timer(fn, 'fn_execution_time')(2, 2); + + // Async timer: Similar to timer above, but you instead pass in a function + // that returns a Promise. And then it returns a Promise that will record the timing. + var fn = function () { return new Promise(function (resolve, reject) { setTimeout(resolve, n); }); }; + var instrumented = statsd.asyncTimer(fn, 'fn_execution_time'); + instrumented().then(function() { + console.log('Code run and metric sent'); + }); + + // Async timer: Similar to asyncTimer above, but it instead emits a distribution. + var fn = function () { return new Promise(function (resolve, reject) { setTimeout(resolve, n); }); }; + var instrumented = statsd.asyncDistTimer(fn, 'fn_execution_time'); + instrumented().then(function() { + console.log('Code run and metric sent'); + }); + + // Sampling, tags and callback are optional and could be used in any combination (DataDog and Telegraf only) + client.histogram('my_histogram', 42, 0.25); // 25% Sample Rate + client.histogram('my_histogram', 42, { tag: 'value'}); // User-defined tag + client.histogram('my_histogram', 42, ['tag:value']); // Tags as an array + client.histogram('my_histogram', 42, next); // Callback + client.histogram('my_histogram', 42, 0.25, ['tag']); + client.histogram('my_histogram', 42, 0.25, next); + client.histogram('my_histogram', 42, { tag: 'value'}, next); + client.histogram('my_histogram', 42, 0.25, { tag: 'value'}, next); + + // Use a child client to add more context to the client. + // Clients can be nested. + var childClient = client.childClient({ + prefix: 'additionalPrefix.', + suffix: '.additionalSuffix', + globalTags: { globalTag1: 'forAllMetricsFromChildClient'} + }); + childClient.increment('my_counter_with_more_tags'); + + // Close statsd. This will ensure all stats are sent and stop statsd + // from doing anything more. + client.close(function(err) { + console.log('The close did not work quite right: ', err); + }); +``` + +## DogStatsD and Telegraf functionality + +Some of the functionality mentioned above is specific to DogStatsD or Telegraf. They will not do anything if you are using the regular statsd client. +* globalTags parameter- DogStatsD or Telegraf +* tags parameter- DogStatsD or Telegraf. +* telegraf parameter- Telegraf +* uds option in protocol parameter- DogStatsD +* histogram method- DogStatsD or Telegraf +* event method- DogStatsD +* check method- DogStatsD + +## Errors + +As usual, callbacks will have an error as their first parameter. You can have an error in both the message and close callbacks. + +If the optional callback is not given, an error is thrown in some +cases and a console.log message is used in others. An error will only +be explicitly thrown when there is a missing callback or if it is some potential configuration issue to be fixed. + +If you would like to ensure all errors are caught, specify an `errorHandler` in your root +client. This will catch errors in socket setup, sending of messages, +and closing of the socket. If you specify an errorHandler and a callback, the callback will take precedence. + +```javascript +// Using errorHandler +var client = new StatsD({ + errorHandler: function (error) { + console.log("Socket errors caught here: ", error); + } +}) +``` + +### Congestion error + +If you get an error like `Error sending hot-shots message: Error: congestion` with an error code of `1`, +it is probably because you are sending large volumes of metrics to a single agent/ server. +This error only arises when using the UDS protocol and means that packages are being dropped. +Take a look at the [Datadog docs](https://docs.datadoghq.com/developers/dogstatsd/high_throughput/?#over-uds-unix-domain-socket) for some tips on tuning your connection. + +## Unix domain socket support + +The 'uds' option as the protocol is to support [Unix Domain Sockets for Datadog](https://docs.datadoghq.com/developers/dogstatsd/unix_socket/). It has the following limitations: +- It only works where 'node-gyp' works. If you don't know what this is, this +is probably fine for you. If you had an troubles with libraries that +you 'node-gyp' before, you will have problems here as well. +- It does not work on Windows + +The above will cause the underlying library that is used, unix-dgram, +to not install properly. Given the library is listed as an +optionalDependency, and how it's used in the codebase, this install +failure will not cause any problems. It only means that you can't use +the uds feature. + +## Submitting changes + +Thanks for considering making any updates to this project! Here are the steps to take in your fork: + +1. Run "npm install" +2. Add your changes in your fork as well as any new tests needed +3. Run "npm test" +4. Update README.md with any needed documentation +5. If you have made any API changes, update types.d.ts +6. Push your changes and create the PR + +When you've done all this we're happy to try to get this merged in right away. + +## Package versioning and security + +Versions will attempt to follow semantic versioning, with major changes only coming in major versions. + +npm publishing is possible by one person, [bdeitte](https://github.com/bdeitte), who has two-factor authentication enabled for publishes. Publishes only contain one additional library, unix-dgram. + +## Name + +Why is this project named hot-shots? Because: + +1. It's impossible to find another statsd name on npm +2. It's the name of a dumb movie +3. No good reason + +## License + +hot-shots is licensed under the MIT license. diff --git a/tests/fixtures/npm/node_modules/hot-shots/index.js b/tests/fixtures/npm/node_modules/hot-shots/index.js new file mode 100644 index 000000000..a4dc5120c --- /dev/null +++ b/tests/fixtures/npm/node_modules/hot-shots/index.js @@ -0,0 +1 @@ +module.exports = require('./lib/statsd'); diff --git a/tests/fixtures/npm/node_modules/hot-shots/lib/constants.js b/tests/fixtures/npm/node_modules/hot-shots/lib/constants.js new file mode 100644 index 000000000..7fb2bc703 --- /dev/null +++ b/tests/fixtures/npm/node_modules/hot-shots/lib/constants.js @@ -0,0 +1,35 @@ +const os = require('os'), + process = require('process'); + +exports.PROTOCOL = { + TCP: 'tcp', + UDS: 'uds', + UDP: 'udp', + STREAM: 'stream' +}; + +/** + * Determines error codes that signify a connection to a Unix Domain Socket (UDS) + * has failed in a way that can be retried. This codes are OS-specific. + * @returns {number[]} An array of the error codes. + */ +function udsErrors() { + if (process.platform === 'linux') { + return [ + os.constants.errno.ENOTCONN, + os.constants.errno.ECONNREFUSED, + ]; + } + + if (process.platform === 'darwin') { + return [ + os.constants.errno.EDESTADDRREQ, + os.constants.errno.ECONNRESET, + ]; + } + + // Unknown / not yet implemented + return []; +} + +exports.udsErrors = udsErrors; diff --git a/tests/fixtures/npm/node_modules/hot-shots/lib/helpers.js b/tests/fixtures/npm/node_modules/hot-shots/lib/helpers.js new file mode 100644 index 000000000..d14f1682d --- /dev/null +++ b/tests/fixtures/npm/node_modules/hot-shots/lib/helpers.js @@ -0,0 +1,118 @@ +const fs = require('fs'); + +/** + * Replace any characters that can't be sent on with an underscore + */ +function sanitizeTags(value, telegraf) { + const blacklist = telegraf ? /:|\||,/g : /:|\||@|,/g; + // Replace reserved chars with underscores. + return String(value).replace(blacklist, '_'); +} + +/** + * Format tags properly before sending on + */ +function formatTags(tags, telegraf) { + if (Array.isArray(tags)) { + return tags; + + } else { + return Object.keys(tags).map(key => { + return `${sanitizeTags(key, telegraf)}:${sanitizeTags(tags[key], telegraf)}`; + }); + } +} + +/** + * Overrides tags in parent with tags from child with the same name (case sensitive) and return the result as new + * array. parent and child are not mutated. + */ +function overrideTags (parent, child, telegraf) { + const childCopy = {}; + const toAppend = []; + formatTags(child, telegraf).forEach(tag => { + const idx = typeof tag === 'string' ? tag.indexOf(':') : -1; + if (idx < 1) { // Not found or first character + toAppend.push(tag); + } else { + const key = tag.substring(0, idx); + const value = tag.substring(idx + 1); + childCopy[key] = childCopy[key] || []; + childCopy[key].push(value); + } + }); + const result = parent.filter(tag => { + const idx = typeof tag === 'string' ? tag.indexOf(':') : -1; + if (idx < 1) { // Not found or first character + return true; + } + + const key = tag.substring(0, idx); + + return !childCopy.hasOwnProperty(key); + }); + + Object.keys(childCopy).forEach(key => { + for (const value of childCopy[key]) { + result.push(`${key}:${value}`); + } + }); + return result.concat(toAppend); +} + +/** + * Formats a date for use with DataDog + */ +function formatDate(date) { + let timestamp; + if (date instanceof Date) { + // Datadog expects seconds. + timestamp = Math.round(date.getTime() / 1000); + } else if (date instanceof Number || typeof date === 'number') { + // Make sure it is an integer, not a float. + timestamp = Math.round(date); + } + return timestamp; +} + +/** + * Converts int to a string IP + */ +function intToIP(int) { + const part1 = int & 255; + const part2 = ((int >> 8) & 255); + const part3 = ((int >> 16) & 255); + const part4 = ((int >> 24) & 255); + + return `${part4}.${part3}.${part2}.${part1}`; +} + +/** + * Returns the system default interface on Linux + */ +function getDefaultRoute() { + try { + const fileContents = fs.readFileSync('/proc/net/route', 'utf8'); // eslint-disable-line no-sync + const routes = fileContents.split('\n'); + for (const routeIdx in routes) { + const fields = routes[routeIdx].trim().split('\t'); + if (fields[1] === '00000000') { + const address = fields[2]; + // Convert to little endian by splitting every 2 digits and reversing that list + const littleEndianAddress = address.match(/.{2}/g).reverse().join(''); + return intToIP(parseInt(littleEndianAddress, 16)); + } + } + } catch (e) { + console.error('Could not get default route from /proc/net/route'); + } + return null; +} + +module.exports = { + formatTags: formatTags, + overrideTags: overrideTags, + formatDate: formatDate, + getDefaultRoute: getDefaultRoute, + sanitizeTags: sanitizeTags +}; diff --git a/tests/fixtures/npm/node_modules/hot-shots/lib/statsFunctions.js b/tests/fixtures/npm/node_modules/hot-shots/lib/statsFunctions.js new file mode 100644 index 000000000..3735346ad --- /dev/null +++ b/tests/fixtures/npm/node_modules/hot-shots/lib/statsFunctions.js @@ -0,0 +1,349 @@ +const helpers = require('./helpers'); + +/** + * Separated out of statsd.js for clarity, these are the timing and other stats functions that are what are called the most + * when using hot-shots + */ +function applyStatsFns (Client) { + + /** + * Represents the timing stat + * @param stat {String|Array} The stat(s) to send + * @param time {Number|Date} The time in milliseconds to send or Date object of which the difference is calculated + * @param sampleRate {Number=} The Number of times to sample (0 to 1). Optional. + * @param tags {Array=} The Array of tags to add to metrics. Optional. + * @param callback {Function=} Callback when message is done being delivered. Optional. + */ + Client.prototype.timing = function (stat, time, sampleRate, tags, callback) { + const t = time instanceof Date ? new Date() - time : time; + this.sendAll(stat, t, 'ms', sampleRate, tags, callback); + }; + + /** + * Represents the timing stat by recording the duration a function takes to run (in milliseconds) + * @param func {Function} The function to run + * @param stat {String|Array} The stat(s) to send + * @param sampleRate {Number=} The Number of times to sample (0 to 1). Optional. + * @param tags {Array=} The Array of tags to add to metrics. Optional. + * @param callback {Function=} Callback when message is done being delivered. Optional. + */ + Client.prototype.timer = function (func, stat, sampleRate, tags, callback) { + const _this = this; + + return (...args) => { + const start = process.hrtime(); + try { + return func(...args); + } finally { + // get duration in milliseconds + const durationComponents = process.hrtime(start); + const seconds = durationComponents[0]; + const nanoseconds = durationComponents[1]; + const duration = (seconds * 1000) + (nanoseconds / 1E6); + + _this.timing( + stat, + duration, + sampleRate, + tags, + callback + ); + } + }; + }; + + /** + * Decorates an async function with timing recording behaviour. + * + * This version of `timer` will record the time take for the asynchronous action returned by `func` + * not just the execution time of `func` itself. + * + * @param func {(...A):Promise} The function to run + * @param stat {String|Array} The stat(s) to send + * @param sampleRate {Number=} The Number of times to sample (0 to 1). Optional. + * @param tags {Array=} The Array of tags to add to metrics. Optional. + * @param callback {Function=} Callback when message is done being delivered. Optional. + */ + Client.prototype.asyncTimer = function (func, stat, sampleRate, tags, callback) { + const self = this; + return (...args) => { + const end = hrtimer(); + const p = func(...args); + const recordStat = () => { self.timing(stat, end(), sampleRate, tags, callback); }; + p.then(recordStat, recordStat); + return p; + }; + }; + + /** + * Decorates an async function with timing recording behaviour, reported as a distribution. + * + * This version of `timer` will record the time take for the asynchronous action returned by `func` + * not just the execution time of `func` itself. + * + * @param func {(...A):Promise} The function to run + * @param stat {String|Array} The stat(s) to send + * @param sampleRate {Number=} The Number of times to sample (0 to 1). Optional. + * @param tags {Array=} The Array of tags to add to metrics. Optional. + * @param callback {Function=} Callback when message is done being delivered. Optional. + */ + Client.prototype.asyncDistTimer = function (func, stat, sampleRate, tags, callback) { + const self = this; + return (...args) => { + const end = hrtimer(); + const p = func(...args); + const recordStat = () => self.distribution(stat, end(), sampleRate, tags, callback); + p.then(recordStat, recordStat); + return p; + }; + }; + + /** + * High-resolution timer + */ + function hrtimer() { + const start = process.hrtime(); + + return () => { + const durationComponents = process.hrtime(start); + const seconds = durationComponents[0]; + const nanoseconds = durationComponents[1]; + const duration = (seconds * 1000) + (nanoseconds / 1E6); + return duration; + }; + } + + + /** + * Increments a stat by a specified amount + * @param stat {String|Array} The stat(s) to send + * @param value The value to send + * @param sampleRate {Number=} The Number of times to sample (0 to 1). Optional. + * @param tags {Array=} The Array of tags to add to metrics. Optional. + * @param callback {Function=} Callback when message is done being delivered. Optional. + */ + Client.prototype.increment = function (stat, value, sampleRate, tags, callback) { + // allow use of tags without explicit value or sampleRate + if (arguments.length < 3) { + if (typeof value !== 'number') { + tags = value; + value = undefined; + } + } + // we explicitly check for undefined and null (and don't do a "! value" check) + // so that 0 values are allowed and sent through as-is + if (value === undefined || value === null) { + value = 1; + } + this.sendAll(stat, value, 'c', sampleRate, tags, callback); + }; + + /** + * Decrements a stat by a specified amount + * @param stat {String|Array} The stat(s) to send + * @param value The value to send + * @param sampleRate {Number=} The Number of times to sample (0 to 1). Optional. + * @param tags {Array=} The Array of tags to add to metrics. Optional. + * @param callback {Function=} Callback when message is done being delivered. Optional. + */ + Client.prototype.decrement = function (stat, value, sampleRate, tags, callback) { + // allow use of tags without explicit value or sampleRate + if (arguments.length < 3) { + if (typeof value !== 'number') { + tags = value; + value = undefined; + } + } + // we explicitly check for undefined and null (and don't do a "! value" check) + // so that 0 values are allowed and sent through as-is + if (value === undefined || value === null) { + value = 1; + } + + this.sendAll(stat, -value, 'c', sampleRate, tags, callback); + }; + + /** + * Represents the histogram stat + * @param stat {String|Array} The stat(s) to send + * @param value The value to send + * @param sampleRate {Number=} The Number of times to sample (0 to 1). Optional. + * @param tags {Array=} The Array of tags to add to metrics. Optional. + * @param callback {Function=} Callback when message is done being delivered. Optional. + */ + Client.prototype.histogram = function (stat, value, sampleRate, tags, callback) { + this.sendAll(stat, value, 'h', sampleRate, tags, callback); + }; + + /** + * Represents the distribution stat + * @param stat {String|Array} The stat(s) to send + * @param value The value to send + * @param sampleRate {Number=} The Number of times to sample (0 to 1). Optional. + * @param tags {Array=} The Array of tags to add to metrics. Optional. + * @param callback {Function=} Callback when message is done being delivered. Optional. + */ + Client.prototype.distribution = function (stat, value, sampleRate, tags, callback) { + this.sendAll(stat, value, 'd', sampleRate, tags, callback); + }; + + + /** + * Gauges a stat by a specified amount + * @param stat {String|Array} The stat(s) to send + * @param value The value to send + * @param sampleRate {Number=} The Number of times to sample (0 to 1). Optional. + * @param tags {Array=} The Array of tags to add to metrics. Optional. + * @param callback {Function=} Callback when message is done being delivered. Optional. + */ + Client.prototype.gauge = function (stat, value, sampleRate, tags, callback) { + this.sendAll(stat, value, 'g', sampleRate, tags, callback); + }; + + /** + * Counts unique values by a specified amount + * @param stat {String|Array} The stat(s) to send + * @param value The value to send + * @param sampleRate {Number=} The Number of times to sample (0 to 1). Optional. + * @param tags {Array=} The Array of tags to add to metrics. Optional. + * @param callback {Function=} Callback when message is done being delivered. Optional. + */ + Client.prototype.unique = Client.prototype.set = function (stat, value, sampleRate, tags, callback) { + this.sendAll(stat, value, 's', sampleRate, tags, callback); + }; + + /** + * Send a service check + * @param name {String} The name of the service check + * @param status {Number=} The status of the service check (0 to 3). + * @param options + * @option date_happened {Date} Assign a timestamp to the event. Default is now. + * @option hostname {String} Assign a hostname to the check. + * @option message {String} Assign a message to the check. + * @param tags {Array=} The Array of tags to add to the check. Optional. + * @param callback {Function=} Callback when message is done being delivered. Optional. + */ + Client.prototype.check = function (name, status, options, tags, callback) { + if (this.telegraf) { + const err = new Error('Not supported by Telegraf / InfluxDB'); + if (callback) { + return callback(err); + } + else if (this.errorHandler) { + return this.errorHandler(err); + } + + throw err; + } + + const check = ['_sc', this.prefix + name + this.suffix, status], metadata = options || {}; + + if (metadata.date_happened) { + const timestamp = helpers.formatDate(metadata.date_happened); + if (timestamp) { + check.push(`d:${timestamp}`); + } + } + if (metadata.hostname) { + check.push(`h:${metadata.hostname}`); + } + + let mergedTags = this.globalTags; + if (tags && typeof(tags) === 'object') { + mergedTags = helpers.overrideTags(mergedTags, tags, this.telegraf); + } + if (mergedTags.length > 0) { + check.push(`#${mergedTags.join(',')}`); + } + + // message has to be the last part of a service check + if (metadata.message) { + check.push(`m:${metadata.message}`); + } + + // allow for tags to be omitted and callback to be used in its place + if (typeof tags === 'function' && callback === undefined) { + callback = tags; + } + + const message = check.join('|'); + // Service checks are unique in that message has to be the last element in + // the stat if provided, so we can't append tags like other checks. This + // directly calls the `_send` method to avoid appending tags, since we've + // already added them. + this._send(message, callback); + }; + + /** + * Send on an event + * @param title {String} The title of the event + * @param text {String} The description of the event. Optional- title is used if not given. + * @param options + * @option date_happened {Date} Assign a timestamp to the event. Default is now. + * @option hostname {String} Assign a hostname to the event. + * @option aggregation_key {String} Assign an aggregation key to the event, to group it with some others. + * @option priority {String} Can be ‘normal’ or ‘low’. Default is 'normal'. + * @option source_type_name {String} Assign a source type to the event. + * @option alert_type {String} Can be ‘error’, ‘warning’, ‘info’ or ‘success’. Default is 'info'. + * @param tags {Array=} The Array of tags to add to metrics. Optional. + * @param callback {Function=} Callback when message is done being delivered. Optional. + */ + Client.prototype.event = function (title, text, options, tags, callback) { + if (this.telegraf) { + const err = new Error('Not supported by Telegraf / InfluxDB'); + if (callback) { + return callback(err); + } + else if (this.errorHandler) { + return this.errorHandler(err); + } + + throw err; + } + + // Convert to strings + let message; + + const msgTitle = String(title ? title : ''); + let msgText = String(text ? text : msgTitle); + // Escape new lines (unescaping is supported by DataDog) + msgText = msgText.replace(/\n/g, '\\n'); + + // start out the message with the event-specific title and text info + message = `_e{${msgTitle.length},${msgText.length}}:${msgTitle}|${msgText}`; + + // add in the event-specific options + if (options) { + if (options.date_happened) { + const timestamp = helpers.formatDate(options.date_happened); + if (timestamp) { + message += `|d:${timestamp}`; + } + } + if (options.hostname) { + message += `|h:${options.hostname}`; + } + if (options.aggregation_key) { + message += `|k:${options.aggregation_key}`; + } + if (options.priority) { + message += `|p:${options.priority}`; + } + if (options.source_type_name) { + message += `|s:${options.source_type_name}`; + } + if (options.alert_type) { + message += `|t:${options.alert_type}`; + } + } + + // allow for tags to be omitted and callback to be used in its place + if (typeof tags === 'function' && callback === undefined) { + callback = tags; + } + + this.send(message, tags, callback); + }; +} + +module.exports = applyStatsFns; diff --git a/tests/fixtures/npm/node_modules/hot-shots/lib/statsd.js b/tests/fixtures/npm/node_modules/hot-shots/lib/statsd.js new file mode 100644 index 000000000..5a46d6064 --- /dev/null +++ b/tests/fixtures/npm/node_modules/hot-shots/lib/statsd.js @@ -0,0 +1,598 @@ +const process = require('process'), + util = require('util'), + helpers = require('./helpers'), + applyStatsFns = require('./statsFunctions'); + +const constants = require('./constants'); +const createTransport = require('./transport'); + +const PROTOCOL = constants.PROTOCOL; +const UDS_ERROR_CODES = constants.udsErrors(); +const UDS_DEFAULT_GRACEFUL_RESTART_LIMIT = 1000; +const CACHE_DNS_TTL_DEFAULT = 60000; + +/** + * The Client for StatsD. The main entry-point for hot-shots. Note adding new parameters + * to the constructor is deprecated- please use the constructor as one options object. + * @constructor + */ +const Client = function (host, port, prefix, suffix, globalize, cacheDns, mock, + globalTags, maxBufferSize, bufferFlushInterval, telegraf, sampleRate, protocol) { + let options = host || {}; + + // Adding options below is DEPRECATED. Use the options object instead. + if (arguments.length > 1 || typeof(host) === 'string') { + options = { + host : host, + port : port, + prefix : prefix, + suffix : suffix, + globalize : globalize, + cacheDns : cacheDns, + mock : mock === true, + globalTags : globalTags, + maxBufferSize : maxBufferSize, + bufferFlushInterval: bufferFlushInterval, + telegraf : telegraf, + sampleRate : sampleRate, + protocol : protocol + }; + } + + + // hidden global_tags option for backwards compatibility + options.globalTags = options.globalTags || options.global_tags; + + this.protocol = (options.protocol && options.protocol.toLowerCase()); + if (! this.protocol) { + this.protocol = PROTOCOL.UDP; + } + this.cacheDns = options.cacheDns === true; + this.cacheDnsTtl = options.cacheDnsTtl || CACHE_DNS_TTL_DEFAULT; + this.host = options.host || process.env.DD_AGENT_HOST; + this.port = options.port || parseInt(process.env.DD_DOGSTATSD_PORT, 10) || 8125; + this.path = options.path; + this.stream = options.stream; + this.prefix = options.prefix || ''; + this.suffix = options.suffix || ''; + this.tagPrefix = options.tagPrefix || '#'; + this.tagSeparator = options.tagSeparator || ','; + this.mock = options.mock; + this.globalTags = typeof options.globalTags === 'object' ? + helpers.formatTags(options.globalTags, options.telegraf) : []; + if (process.env.DD_ENTITY_ID) { + this.globalTags = this.globalTags.filter((item) => { + return item.indexOf('dd.internal.entity_id:') !== 0; + }); + this.globalTags.push('dd.internal.entity_id:'.concat(helpers.sanitizeTags(process.env.DD_ENTITY_ID))); + } + this.telegraf = options.telegraf || false; + this.maxBufferSize = options.maxBufferSize || 0; + this.sampleRate = options.sampleRate || 1; + this.bufferFlushInterval = options.bufferFlushInterval || 1000; + this.bufferHolder = options.isChild ? options.bufferHolder : { buffer: '' }; + this.errorHandler = options.errorHandler; + this.udsGracefulErrorHandling = 'udsGracefulErrorHandling' in options ? options.udsGracefulErrorHandling : true; + this.udsGracefulRestartRateLimit = options.udsGracefulRestartRateLimit || UDS_DEFAULT_GRACEFUL_RESTART_LIMIT; // only recreate once per second + this.isChild = options.isChild; + this.closingFlushInterval = options.closingFlushInterval || 50; + + // If we're mocking the client, create a buffer to record the outgoing calls. + if (this.mock) { + this.mockBuffer = []; + } + + // We only want a single flush event per parent and all its child clients + if (!options.isChild && this.maxBufferSize > 0) { + this.intervalHandle = setInterval(this.onBufferFlushInterval.bind(this), this.bufferFlushInterval); + // do not block node from shutting down + this.intervalHandle.unref(); + } + + if (options.isChild) { + if (options.dnsError) { + this.dnsError = options.dnsError; + } + this.socket = options.socket; + } else if (options.useDefaultRoute) { + const defaultRoute = helpers.getDefaultRoute(); + if (defaultRoute) { + console.log(`Got ${defaultRoute} for the system's default route`); + this.host = defaultRoute; + } + } + + if (!this.socket) { + trySetNewSocket(this); + } + + if (this.socket && !options.isChild && options.errorHandler) { + this.socket.on('error', options.errorHandler); + } + + if (options.globalize) { + global.statsd = this; + } + + // only for uds (options.protocol uds) + // enabled with the extra flag options.udsGracefulErrorHandling + // will gracefully (attempt) to re-open the socket with a small delay + // options.udsGracefulRestartRateLimit is the minimum time (ms) between creating sockets + // does not support options.isChild (how to re-create a socket you didn't create?) + if (this.socket && options.protocol === PROTOCOL.UDS) { + maybeAddUDSErrorHandler(this); + } + + this.messagesInFlight = 0; + this.CHECKS = { + OK: 0, + WARNING: 1, + CRITICAL: 2, + UNKNOWN: 3, + }; +}; + +applyStatsFns(Client); + +/** + * Checks if stats is an array and sends all stats calling back once all have sent + * @param stat {String|Array} The stat(s) to send + * @param value The value to send + * @param type The type of the metric + * @param sampleRate {Number=} The Number of times to sample (0 to 1). Optional. + * @param tags {Array=} The Array of tags to add to metrics. Optional. + * @param callback {Function=} Callback when message is done being delivered. Optional. + */ +Client.prototype.sendAll = function (stat, value, type, sampleRate, tags, callback) { + let completed = 0; + let calledback = false; + let sentBytes = 0; + const self = this; + + if (sampleRate && typeof sampleRate !== 'number') { + callback = tags; + tags = sampleRate; + sampleRate = undefined; + } + + if (tags && typeof tags !== 'object') { + callback = tags; + tags = undefined; + } + + /** + * Gets called once for each callback, when all callbacks return we will + * call back from the function + * @private + */ + function onSend(error, bytes) { + completed += 1; + if (calledback) { + return; + } + + if (error) { + if (typeof callback === 'function') { + calledback = true; + callback(error); + } else if (self.errorHandler) { + calledback = true; + self.errorHandler(error); + } + return; + } + + if (bytes) { + sentBytes += bytes; + } + + if (completed === stat.length && typeof callback === 'function') { + callback(null, sentBytes); + } + } + + if (Array.isArray(stat)) { + stat.forEach(item => { + self.sendStat(item, value, type, sampleRate, tags, onSend); + }); + } else { + this.sendStat(stat, value, type, sampleRate, tags, callback); + } +}; + +/** + * Sends a stat across the wire + * @param stat {String|Array} The stat(s) to send + * @param value The value to send + * @param type {String} The type of message to send to statsd + * @param sampleRate {Number} The Number of times to sample (0 to 1) + * @param tags {Array} The Array of tags to add to metrics + * @param callback {Function=} Callback when message is done being delivered. Optional. + */ +Client.prototype.sendStat = function (stat, value, type, sampleRate, tags, callback) { + let message = `${this.prefix + stat + this.suffix}:${value}|${type}`; + sampleRate = sampleRate || this.sampleRate; + if (sampleRate && sampleRate < 1) { + if (Math.random() < sampleRate) { + message += `|@${sampleRate}`; + } else { + // don't want to send if we don't meet the sample ratio + return callback ? callback() : undefined; + } + } + this.send(message, tags, callback); +}; + +/** + * Send a stat or event across the wire + * @param message {String} The constructed message without tags + * @param tags {Array} The tags to include (along with global tags). Optional. + * @param callback {Function=} Callback when message is done being delivered (only if maxBufferSize == 0). Optional. + */ +Client.prototype.send = function (message, tags, callback) { + let mergedTags = this.globalTags; + if (tags && typeof tags === 'object') { + mergedTags = helpers.overrideTags(mergedTags, tags, this.telegraf); + } + if (mergedTags.length > 0) { + if (this.telegraf) { + message = message.split(':'); + message = `${message[0]},${mergedTags.join(',').replace(/:/g, '=')}:${message.slice(1).join(':')}`; + } else { + message += `|${this.tagPrefix}${mergedTags.join(this.tagSeparator)}`; + } + } + + this._send(message, callback); +}; + +/** + * Send a stat or event across the wire + * @param message {String} The constructed message without tags + * @param callback {Function=} Callback when message is done being delivered (only if maxBufferSize == 0). Optional. + */ +Client.prototype._send = function (message, callback) { + // we may have a cached error rather than a cached lookup, so + // throw it on + if (this.dnsError) { + if (callback) { + return callback(this.dnsError); + } else if (this.errorHandler) { + return this.errorHandler(this.dnsError); + } + throw this.dnsError; + } + + // Only send this stat if we're not a mock Client. + if (!this.mock) { + if (this.maxBufferSize === 0) { + this.sendMessage(message, callback); + } else { + this.enqueue(message, callback); + } + } else { + this.mockBuffer.push(message); + if (typeof callback === 'function') { + callback(null, 0); + } + } +}; + +/** + * Add the message to the buffer and flush the buffer if needed + * + * @param message {String} The constructed message without tags + */ +Client.prototype.enqueue = function (message, callback) { + message += '\n'; + + if (this.bufferHolder.buffer.length + message.length > this.maxBufferSize) { + this.flushQueue(callback); + this.bufferHolder.buffer += message; + } + else { + this.bufferHolder.buffer += message; + if (callback) { + callback(null); + } + } +}; + +/** + * Flush the buffer, sending on the messages + */ +Client.prototype.flushQueue = function (callback) { + this.sendMessage(this.bufferHolder.buffer, callback); + this.bufferHolder.buffer = ''; +}; + +/** + * Send on the message through the socket + * + * @param message {String} The constructed message without tags + * @param callback {Function=} Callback when message is done being delivered. Optional. + */ +Client.prototype.sendMessage = function (message, callback) { + // don't waste the time if we aren't sending anything + if (message === '') { + if (callback) { + callback(null); + } + return; + } + + const socketWasMissing = !this.socket; + if (socketWasMissing && this.protocol === PROTOCOL.UDS) { + trySetNewSocket(this); + if (this.socket) { + // On success, add custom UDS error handling. + maybeAddUDSErrorHandler(this, Date.now()); + } + } + + if (socketWasMissing) { + const error = new Error('Socket not created properly. Check previous errors for details.'); + if (callback) { + return callback(error); + } else if (this.errorHandler) { + return this.errorHandler(error); + } else { + return console.error(String(error)); + } + } + + const handleCallback = (err, bytes) => { + this.messagesInFlight--; + const errFormatted = err ? new Error(`Error sending hot-shots message: ${err}`) : null; + if (errFormatted) { + errFormatted.code = err.code; + // handle UDS error that requires socket replacement when we are not + // emitting the `error` event on `this.socket` + if (this.protocol === PROTOCOL.UDS && (callback || this.errorHandler)) { + udsErrorHandler(this, err); + } + } + if (callback) { + callback(errFormatted, bytes); + } else if (errFormatted) { + if (this.errorHandler) { + this.errorHandler(errFormatted); + } else { + console.error(String(errFormatted)); + // emit error ourselves on the socket for backwards compatibility + this.socket.emit('error', errFormatted); + } + } + }; + + try { + this.messagesInFlight++; + this.socket.send(Buffer.from(message), handleCallback); + } catch (err) { + handleCallback(err); + } +}; + +/** + * Called every bufferFlushInterval to flush any buffer that is around + */ +Client.prototype.onBufferFlushInterval = function () { + this.flushQueue(); +}; + +/** + * Close the underlying socket and stop listening for data on it. + */ +Client.prototype.close = function (callback) { + // stop trying to flush the queue on an interval + if (this.intervalHandle) { + clearInterval(this.intervalHandle); + } + + // flush the queue one last time, if needed + this.flushQueue((err) => { + if (err) { + if (callback) { + return callback(err); + } + else { + return console.error(err); + } + } + + // FIXME: we have entered callback hell, and this whole file is in need of an async rework + + // wait until there are no more messages in flight before really closing the socket + let intervalAttempts = 0; + const waitForMessages = setInterval(() => { + intervalAttempts++; + if (intervalAttempts > 10) { + console.log('hot-shots could not clear out messages in flight but closing anyways'); + this.messagesInFlight = 0; + } + if (this.messagesInFlight <= 0) { + clearInterval(waitForMessages); + this._close(callback); + } + }, this.closingFlushInterval); + }); +}; + +/** + * Really close the socket and handle any errors related to it + */ +Client.prototype._close = function (callback) { + // error function to use in callback and catch below + let handledError = false; + const handleErr = (err) => { + const errMessage = `Error closing hot-shots socket: ${err}`; + if (handledError) { + console.error(errMessage); + } + else { + // The combination of catch and error can lead to some errors + // showing up twice. So we just show one of the errors that occur + // on close. + handledError = true; + + if (callback) { + callback(new Error(errMessage)); + } else if (this.errorHandler) { + this.errorHandler(new Error(errMessage)); + } else { + console.error(errMessage); + } + } + }; + + if (this.errorHandler) { + this.socket.removeListener('error', this.errorHandler); + } + + // handle error and close events + this.socket.on('error', handleErr); + if (callback) { + this.socket.on('close', err => { + if (! handledError && callback) { + callback(err); + } + }); + } + + try { + this.socket.close(); + } catch (err) { + handleErr(err); + } +}; + +const ChildClient = function (parent, options) { + options = options || {}; + Client.call(this, { + isChild : true, + socket : parent.socket, // Child inherits socket from parent. Parent itself can be a child. + // All children and parent share the same buffer via sharing an object (cannot mutate strings) + bufferHolder: parent.bufferHolder, + dnsError : parent.dnsError, // Child inherits an error from parent (if it is there) + errorHandler: options.errorHandler || parent.errorHandler, // Handler for callback errors + host : parent.host, + port : parent.port, + tagPrefix : parent.tagPrefix, + tagSeparator : parent.tagSeparator, + prefix : (options.prefix || '') + parent.prefix, // Child has its prefix prepended to parent's prefix + suffix : parent.suffix + (options.suffix || ''), // Child has its suffix appended to parent's suffix + globalize : false, // Only 'root' client can be global + mock : parent.mock, + // Append child's tags to parent's tags + globalTags : typeof options.globalTags === 'object' ? + helpers.overrideTags(parent.globalTags, options.globalTags, parent.telegraf) : parent.globalTags, + maxBufferSize : parent.maxBufferSize, + bufferFlushInterval: parent.bufferFlushInterval, + telegraf : parent.telegraf, + protocol : parent.protocol, + closingFlushInterval : parent.closingFlushInterval + }); +}; +util.inherits(ChildClient, Client); + +/** + * Creates a child client that adds prefix, suffix and/or tags to this client. Child client can itself have children. + * @param options + * @option prefix {String} An optional prefix to assign to each stat name sent + * @option suffix {String} An optional suffix to assign to each stat name sent + * @option globalTags {Array=} Optional tags that will be added to every metric + */ +Client.prototype.childClient = function (options) { + return new ChildClient(this, options); +}; + +exports = module.exports = Client; +exports.StatsD = Client; + +/** + * Detect and handle an error connecting to a Unix Domain Socket (UDS). This will + * attempt to create a new socket and replace and close the client's current + * socket, registering a **new** `udsErrorHandler()` on the newly created socket. + * If a new socket can't be created (e.g. if no UDS currently exists at + * `client.path`) then this will leave the existing socket intact. + * + * Note that this will no-op with an early exit if the last socket create time + * was too recent (within the UDS graceful restart rate limit). + * @param client Client The statsd Client that may be getting a UDS error handler. + * @param err The error that we will handle if a UDS connection error is detected. + */ +function udsErrorHandler(client, err) { + if (!err || !UDS_ERROR_CODES.includes(-err.code)) { + return; + } + + // recreate the socket, but only once within `udsGracefulRestartRateLimit`. + if (!client.socket.createdAt || + Date.now() - client.socket.createdAt < client.udsGracefulRestartRateLimit) { + return; + } + + if (client.errorHandler) { + client.socket.removeListener('error', client.errorHandler); + } + + const newSocket = createTransport(client, { + host: client.host, + path: client.path, + port: client.port, + protocol: client.protocol, + }); + if (newSocket) { + client.socket.close(); + client.socket = newSocket; + maybeAddUDSErrorHandler(client); + } else { + const errorMessage = 'Could not replace UDS connection with new socket'; + if (client.errorHandler) { + client.errorHandler(new Error(errorMessage)); + } else { + console.error(errorMessage); + } + return; + } + + if (client.errorHandler) { + client.socket.on('error', client.errorHandler); + } else { + client.socket.on('error', (error) => console.error(`hot-shots UDS error: ${error}`)); + } +} + +/** + * Add a Unix Domain Socket (UDS) error handler to the client's socket, if the + * client is not a "child" client and has graceful error handling enabled for + * UDS. + * @param client Client The statsd Client that may be getting a UDS error handler. + */ +function maybeAddUDSErrorHandler(client) { + if (client.isChild || !client.udsGracefulErrorHandling) { + return; + } + + client.socket.on('error', (err) => { + udsErrorHandler(client, err); + }); +} + +/** + * Try to replace a client's socket with a new transport. If `createTransport()` + * returns `null` this will still set the client's socket to `null`. This also + * updates the socket creation time for UDS error handling. + * @param client Client The statsd Client that will be getting a new socket + */ +function trySetNewSocket(client) { + client.socket = createTransport(client, { + host: client.host, + cacheDns: client.cacheDns, + cacheDnsTtl: client.cacheDnsTtl, + path: client.path, + port: client.port, + protocol: client.protocol, + stream: client.stream, + }); +} diff --git a/tests/fixtures/npm/node_modules/hot-shots/lib/transport.js b/tests/fixtures/npm/node_modules/hot-shots/lib/transport.js new file mode 100644 index 000000000..4cd699c6d --- /dev/null +++ b/tests/fixtures/npm/node_modules/hot-shots/lib/transport.js @@ -0,0 +1,177 @@ +const assert = require('assert'); +const dgram = require('dgram'); +const net = require('net'); +const dns = require('dns'); +const { PROTOCOL } = require('./constants'); + +// Imported below, only if needed +let unixDgram; + +const UDS_PATH_DEFAULT = '/var/run/datadog/dsd.socket'; + +const addEol = (buf) => { + let msg = buf.toString(); + if (msg.length > 0 && msg[msg.length - 1] !== '\n') { + msg += '\n'; + } + return msg; +}; + +// interface Transport { +// emit(name: string, payload: any):void; +// on(name: string, listener: Function):void; +// removeListener(name: string, listener: Function):void; +// send(buf: Buffer, callback: Function):void; +// close():void; +// unref(): void; +// } +const createTcpTransport = args => { + const socket = net.connect(args.port, args.host); + socket.setKeepAlive(true); + // do not block node from shutting down + socket.unref(); + return { + emit: socket.emit.bind(socket), + on: socket.on.bind(socket), + removeListener: socket.removeListener.bind(socket), + send: (buf, callback) => { + socket.write(addEol(buf), 'ascii', callback); + }, + close: () => socket.destroy(), + unref: socket.unref.bind(socket) + + }; +}; + +const createUdpTransport = args => { + const socket = dgram.createSocket('udp4'); + // do not block node from shutting down + socket.unref(); + + const dnsResolutionData = { + timestamp: new Date(0), + resolvedAddress: undefined + }; + + const sendUsingDnsCache = (callback, buf) => { + const now = Date.now(); + if (dnsResolutionData.resolvedAddress === undefined || (now - dnsResolutionData.timestamp > args.cacheDnsTtl)) { + dns.lookup(args.host, (error, address) => { + if (error) { + callback(error); + return; + } + dnsResolutionData.resolvedAddress = address; + dnsResolutionData.timestamp = now; + socket.send(buf, 0, buf.length, args.port, dnsResolutionData.resolvedAddress, callback); + }); + } else { + socket.send(buf, 0, buf.length, args.port, dnsResolutionData.resolvedAddress, callback); + } + }; + + return { + emit: socket.emit.bind(socket), + on: socket.on.bind(socket), + removeListener: socket.removeListener.bind(socket), + send: function (buf, callback) { + if (args.cacheDns) { + sendUsingDnsCache(callback, buf); + } else { + socket.send(buf, 0, buf.length, args.port, args.host, callback); + } + }, + close: socket.close.bind(socket), + unref: socket.unref.bind(socket) + }; +}; + +const createUdsTransport = args => { + try { + // This will not always be available, as noted in the error message below + unixDgram = require('unix-dgram'); // eslint-disable-line global-require + } catch (err) { + throw new Error( + 'The library `unix_dgram`, needed for the uds protocol to work, is not installed. ' + + 'You need to pick another protocol to use hot-shots. ' + + 'See the hot-shots README for additional details.' + ); + } + const udsPath = args.path ? args.path : UDS_PATH_DEFAULT; + const socket = unixDgram.createSocket('unix_dgram'); + + try { + socket.connect(udsPath); + } catch (err) { + socket.close(); + throw err; + } + + return { + emit: socket.emit.bind(socket), + on: socket.on.bind(socket), + removeListener: socket.removeListener.bind(socket), + send: socket.send.bind(socket), + close: () => { + socket.close(); + // close is synchronous, and the socket will not emit a + // close event, hence emulating standard behaviour by doing this: + socket.emit('close'); + }, + unref: () => { + throw new Error('unix-dgram does not implement unref for sockets'); + } + }; +}; + +const createStreamTransport = (args) => { + const stream = args.stream; + assert(stream, '`stream` option required'); + + return { + emit: stream.emit.bind(stream), + on: stream.on.bind(stream), + removeListener: stream.removeListener.bind(stream), + send: (buf, callback) => stream.write(addEol(buf), callback), + close: () => { + stream.destroy(); + + // Node v8 doesn't fire `close` event on stream destroy. + if (process.version.split('.').shift() === 'v8') { + stream.emit('close'); + } + }, + unref: () => { + throw new Error('stream transport does not support unref'); + } + }; +}; + +module.exports = (instance, args) => { + let transport = null; + const protocol = args.protocol || PROTOCOL.UDP; + + try { + if (protocol === PROTOCOL.TCP) { + transport = createTcpTransport(args); + } else if (protocol === PROTOCOL.UDS) { + transport = createUdsTransport(args); + } else if (protocol === PROTOCOL.UDP) { + transport = createUdpTransport(args); + } else if (protocol === PROTOCOL.STREAM) { + transport = createStreamTransport(args); + } else { + throw new Error(`Unsupported protocol '${protocol}'`); + } + transport.type = protocol; + transport.createdAt = Date.now(); + } catch (e) { + if (instance.errorHandler) { + instance.errorHandler(e); + } else { + console.error(e); + } + } + + return transport; +}; diff --git a/tests/fixtures/npm/node_modules/hot-shots/package.json b/tests/fixtures/npm/node_modules/hot-shots/package.json new file mode 100644 index 000000000..f4c2d22f5 --- /dev/null +++ b/tests/fixtures/npm/node_modules/hot-shots/package.json @@ -0,0 +1,47 @@ +{ + "name": "hot-shots", + "description": "Node.js client for StatsD, DogStatsD, and Telegraf", + "version": "8.5.0", + "author": "Steve Ivy", + "types": "./types.d.ts", + "contributors": [ + "Russ Bradberry ", + "Brian Deitte " + ], + "keywords": [ + "statsd", + "dogstatsd", + "datadog", + "metrics", + "telegraf" + ], + "repository": { + "type": "git", + "url": "git://github.com/brightcove/hot-shots.git" + }, + "bugs": { + "url": "https://github.com/brightcove/hot-shots/issues" + }, + "directories": { + "lib": "./lib/" + }, + "engines": { + "node": ">=6.0.0" + }, + "scripts": { + "coverage": "nyc --reporter=lcov --reporter=text npm test", + "test": "mocha -R spec --timeout 5000 test/*.js", + "lint": "eslint lib/**/*.js test/**/*.js", + "pretest": "npm run lint" + }, + "optionalDependencies": { + "unix-dgram": "2.0.x" + }, + "devDependencies": { + "eslint": "5.9.x", + "mocha": "6.2.x", + "nyc": "15.x" + }, + "license": "MIT", + "dependencies": {} +} diff --git a/tests/fixtures/npm/node_modules/hot-shots/types.d.ts b/tests/fixtures/npm/node_modules/hot-shots/types.d.ts new file mode 100644 index 000000000..01dce461a --- /dev/null +++ b/tests/fixtures/npm/node_modules/hot-shots/types.d.ts @@ -0,0 +1,147 @@ +import dgram = require("dgram"); + +declare module "hot-shots" { + export type Tags = { [key: string]: string } | string[]; + export interface ClientOptions { + bufferFlushInterval?: number; + bufferHolder?: { buffer: string }; + cacheDns?: boolean; + cacheDnsTtl?: number; + errorHandler?: (err: Error) => void; + globalTags?: Tags; + globalize?: boolean; + host?: string; + isChild?: boolean; + maxBufferSize?: number; + mock?: boolean; + path?: string; + port?: number; + prefix?: string; + protocol?: 'tcp' | 'udp' | 'uds'; + sampleRate?: number; + socket?: dgram.Socket; + suffix?: string; + telegraf?: boolean; + useDefaultRoute?: boolean; + tagPrefix?: string; + tagSeparator?: string; + udsGracefulErrorHandling?: boolean; + udsGracefulRestartRateLimit?: number; + closingFlushInterval?: number; + } + + export interface ChildClientOptions { + globalTags?: Tags; + prefix?: string; + suffix?: string; + } + + export interface CheckOptions { + date_happened?: Date; + hostname?: string; + message?: string; + } + + export interface DatadogChecks { + OK: 0; + WARNING: 1; + CRITICAL: 2; + UNKNOWN: 3; + } + + type unionFromInterfaceValues4< + T, + K1 extends keyof T, + K2 extends keyof T, + K3 extends keyof T, + K4 extends keyof T, + > = T[K1] | T[K2] | T[K3] | T[K4]; + + export type DatadogChecksValues = unionFromInterfaceValues4; + + export interface EventOptions { + aggregation_key?: string; + alert_type?: "info" | "warning" | "success" | "error"; + date_happened?: Date; + hostname?: string; + priority?: "low" | "normal"; + source_type_name?: string; + } + + export type StatsCb = (error: Error | undefined, bytes: any) => void; + + export class StatsD { + constructor(options?: ClientOptions); + childClient(options?: ChildClientOptions): StatsD; + + increment(stat: string, tags?: Tags): void; + increment(stat: string | string[], value: number, sampleRate?: number, tags?: Tags, callback?: StatsCb): void; + increment(stat: string | string[], value: number, tags?: Tags, callback?: StatsCb): void; + increment(stat: string | string[], value: number, callback?: StatsCb): void; + increment(stat: string | string[], value: number, sampleRate?: number, callback?: StatsCb): void; + + decrement(stat: string): void; + decrement(stat: string | string[], value: number, sampleRate?: number, tags?: Tags, callback?: StatsCb): void; + decrement(stat: string | string[], value: number, tags?: Tags, callback?: StatsCb): void; + decrement(stat: string | string[], value: number, callback?: StatsCb): void; + decrement(stat: string | string[], value: number, sampleRate?: number, callback?: StatsCb): void; + + timing(stat: string | string[], value: number | Date, sampleRate?: number, tags?: Tags, callback?: StatsCb): void; + timing(stat: string | string[], value: number | Date, tags?: Tags, callback?: StatsCb): void; + timing(stat: string | string[], value: number | Date, callback?: StatsCb): void; + timing(stat: string | string[], value: number | Date, sampleRate?: number, callback?: StatsCb): void; + + timer

(func: (...args: P) => R, stat: string | string[], sampleRate?: number, tags?: Tags, callback?: StatsCb): (...args: P) => R; + timer

(func: (...args: P) => R, stat: string | string[], tags?: Tags, callback?: StatsCb): (...args: P) => R; + timer

(func: (...args: P) => R, stat: string | string[], callback?: StatsCb): (...args: P) => R; + timer

(func: (...args: P) => R, stat: string | string[], sampleRate?: number, callback?: StatsCb): (...args: P) => R; + + asyncTimer

(func: (...args: P) => Promise, stat: string | string[], sampleRate?: number, tags?: Tags, callback?: StatsCb): (...args: P) => Promise; + asyncTimer

(func: (...args: P) => Promise, stat: string | string[], tags?: Tags, callback?: StatsCb): (...args: P) => Promise; + asyncTimer

(func: (...args: P) => Promise, stat: string | string[], callback?: StatsCb): (...args: P) => Promise; + asyncTimer

(func: (...args: P) => Promise, stat: string | string[], sampleRate?: number, callback?: StatsCb): (...args: P) => Promise; + + asyncDistTimer

(func: (...args: P) => Promise, stat: string | string[], sampleRate?: number, tags?: Tags, callback?: StatsCb): (...args: P) => Promise; + asyncDistTimer

(func: (...args: P) => Promise, stat: string | string[], tags?: Tags, callback?: StatsCb): (...args: P) => Promise; + asyncDistTimer

(func: (...args: P) => Promise, stat: string | string[], callback?: StatsCb): (...args: P) => Promise; + asyncDistTimer

(func: (...args: P) => Promise, stat: string | string[], sampleRate?: number, callback?: StatsCb): (...args: P) => Promise; + + histogram(stat: string | string[], value: number, sampleRate?: number, tags?: Tags, callback?: StatsCb): void; + histogram(stat: string | string[], value: number, tags?: Tags, callback?: StatsCb): void; + histogram(stat: string | string[], value: number, callback?: StatsCb): void; + histogram(stat: string | string[], value: number, sampleRate?: number, callback?: StatsCb): void; + + distribution(stat: string | string[], value: number, sampleRate?: number, tags?: Tags, callback?: StatsCb): void; + distribution(stat: string | string[], value: number, tags?: Tags, callback?: StatsCb): void; + distribution(stat: string | string[], value: number, callback?: StatsCb): void; + distribution(stat: string | string[], value: number, sampleRate?: number, callback?: StatsCb): void; + + gauge(stat: string | string[], value: number, sampleRate?: number, tags?: Tags, callback?: StatsCb): void; + gauge(stat: string | string[], value: number, tags?: Tags, callback?: StatsCb): void; + gauge(stat: string | string[], value: number, callback?: StatsCb): void; + gauge(stat: string | string[], value: number, sampleRate?: number, callback?: StatsCb): void; + + set(stat: string | string[], value: number | string, sampleRate?: number, tags?: Tags, callback?: StatsCb): void; + set(stat: string | string[], value: number | string, tags?: Tags, callback?: StatsCb): void; + set(stat: string | string[], value: number | string, callback?: StatsCb): void; + set(stat: string | string[], value: number | string, sampleRate?: number, callback?: StatsCb): void; + + unique(stat: string | string[], value: number | string, sampleRate?: number, tags?: Tags, callback?: StatsCb): void; + unique(stat: string | string[], value: number | string, tags?: Tags, callback?: StatsCb): void; + unique(stat: string | string[], value: number | string, callback?: StatsCb): void; + unique(stat: string | string[], value: number | string, sampleRate?: number, callback?: StatsCb): void; + + close(callback?: (error?: Error) => void): void; + + event(title: string, text?: string, options?: EventOptions, tags?: Tags, callback?: StatsCb): void; + event(title: string, text?: string, options?: EventOptions, callback?: StatsCb): void; + check(name: string, status: DatadogChecksValues, options?: CheckOptions, tags?: Tags, callback?: StatsCb): void; + + public CHECKS: DatadogChecks; + public mockBuffer?: string[]; + public socket: dgram.Socket; + } +} + +declare const StatsDClient: new (options?: ClientOptions) => StatsD; +export default StatsDClient; diff --git a/tests/harness/self-hosting-native-toolchain.test.ts b/tests/harness/self-hosting-native-toolchain.test.ts index 83cc0d2fd..1f1e885ea 100644 --- a/tests/harness/self-hosting-native-toolchain.test.ts +++ b/tests/harness/self-hosting-native-toolchain.test.ts @@ -39,6 +39,7 @@ const features: NativeLinkFeatures = { http2: false, dgram: false, dnsPromises: false, + dgramDyn: false, watch: false, foreignFfi: false, nodeTest: false, diff --git a/tests/harness/self-hosting-runtime-pack.test.ts b/tests/harness/self-hosting-runtime-pack.test.ts index 2b9e51969..284c0cc46 100644 --- a/tests/harness/self-hosting-runtime-pack.test.ts +++ b/tests/harness/self-hosting-runtime-pack.test.ts @@ -40,6 +40,7 @@ const base: NativeLinkFeatures = { http2: false, dgram: false, dnsPromises: false, + dgramDyn: false, watch: false, foreignFfi: false, nodeTest: false, From 25a11d9864aefcab48bc06726e8b290dcb09d5b2 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 20 Oct 2026 18:08:33 -0500 Subject: [PATCH 20/38] Check that the limitations partials keep one sentence per line The Limitations page partials keep one sentence per line so edits to different sentences merge, but AGENTS.md's general Markdown rule asks for paragraphs on one line, and changes that followed it conflicted with the split text. AGENTS.md now names the partials as the exception, and limitations-partials.mjs gains a check, run by pnpm lint, that fails on a partial line holding two sentences, and a fix mode that splits such lines. The one partial that held several sentences per line is split; Markdown joins the lines, so the page renders the same. --- AGENTS.md | 4 +- .../limitations/unix-domain-sockets.mdx | 9 ++- docs/scripts/limitations-partials.mjs | 56 ++++++++++++++++++- package.json | 2 +- 4 files changed, 64 insertions(+), 7 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index ba32fe688..a7741b503 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,7 +4,7 @@ This is the only agent guide for the repository. Its rules apply everywhere, inc ## Markdown style -Write prose paragraphs and list items as single source lines and let editors and renderers wrap them naturally. Do not hard-wrap Markdown at a fixed column; preserve separate lines only where Markdown syntax or intentional formatting requires them, such as headings, lists, tables, blockquotes, and code blocks. +Write prose paragraphs and list items as single source lines and let editors and renderers wrap them naturally. Do not hard-wrap Markdown at a fixed column; preserve separate lines only where Markdown syntax or intentional formatting requires them, such as headings, lists, tables, blockquotes, and code blocks. The Limitations page partials under `docs/content/limitations/` are the exception: they keep one sentence per line (see Limitations page partials). ## Build and test @@ -196,7 +196,7 @@ The docs site is a standalone pnpm workspace under `docs/`. It uses Next.js App - A labeled bullet (`- **Crypto:** ...`) is `docs/content/limitations/

/