From 65a68470cf0e5dba6e51e473faebeaf9f0f2e7b0 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 11:12:23 -0500 Subject: [PATCH 01/53] Free a typed teardown's blocks in the order it met them A typed teardown freed each block onto its free list as it went, and the free lists, which are LIFO, handed a torn-down structure's blocks to the next allocations in the reverse order. A structure that a loop rebuilds the same way each time was therefore laid out mirrored every other generation, and every walk over it, its teardown included, ran through memory backwards, which the hardware prefetchers handle three to four times worse. A program that rebuilds a tree of 2^21 - 1 object literals 12 times, summing each generation before dropping it, took 26 to 28 ns per node to build, 24 to sum and 16 to drop in those generations, against 6.8, 6.3 and 5.3 in the others. With this change, every generation runs at the second set of times, and the program takes 0.50 s instead of 1.18 s (Node: 0.91 s). The teardown of a class or record type with children of its own type now keeps the blocks it frees in a segment, linked through their first words and built in order through a tail pointer. A node releases its own block first, places it ahead of its children's when its address is the lower (object literals allocate before their fields) and behind them otherwise (constructor arguments build the children first), and the chain's first teardown frees the whole segment onto the free list in one step. A teardown past DIRECT_DESTROY_CHAIN appends through scr_rc_destroy_seg, which keeps scr_rc_destroy's depth budget and queue. Blocks outside the calling thread's arena are freed as before, and targets without the inline allocation paths keep the old teardown. --- packages/compiler/src/backend/llvm/alloc.ts | 77 +++++- .../compiler/src/backend/llvm/classes.test.ts | 59 +++++ packages/compiler/src/backend/llvm/classes.ts | 73 ++++-- packages/compiler/src/backend/llvm/shapes.ts | 242 ++++++++++++++++-- .../corpus/7260-teardown-segments.ts.json | 6 + packages/runtime/src/scr_cycle.c | 56 ++-- packages/runtime/src/scr_rc.h | 8 + scripts/ci-test-costs.json | 2 + tests/corpus/7260-teardown-segments.ts | 179 +++++++++++++ .../7260-teardown-segments.ts.json | 1 + tests/harness/teardown-segments.test.ts | 97 +++++++ 11 files changed, 732 insertions(+), 68 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7260-teardown-segments.ts.json create mode 100644 tests/corpus/7260-teardown-segments.ts create mode 100644 tests/harness/corpus-costs/7260-teardown-segments.ts.json create mode 100644 tests/harness/teardown-segments.test.ts diff --git a/packages/compiler/src/backend/llvm/alloc.ts b/packages/compiler/src/backend/llvm/alloc.ts index aa6345dab..2efeb96ac 100644 --- a/packages/compiler/src/backend/llvm/alloc.ts +++ b/packages/compiler/src/backend/llvm/alloc.ts @@ -459,18 +459,72 @@ export type FreeObservers = "weak" | "weak-or-source"; * type's disposals), and the block — `scr_cyc_free` for collector-headered * objects, `scr_rt_free` otherwise. Labels use the `sf.` prefix; the lines * end in an open block. + * + * With `keep` (an alloca of ptr holding NULL; inline allocation paths only), + * a block that would go onto the calling thread's free list is stored there + * instead, for a teardown segment (emitSegmentFree); a block the system or + * another arena owns is still freed. */ export function emitObjectFree( host: AllocHost, cyc: boolean, observers: FreeObservers = "weak", + keep: string | null = null, ): string[] { - const lines = threadLocalState(host, "sf", freeLines(host, cyc, observers)); + const lines = threadLocalState(host, "sf", freeLines(host, cyc, observers, keep)); if (host.objectAudit === false) return lines.filter((line) => line !== FREE_NOTE); host.declare(`declare void @scr_obj_free_note()`); return lines; } +/** Lines that put the blocks `first` to `last` (linked through their first + * words; one block when they are equal) of the class at reservation offset + * `off` on the calling thread's free list. */ +function freeListPushLines(prefix: string, off: string, first: string, last: string): string[] { + return [ + ` %${prefix}.shp = getelementptr inbounds i8, ptr @scr_sa, i64 ${SA_SHIFT}`, + ` %${prefix}.sh32 = load i32, ptr %${prefix}.shp`, + ` %${prefix}.sh = zext i32 %${prefix}.sh32 to i64`, + ` %${prefix}.c = lshr i64 ${off}, %${prefix}.sh`, + ` %${prefix}.c8 = shl i64 %${prefix}.c, 3`, + ` %${prefix}.fo = add i64 %${prefix}.c8, ${SA_FREE}`, + ` %${prefix}.fp = getelementptr inbounds i8, ptr @scr_sa, i64 %${prefix}.fo`, + ` %${prefix}.head = load ptr, ptr %${prefix}.fp`, + ` store ptr %${prefix}.head, ptr ${last}`, + ` store ptr ${first}, ptr %${prefix}.fp`, + ]; +} + +/** + * Lines that free a teardown segment: the blocks `first` to `last` (ptr + * values, both NULL for an empty segment), linked through their first words, + * which emitObjectFree kept. Every block of a segment is in the calling + * thread's arena and of one size class, so the segment goes onto that + * class's free list in one step, and the next allocations of the class take + * its blocks in segment order. Labels use `prefix`; the lines end in the + * open block `${prefix}.done`. + */ +export function emitSegmentFree( + host: AllocHost, + prefix: string, + first: string, + last: string, +): string[] { + declareState(host); + const lines = [ + ` %${prefix}.none = icmp eq ptr ${first}, null`, + ` br i1 %${prefix}.none, label %${prefix}.done, label %${prefix}.push`, + `${prefix}.push:`, + ` %${prefix}.base = load i64, ptr @scr_sa`, + ` %${prefix}.bi = ptrtoint ptr ${first} to i64`, + ` %${prefix}.off = sub i64 %${prefix}.bi, %${prefix}.base`, + ...freeListPushLines(prefix, `%${prefix}.off`, first, last), + ` br label %${prefix}.done`, + `${prefix}.done:`, + ]; + return threadLocalState(host, prefix, lines); +} + /** Lines that leave in `%.obs` whether the program can observe a * disposal of this type (folds to a constant). */ function observedLines(prefix: string, observers: FreeObservers): string[] { @@ -482,8 +536,14 @@ function observedLines(prefix: string, observers: FreeObservers): string[] { ]; } -function freeLines(host: AllocHost, cyc: boolean, observers: FreeObservers): string[] { +function freeLines( + host: AllocHost, + cyc: boolean, + observers: FreeObservers, + keep: string | null, +): string[] { if (host.inlineAlloc !== true) { + if (keep !== null) throw new Error("a kept block needs the inline allocation paths"); if (cyc) { host.declare(`declare void @scr_cyc_free(ptr)`); return [` call void @scr_obj_free_note()`, ` call void @scr_cyc_free(ptr %o)`]; @@ -548,16 +608,9 @@ function freeLines(host: AllocHost, cyc: boolean, observers: FreeObservers): str ` %sf.inx = call i1 @llvm.expect.i1(i1 %sf.in, i1 true)`, ` br i1 %sf.inx, label %sf.push, label %sf.sys`, `sf.push:`, - ` %sf.shp = getelementptr inbounds i8, ptr @scr_sa, i64 ${SA_SHIFT}`, - ` %sf.sh32 = load i32, ptr %sf.shp`, - ` %sf.sh = zext i32 %sf.sh32 to i64`, - ` %sf.c = lshr i64 %sf.off, %sf.sh`, - ` %sf.c8 = shl i64 %sf.c, 3`, - ` %sf.fo = add i64 %sf.c8, ${SA_FREE}`, - ` %sf.fp = getelementptr inbounds i8, ptr @scr_sa, i64 %sf.fo`, - ` %sf.head = load ptr, ptr %sf.fp`, - ` store ptr %sf.head, ptr ${blk}`, - ` store ptr ${blk}, ptr %sf.fp`, + ...(keep !== null + ? [` store ptr ${blk}, ptr ${keep}`] + : freeListPushLines("sf", "%sf.off", blk, blk)), ` br label %sf.done`, `sf.sys: ; outside the reservation, or the allocator is compiled out`, ` call void @scr_obj_free_note()`, diff --git a/packages/compiler/src/backend/llvm/classes.test.ts b/packages/compiler/src/backend/llvm/classes.test.ts index 2d95e942c..0c0b73aeb 100644 --- a/packages/compiler/src/backend/llvm/classes.test.ts +++ b/packages/compiler/src/backend/llvm/classes.test.ts @@ -68,6 +68,65 @@ test("leaf layouts keep direct teardown while recursive layouts share the depth } }); +test("teardowns of types with children of their own type free their blocks as one segment", () => { + const linked = { kind: "object" as const, className: "Linked" }; + const mod: IrModule = { + irVersion: 17, + sourceFile: loc.file, + entry: "main", + functions: [{ name: "main", params: [], returnType: VOID, locals: [], body: [], loc }], + classes: [ + { + name: "Linked", + fields: [ + { name: "next", type: linked }, + { name: "other", type: linked }, + ], + methods: [], + loc, + }, + ], + records: [{ id: "branch", fields: [{ name: "next", type: linked }] }], + }; + const define = (llvm: string, name: string): string => + new RegExp(`^define internal [^@]*@${name}\\([^]*?^}`, "m").exec(llvm)?.[0] ?? ""; + const linkedDestroy = `${mangleClassRelease("Linked")}_destroy`; + const branchDestroy = `${mangleRecordRelease("branch")}_destroy`; + for (const workers of [false, true]) { + const llvm = emitLlvmModule({ ...mod, workers }, { pointerBits: 64 }); + // Linked's teardown keeps its block and its children's in a segment: it + // releases its own block first and places it by address against its + // first child, a child torn down directly appends to the segment, one + // past the direct chain appends through the runtime, and the chain's + // first teardown frees the segment onto the free list in one step. + const seg = define(llvm, `${linkedDestroy}_s`); + expect(seg.startsWith("define internal ptr"), "segment form").toBe(true); + expect(seg).toMatch( + /store ptr %(?:sf\.blk|o), ptr %sg\.own[^]*call ptr @sc_release_Linked_destroy_s\(/, + ); + // The children's releases have a copy for each place of the own block. + for (const copy of ["ta", "tb"]) + expect(seg).toContain( + `call ptr @scr_rc_destroy_seg(ptr %v0, ptr @${linkedDestroy}_sc, ptr @${linkedDestroy}, ptr %${copy}0.bt)`, + ); + expect(seg).not.toContain("call void @scr_rc_destroy("); + expect(seg).toContain("icmp ult ptr %sg.o, %sg.r1"); + const first = define(llvm, `${linkedDestroy}_d`); + expect(first).toContain(`call ptr @${linkedDestroy}_s(ptr %o, i32 %d, ptr %head)`); + expect(first).toContain("store ptr %first, ptr %sgf.fp"); + expect(first.includes("llvm.threadlocal.address"), "thread-local state").toBe(workers); + expect(define(llvm, linkedDestroy)).toContain(`call void @${linkedDestroy}_d(ptr %o, i32 0)`); + // A record whose child is another type frees its own block as before. + expect(define(llvm, `${branchDestroy}_s`)).toBe(""); + expect(define(llvm, branchDestroy)).toContain(`call void @${mangleClassRelease("Linked")}(`); + } + // Without the inline allocation paths (32-bit targets) a teardown frees + // through the runtime, block by block. + const llvm32 = emitLlvmModule(mod, { pointerBits: 32 }); + expect(define(llvm32, `${linkedDestroy}_s`)).toBe(""); + expect(define(llvm32, `${linkedDestroy}_d`)).toContain(`call void @${linkedDestroy}_d(`); +}); + test("releases split into an inlinable fast path and an out-of-line remainder", () => { // An array of the class itself makes the hierarchy cycle-capable. const kids = { kind: "array" as const, elem: { kind: "object" as const, className: "Node" } }; diff --git a/packages/compiler/src/backend/llvm/classes.ts b/packages/compiler/src/backend/llvm/classes.ts index 3073bd1bd..3d67fbfd3 100644 --- a/packages/compiler/src/backend/llvm/classes.ts +++ b/packages/compiler/src/backend/llvm/classes.ts @@ -43,6 +43,9 @@ import { destroyDefinitions, directChildReleaseLines, directDestroyTarget, + segmentPrologueLines, + segmentTeardownLines, + segmentedTeardown, usedDirectTargets, type DirectDestroy, type RegionHooks, @@ -1078,25 +1081,62 @@ export function emitClassShapes( bounded && (directTargets.has(`object:${cls.name}`) || refFields.some((f) => directDestroyTarget(host, f.type, directTargets) !== null)); + // The teardown's own symbol, and whether it keeps a segment (shapes.ts + // segmentedTeardown). + const destroy = `${meta.hierarchy ? mangleClassReleaseDirect(cls.name) : mangleClassRelease(cls.name)}_destroy`; + const segmented = + chained && + !meta.compact && + runtimePrefix === null && + !isStreamRooted && + !meta.threadSlot && + segmentedTeardown( + host, + destroy, + refFields.map((f) => f.type), + directTargets, + ); // The field-releasing teardown body shared by both release shapes // (the public one on standalone classes, the DIRECT one on hierarchy // members). Runs at rc == 0. const teardown = (lines: string[]): void => { - refFields.forEach((f, t) => { - lines.push( - ` %f${t} = getelementptr inbounds %${struct}, ptr %o, i64 0, i32 ${f.index}`, - ` %v${t} = load ptr, ptr %f${t}`, - ); + const release = (prefix: string, f: (typeof refFields)[number], t: number): string[] => { const direct = bounded ? directDestroyTarget(host, f.type, directTargets) : null; - if (direct) - lines.push( - ...directChildReleaseLines(host, `td${t}`, `%v${t}`, direct, llvmCommentText(f.name)), - ); - else - lines.push( + if (direct === null) + return [ ` call void ${releaseSym(host, f.type)}(ptr %v${t}) ; ${llvmCommentText(f.name)}`, - ); - }); + ]; + const own = segmented && direct.destroy === destroy; + return directChildReleaseLines( + host, + `${prefix}${t}`, + `%v${t}`, + direct, + llvmCommentText(f.name), + own, + ); + }; + const loads = refFields.map((f, t) => [ + ` %f${t} = getelementptr inbounds %${struct}, ptr %o, i64 0, i32 ${f.index}`, + ` %v${t} = load ptr, ptr %f${t}`, + ]); + if (segmented) { + // A segmented teardown loads its fields, releases its own block, and + // then releases the fields (shapes.ts segmentTeardownLines). + const ownType = refFields.flatMap((f, t) => + directDestroyTarget(host, f.type, directTargets)?.destroy === destroy ? [`%v${t}`] : [], + ); + lines.push( + ...segmentPrologueLines(), + ...loads.flat(), + ...emitObjectFree(host, traced, "weak", "%sg.own"), + ...segmentTeardownLines(ownType, (prefix) => + refFields.flatMap((f, t) => release(prefix, f, t)), + ), + ); + return; + } + refFields.forEach((f, t) => lines.push(...loads[t]!, ...release("td", f, t))); if (runtimePrefix) lines.push(...regCall("td", runtimePrefix.drop, "")); if (isStreamRooted) lines.push(...stCall("tds", "scr_stream_st_release", "")); if (meta.threadSlot) lines.push(...slotForgetLines(host, struct, meta, "tdw")); @@ -1178,10 +1218,12 @@ export function emitClassShapes( ]; if (traced) reld.push(...cycleDeadLines(host, "teardown")); if (bounded) { - const destroy = `${mangleClassReleaseDirect(cls.name)}_destroy`; const destroyBody: string[] = []; teardown(destroyBody); - defs.push(...destroyDefinitions(destroy, destroyBody, chained), ``); + defs.push( + ...destroyDefinitions(destroy, destroyBody, chained, segmented ? host : null), + ``, + ); host.declare(`declare void @scr_rc_destroy(ptr, ptr)`); reld.push(` call void @scr_rc_destroy(ptr %o, ptr @${destroy})`); } else teardown(reld); @@ -1203,6 +1245,7 @@ export function emitClassShapes( bounded, lifetime, chained, + segmented, ), ``, ); diff --git a/packages/compiler/src/backend/llvm/shapes.ts b/packages/compiler/src/backend/llvm/shapes.ts index 6de92f667..ae217a76c 100644 --- a/packages/compiler/src/backend/llvm/shapes.ts +++ b/packages/compiler/src/backend/llvm/shapes.ts @@ -34,6 +34,7 @@ import { cycTypeDescriptor, emitObjectAlloc, emitObjectFree, + emitSegmentFree, CYC_APPEND, cycleAppendDecls, cycleAppendHelper, @@ -1340,17 +1341,23 @@ export interface DirectDestroy { /** The release of child `value` (a loaded field, possibly NULL) inside a * teardown whose chain length is `%d`: the child's own release for a shared * or immortal reference, its teardown for the last one. Labels use `tag`; - * the lines end in the open block `${tag}.next`. */ + * the lines end in the open block `${tag}.next`. + * + * With `segment` (the child has the teardown's own type, in a segmented + * teardown; segmentPrologueLines), the child's teardown appends its blocks + * to the teardown's segment. */ export function directChildReleaseLines( host: ShapeHost, tag: string, value: string, target: DirectDestroy, comment: string, + segment = false, ): string[] { const S = host.sizeType; const C = target.count ?? S; - host.declare(`declare void @scr_rc_destroy(ptr, ptr)`); + if (segment) host.declare(`declare ptr @scr_rc_destroy_seg(ptr, ptr, ptr, ptr)`); + else host.declare(`declare void @scr_rc_destroy(ptr, ptr)`); const lines = [ ` %${tag}.nn = icmp ne ptr ${value}, null`, ` br i1 %${tag}.nn, label %${tag}.chk, label %${tag}.next`, @@ -1382,16 +1389,139 @@ export function directChildReleaseLines( ` br i1 %${tag}.short, label %${tag}.direct, label %${tag}.bounded`, `${tag}.direct:`, ` %${tag}.d = add i32 %d, 1`, - ` call void @${target.destroy}_d(ptr ${value}, i32 %${tag}.d)`, + ); + if (!segment) { + lines.push( + ` call void @${target.destroy}_d(ptr ${value}, i32 %${tag}.d)`, + ` br label %${tag}.next`, + `${tag}.bounded:`, + ` call void @scr_rc_destroy(ptr ${value}, ptr @${target.destroy})`, + ` br label %${tag}.next`, + `${tag}.next:`, + ); + return lines; + } + lines.push( + ` %${tag}.t = load ptr, ptr %sg.t`, + ` %${tag}.dt = call ptr @${target.destroy}_s(ptr ${value}, i32 %${tag}.d, ptr %${tag}.t)`, + ` store ptr %${tag}.dt, ptr %sg.t`, ` br label %${tag}.next`, `${tag}.bounded:`, - ` call void @scr_rc_destroy(ptr ${value}, ptr @${target.destroy})`, + ` %${tag}.bt = load ptr, ptr %sg.t`, + ` %${tag}.rt = call ptr @scr_rc_destroy_seg(ptr ${value}, ptr @${target.destroy}_sc, ptr @${target.destroy}, ptr %${tag}.bt)`, + ` store ptr %${tag}.rt, ptr %sg.t`, ` br label %${tag}.next`, `${tag}.next:`, ); return lines; } +/* Teardown segments. A typed teardown that frees each block onto its free + * list as it goes frees a tree in walk order, and the free lists, which are + * LIFO, hand the blocks to the next allocations in the reverse order: a tree + * built again in the same order as the last (each generation of a structure + * a loop rebuilds) is laid out mirrored, and every walk over it, the next + * teardown included, runs through memory backwards, which defeats the + * hardware prefetchers. The runtime frees large dead sets of the cycle + * collector in address order for the same reason (scr_mem_order_for_free). + * + * So the teardown of a type with children of its own type (the `_s` form, + * segmentedTeardown) keeps the blocks it frees in a segment, linked through + * their first words, which it builds in order through a tail pointer: the + * `_s` form takes the segment's last link (the first word of its last block, + * or the head slot of an empty segment) and returns the new one. A node + * first releases its own block (emitObjectFree, keeping it), then appends it + * ahead of its children's blocks when its address is the lower, behind them + * otherwise (segmentTeardownLines), and the children append theirs in field + * order, so a structure + * built parent first (object literals allocate before their fields) or + * children first (constructor arguments) comes back in ascending order. The + * chain's first teardown (`_d`) frees the whole segment onto the class's + * free list in one step (emitSegmentFree), and the walks over the next + * generation run forward. A teardown past DIRECT_DESTROY_CHAIN appends + * through scr_rc_destroy_seg, under the same depth budget as + * scr_rc_destroy. Blocks outside the calling thread's arena are freed as + * before; one type's blocks are all of one class. */ + +/** Whether the teardown of a type whose reference fields have `fieldTypes` + * (with direct targets `targets`) and whose own teardown is `destroy` keeps + * a segment: a child field has its own type, and the inline allocation paths + * are in use. */ +export function segmentedTeardown( + host: ShapeHost, + destroy: string, + fieldTypes: readonly IrType[], + targets: ReadonlyMap, +): boolean { + if (host.inlineAlloc !== true) return false; + return fieldTypes.some((t) => directDestroyTarget(host, t, targets)?.destroy === destroy); +} + +/** The first lines of a segmented teardown, in its entry block: the + * segment's last link (`%sg.t`, from the `%tail` argument) and the kept own + * block (`%sg.own`, for emitObjectFree). */ +export function segmentPrologueLines(): string[] { + return [ + ` %sg.t = alloca ptr`, + ` %sg.own = alloca ptr`, + ` store ptr %tail, ptr %sg.t`, + ` store ptr null, ptr %sg.own`, + ]; +} + +/** The rest of a segmented teardown after emitObjectFree kept (or freed) + * its own block: the block goes ahead of its children's when its address is + * below the first non-NULL one of `children` (loaded fields of the + * teardown's own type, which stand for their blocks) or there is none, and + * behind them otherwise; then the children's releases (`releases`, given a + * label prefix), and the segment's last link is returned. Each order has its + * own copy of the releases, so neither keeps the decision, or the block once + * it is appended, across the children's teardowns. */ +export function segmentTeardownLines( + children: readonly string[], + releases: (prefix: string) => string[], +): string[] { + const lines = [` %sg.o = load ptr, ptr %sg.own`, ` %sg.kept = icmp ne ptr %sg.o, null`]; + let ref = "null"; + [...children].reverse().forEach((child, i) => { + lines.push( + ` %sg.nn${i} = icmp ne ptr ${child}, null`, + ` %sg.r${i} = select i1 %sg.nn${i}, ptr ${child}, ptr ${ref}`, + ); + ref = `%sg.r${i}`; + }); + lines.push( + ` %sg.leaf = icmp eq ptr ${ref}, null`, + ` %sg.lower = icmp ult ptr %sg.o, ${ref}`, + ` %sg.first = or i1 %sg.leaf, %sg.lower`, + ` %sg.ahead = and i1 %sg.kept, %sg.first`, + ` br i1 %sg.ahead, label %sg.front, label %sg.back`, + `sg.front:`, + ...segmentAppendLines("a"), + ...releases("ta"), + ` br label %sg.ret`, + `sg.back:`, + ...releases("tb"), + ` %sg.keep = icmp ne ptr %sg.o, null`, + ` br i1 %sg.keep, label %sg.last, label %sg.ret`, + `sg.last:`, + ...segmentAppendLines("l"), + ` br label %sg.ret`, + `sg.ret:`, + ` %sg.tr = load ptr, ptr %sg.t`, + ` ret ptr %sg.tr`, + ); + return lines; +} + +function segmentAppendLines(tag: string): string[] { + return [ + ` %sg.t${tag} = load ptr, ptr %sg.t`, + ` store ptr %sg.o, ptr %sg.t${tag}`, + ` store ptr %sg.o, ptr %sg.t`, + ]; +} + /** The direct-teardown target of a reference of `type`, if `targets` (keyed * `record:` / `object:`) has one. Nullable unions hold the arm's * pointer, NULL or an immortal sentinel, which the child release handles. */ @@ -1434,12 +1564,42 @@ export function usedDirectTargets( /** The teardown definitions for `destroy`: with `chained`, the body takes * the chain length `%d` (directChildReleaseLines) and the runtime's - * one-argument entry starts a chain. */ + * one-argument entry starts a chain. With `segment` (the host, for a + * segmented teardown), the body is a segment teardown's + * (segmentPrologueLines to segmentTeardownLines), defined as `_s`, with + * `_sc` for scr_rc_destroy_seg, and `_d` starts a segment and frees it. */ export function destroyDefinitions( destroy: string, body: readonly string[], chained: boolean, + segment: ShapeHost | null = null, ): string[] { + if (segment !== null) + return [ + `define internal ptr @${destroy}_s(ptr %o, i32 %d, ptr %tail) ${FN_ATTRS} {`, + `entry:`, + ...body, + `}`, + `define internal ptr @${destroy}_sc(ptr %o, ptr %tail) ${FN_ATTRS} {`, + `entry:`, + ` %t = call ptr @${destroy}_s(ptr %o, i32 0, ptr %tail)`, + ` ret ptr %t`, + `}`, + `define internal void @${destroy}_d(ptr %o, i32 %d) ${FN_ATTRS} {`, + `entry:`, + ` %head = alloca ptr`, + ` store ptr null, ptr %head`, + ` %last = call ptr @${destroy}_s(ptr %o, i32 %d, ptr %head)`, + ` %first = load ptr, ptr %head`, + ...emitSegmentFree(segment, "sgf", "%first", "%last"), + ` ret void`, + `}`, + `define internal void @${destroy}(ptr %o) ${FN_ATTRS} {`, + `entry:`, + ` call void @${destroy}_d(ptr %o, i32 0)`, + ` ret void`, + `}`, + ]; if (!chained) return [ `define internal void @${destroy}(ptr %o) ${FN_ATTRS} {`, @@ -1478,12 +1638,13 @@ export function releaseBody( bounded = true, immortal = false, chained = false, + segmented = false, ): string[] { const S = host.sizeType; if (bounded) host.declare(`declare void @scr_rc_destroy(ptr, ptr)`); const destroy = `${fnName}_destroy`; const lines = [ - ...(bounded ? destroyDefinitions(destroy, freeBody, chained) : []), + ...(bounded ? destroyDefinitions(destroy, freeBody, chained, segmented ? host : null) : []), ...releaseFastPath(host, fnName, `${fnName}_slow`, traced, comment, immortal), `define internal void @${fnName}_slow(ptr %o) noinline ${FN_ATTRS} {${comment ? ` ; ${comment}` : ""}`, `entry:`, @@ -1715,26 +1876,58 @@ export function emitRecordShapes( const chained = directTargets.has(`record:${shape.id}`) || refMembers.some((m) => directDestroyTarget(host, m.type, directTargets) !== null); - for (const m of refMembers) { - freeBody.push( - ` %f${t} = getelementptr inbounds %${struct}, ptr %o, i64 0, i32 ${m.index}`, - ` %v${t} = load ptr, ptr %f${t}`, + const destroy = `${mangleRecordRelease(shape.id)}_destroy`; + const bounded = refMembers.some((member) => needsBoundedRelease(member.type)); + const segmented = + bounded && + segmentedTeardown( + host, + destroy, + refMembers.map((m) => m.type), + directTargets, ); - const direct = directDestroyTarget(host, m.type, directTargets); - if (direct) { - freeBody.push( - ...directChildReleaseLines(host, `td${t}`, `%v${t}`, direct, llvmCommentText(m.name)), - ); - } else { - freeBody.push( - ` call void ${releaseSym(host, m.type)}(ptr %v${t}) ; ${llvmCommentText(m.name)}`, - ); - } - t++; - } // Closed layouts can carry a kept record source (record-sources.ts). const observers = recordMayKeepSource(shape) ? "weak-or-source" : "weak"; - freeBody.push(...emitObjectFree(host, traced, observers)); + // A segmented teardown loads its fields, releases its own block, and + // then releases the fields (segmentTeardownLines). + const loads: string[] = []; + const ownType: string[] = []; + const release = (prefix: string, m: (typeof refMembers)[number], i: number): string[] => { + const direct = directDestroyTarget(host, m.type, directTargets); + if (direct === null) + return [` call void ${releaseSym(host, m.type)}(ptr %v${i}) ; ${llvmCommentText(m.name)}`]; + const own = segmented && direct.destroy === destroy; + return directChildReleaseLines( + host, + `${prefix}${i}`, + `%v${i}`, + direct, + llvmCommentText(m.name), + own, + ); + }; + for (const m of refMembers) { + const fieldLines = [ + ` %f${t} = getelementptr inbounds %${struct}, ptr %o, i64 0, i32 ${m.index}`, + ` %v${t} = load ptr, ptr %f${t}`, + ]; + if (segmented) { + loads.push(...fieldLines); + if (directDestroyTarget(host, m.type, directTargets)?.destroy === destroy) + ownType.push(`%v${t}`); + } else freeBody.push(...fieldLines, ...release("td", m, t)); + t++; + } + if (segmented) + freeBody.push( + ...segmentPrologueLines(), + ...loads, + ...emitObjectFree(host, traced, observers, "%sg.own"), + ...segmentTeardownLines(ownType, (prefix) => + refMembers.flatMap((m, i) => release(prefix, m, i)), + ), + ); + else freeBody.push(...emitObjectFree(host, traced, observers)); defs.push( ...releaseBody( host, @@ -1742,9 +1935,10 @@ export function emitRecordShapes( traced, freeBody, "", - refMembers.some((member) => needsBoundedRelease(member.type)), + bounded, false, chained, + segmented, ), ``, ); diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7260-teardown-segments.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7260-teardown-segments.ts.json new file mode 100644 index 000000000..a7eac099f --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7260-teardown-segments.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7260-teardown-segments.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_cycle.c b/packages/runtime/src/scr_cycle.c index 5e5029e14..1bd0ff7ac 100644 --- a/packages/runtime/src/scr_cycle.c +++ b/packages/runtime/src/scr_cycle.c @@ -293,24 +293,22 @@ static SCR_TL ScrDestroyEntry *scr_destroy_pending; static SCR_TL size_t scr_destroy_count, scr_destroy_capacity; #endif -void scr_rc_destroy(void *obj, void (*destroy)(void *)) { - if (scr_destroy_depth == SCR_DESTROY_DEPTH) { - if (scr_destroy_count == scr_destroy_capacity) { - size_t capacity = scr_destroy_capacity ? scr_destroy_capacity * 2 : 64; - if (capacity < scr_destroy_capacity || capacity > SIZE_MAX / sizeof(ScrDestroyEntry)) - scr_cyc_oom(); - ScrDestroyEntry *entries = realloc(scr_destroy_pending, capacity * sizeof(*entries)); - if (!entries) scr_cyc_oom(); - scr_destroy_pending = entries; - scr_destroy_capacity = capacity; - } - scr_destroy_pending[scr_destroy_count++] = (ScrDestroyEntry){ obj, destroy }; - return; +/* Queues a disposal past the depth budget; the outermost disposal runs it. */ +static void scr_rc_destroy_defer(void *obj, void (*destroy)(void *)) { + if (scr_destroy_count == scr_destroy_capacity) { + size_t capacity = scr_destroy_capacity ? scr_destroy_capacity * 2 : 64; + if (capacity < scr_destroy_capacity || capacity > SIZE_MAX / sizeof(ScrDestroyEntry)) + scr_cyc_oom(); + ScrDestroyEntry *entries = realloc(scr_destroy_pending, capacity * sizeof(*entries)); + if (!entries) scr_cyc_oom(); + scr_destroy_pending = entries; + scr_destroy_capacity = capacity; } - scr_destroy_depth++; - destroy(obj); - scr_destroy_depth--; - if (scr_destroy_depth) return; + scr_destroy_pending[scr_destroy_count++] = (ScrDestroyEntry){ obj, destroy }; +} + +/* Runs the queued disposals once the outermost one has returned. */ +static void scr_rc_destroy_drain(void) { while (scr_destroy_count) { ScrDestroyEntry entry = scr_destroy_pending[--scr_destroy_count]; scr_destroy_depth = 1; @@ -326,6 +324,30 @@ void scr_rc_destroy(void *obj, void (*destroy)(void *)) { } } +void scr_rc_destroy(void *obj, void (*destroy)(void *)) { + if (scr_destroy_depth == SCR_DESTROY_DEPTH) { + scr_rc_destroy_defer(obj, destroy); + return; + } + scr_destroy_depth++; + destroy(obj); + scr_destroy_depth--; + if (scr_destroy_depth == 0) scr_rc_destroy_drain(); +} + +void *scr_rc_destroy_seg(void *obj, void *(*destroy_seg)(void *, void *), + void (*destroy)(void *), void *tail) { + if (scr_destroy_depth == SCR_DESTROY_DEPTH) { + scr_rc_destroy_defer(obj, destroy); + return tail; + } + scr_destroy_depth++; + tail = destroy_seg(obj, tail); + scr_destroy_depth--; + if (scr_destroy_depth == 0) scr_rc_destroy_drain(); + return tail; +} + /* Live cycle-headered objects and the running count of OLD objects freed. * The inline allocation/free paths (scr_runtime.h, llvm/alloc.ts) maintain * both counters so full passes can measure old-generation reclamation. */ diff --git a/packages/runtime/src/scr_rc.h b/packages/runtime/src/scr_rc.h index 7a921b451..83d0589c1 100644 --- a/packages/runtime/src/scr_rc.h +++ b/packages/runtime/src/scr_rc.h @@ -552,6 +552,14 @@ _Static_assert(sizeof(((ScrCycHdr *)0)->gen) == 1 && SCR_CYC_OLD == 2, * depth; deeper objects wait until the outer disposal drains the worklist. * The callback owns the object and must release its children and free it. */ void scr_rc_destroy(void *obj, void (*destroy)(void *)); +/* scr_rc_destroy for an emitted teardown that keeps the blocks it frees in + * a segment, linked through their first words, instead of freeing them onto + * the free list (llvm/shapes.ts): destroy_seg tears obj down, appending its + * blocks after the segment's last link `tail`, and returns the new last + * link, which this returns. A disposal past the depth budget is queued with + * `destroy`, which frees its own blocks, and leaves `tail` as it was. */ +void *scr_rc_destroy_seg(void *obj, void *(*destroy_seg)(void *, void *), + void (*destroy)(void *), void *tail); /* RC hooks for cycle-headered types. on_release: rc was decremented and * stayed above zero — buffer the object as a possible cycle root (may run diff --git a/scripts/ci-test-costs.json b/scripts/ci-test-costs.json index 4c4964579..99d47e972 100644 --- a/scripts/ci-test-costs.json +++ b/scripts/ci-test-costs.json @@ -409,6 +409,7 @@ "tests/harness/stdlib-surface.test.ts": 48, "tests/harness/strip-vt-linear.test.ts": 1795, "tests/harness/surface-manifest.test.ts": 5309, + "tests/harness/teardown-segments.test.ts": 2600, "tests/harness/terminal-geometry.test.ts": 2730, "tests/harness/test262-runner.test.ts": 207, "tests/harness/three-library.test.ts": 251122, @@ -828,6 +829,7 @@ "tests/harness/stdlib-surface.test.ts": 46, "tests/harness/strip-vt-linear.test.ts": 2546, "tests/harness/surface-manifest.test.ts": 5951, + "tests/harness/teardown-segments.test.ts": 5, "tests/harness/terminal-geometry.test.ts": 3270, "tests/harness/test262-runner.test.ts": 230, "tests/harness/three-library.test.ts": 361470, diff --git a/tests/corpus/7260-teardown-segments.ts b/tests/corpus/7260-teardown-segments.ts new file mode 100644 index 000000000..cbd2b144d --- /dev/null +++ b/tests/corpus/7260-teardown-segments.ts @@ -0,0 +1,179 @@ +// Teardowns that hand their blocks back in order. +// +// A typed teardown of a type with children of its own type keeps the blocks +// it frees in one segment and frees the segment when the chain's first +// teardown returns, so the next structure built of that type reuses them in +// order. This program rebuilds structures of such types for many +// generations and reads every value of each generation before dropping it: +// a block handed out twice, or one freed while still in use, shows up as a +// wrong checksum. The shapes cover the ways a segment is put together: +// record literals (allocated before their fields) and constructors whose +// arguments build the children first, lists built by prepending and by +// appending, trees far deeper than one direct teardown chain and lists +// longer than the runtime's whole depth budget (whose tails wait in its +// queue), subtrees still owned elsewhere when their parent goes, records of +// one type holding records of another, and nodes of a cycle-capable class +// that are still candidates of the cycle collector when the last reference +// goes, and rings of them that only the collector frees. + +interface Node { + left: Node | null; + right: Node | null; + value: number; +} + +function literalTree(depth: number, value: number): Node { + return depth === 0 + ? { left: null, right: null, value } + : { left: literalTree(depth - 1, value * 2), right: literalTree(depth - 1, value * 2 + 1), value }; +} + +function sum(node: Node | null): number { + return node === null ? 0 : node.value + sum(node.left) + sum(node.right); +} + +class Branch { + readonly left: Branch | null; + readonly right: Branch | null; + readonly label: string; + constructor(left: Branch | null, right: Branch | null, label: string) { + this.left = left; + this.right = right; + this.label = label; + } +} + +function classTree(depth: number, label: string): Branch { + return depth === 0 + ? new Branch(null, null, label) + : new Branch(classTree(depth - 1, label + "l"), classTree(depth - 1, label + "r"), label); +} + +function labels(branch: Branch | null): number { + return branch === null ? 0 : branch.label.length + labels(branch.left) + labels(branch.right); +} + +interface Cell { + next: Cell | null; + n: number; +} + +function prepended(count: number, base: number): Cell | null { + let head: Cell | null = null; + for (let i = 0; i < count; i++) head = { next: head, n: base + i }; + return head; +} + +function appended(count: number, base: number): Cell | null { + if (count === 0) return null; + const head: Cell = { next: null, n: base }; + let tail = head; + for (let i = 1; i < count; i++) { + const cell: Cell = { next: null, n: base + i }; + tail.next = cell; + tail = cell; + } + return head; +} + +function cells(head: Cell | null): [number, number] { + let count = 0; + let total = 0; + for (let cell = head; cell !== null; cell = cell.next) { + count++; + total += cell.n; + } + return [count, total]; +} + +// Generations of literal trees: shallow, one chain deep, and several chains +// deep. Each generation is checked against the sum the shape must have. +for (const depth of [0, 1, 5, 16, 17, 18]) { + let ok = true; + for (let generation = 0; generation < 6; generation++) { + const tree = literalTree(depth, 1); + const expected = ((2 ** (depth + 1)) * (2 ** (depth + 1) - 1)) / 2; + if (sum(tree) !== expected) ok = false; + } + console.log(`literal trees of depth ${depth}: ${ok}`); +} + +// Generations of class trees built children first. +for (const depth of [3, 12, 17]) { + let total = 0; + for (let generation = 0; generation < 4; generation++) total += labels(classTree(depth, "t")); + console.log(`class trees of depth ${depth}: ${total}`); +} + +// Lists both ways, alternating, so each reuses the other's blocks; the long +// ones outlast the runtime's depth budget. +for (const count of [1, 15, 16, 17, 1000, 3000, 40000]) { + const seen: string[] = []; + for (let generation = 0; generation < 4; generation++) { + const list = generation % 2 === 0 ? prepended(count, generation) : appended(count, generation); + const [length, total] = cells(list); + seen.push(`${length}/${total}`); + } + console.log(`lists of ${count}: ${seen.join(" ")}`); +} + +// Subtrees owned elsewhere survive their parent's teardown, and the blocks +// freed around them are not handed out while they live. +const kept: Node[] = []; +for (let generation = 0; generation < 5; generation++) { + const tree = literalTree(10, generation + 1); + if (tree.left !== null && tree.left.right !== null) kept.push(tree.left.right); + if (tree.right !== null) kept.push(tree.right); + // Building the next tree reuses the dropped blocks. + literalTree(10, 7); +} +console.log(`kept subtrees: ${kept.length} ${kept.map(sum).join(",")}`); + +// A record of one type holding trees of another: the outer teardown frees +// its own blocks as before and tears each inner tree down as a segment. +interface Forest { + first: Node; + second: Node | null; + rest: Forest | null; +} + +function forest(count: number): Forest | null { + let out: Forest | null = null; + for (let i = 0; i < count; i++) out = { first: literalTree(4, i), second: i % 3 === 0 ? null : literalTree(2, i), rest: out }; + return out; +} + +let forestTotal = 0; +for (let generation = 0; generation < 3; generation++) { + for (let f = forest(500); f !== null; f = f.rest) forestTotal += sum(f.first) + sum(f.second); +} +console.log(`forests: ${forestTotal}`); + +// A cycle-capable class: releases of shared links make nodes candidates of +// the cycle collector, which are then torn down while still buffered, and +// the peer links close some rings, which the collector frees. +class Ring { + next: Ring | null = null; + peer: Ring | null = null; + id: number; + constructor(id: number) { + this.id = id; + } +} + +let ringTotal = 0; +for (let generation = 0; generation < 3; generation++) { + let head: Ring | null = null; + const peers: Ring[] = []; + for (let i = 0; i < 2000; i++) { + const ring: Ring = new Ring(i); + ring.next = head; + if (i % 7 === 0) peers.push(ring); + head = ring; + } + for (let r = head; r !== null; r = r.next) if (r.id % 5 === 0) r.peer = peers[r.id % peers.length] ?? null; + for (const peer of peers) ringTotal += peer.id; + peers.length = 0; + for (let r = head; r !== null; r = r.next) ringTotal += r.peer === null ? 0 : r.peer.id; +} +console.log(`rings: ${ringTotal}`); diff --git a/tests/harness/corpus-costs/7260-teardown-segments.ts.json b/tests/harness/corpus-costs/7260-teardown-segments.ts.json new file mode 100644 index 000000000..7f7ce1c2b --- /dev/null +++ b/tests/harness/corpus-costs/7260-teardown-segments.ts.json @@ -0,0 +1 @@ +{ "plain": 5343, "san": 11960 } diff --git a/tests/harness/teardown-segments.test.ts b/tests/harness/teardown-segments.test.ts new file mode 100644 index 000000000..95a2d2795 --- /dev/null +++ b/tests/harness/teardown-segments.test.ts @@ -0,0 +1,97 @@ +/* Teardown segments (llvm/shapes.ts): a typed teardown of a type with + * children of its own type keeps the blocks it frees in a segment and frees + * the segment onto the free list when the chain's first teardown returns. + * Corpus program 7260 pins what such programs print; this suite checks the + * memory side natively, which Node parity cannot: a program that rebuilds + * large trees and lists for a hundred generations reuses every block of the + * last generation, so its resident set stays at about one generation's + * worth. A segment that lost blocks would grow it by a generation each time. + * The sanitized lane frees through the system allocator (no segments) and + * keeps freed memory in quarantine, so the bound is checked in the plain + * lane only. */ +import { execFile } from "node:child_process"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { expect, test } from "vitest"; +import { compile } from "@scriptc/compiler"; + +const exec = promisify(execFile); +const sanitize = process.env["SCRIPTC_SAN"] === "1"; + +const PROGRAM = ` +interface Node { + left: Node | null; + right: Node | null; + value: number; +} +function tree(depth: number): Node { + return depth === 0 + ? { left: null, right: null, value: 1 } + : { left: tree(depth - 1), right: tree(depth - 1), value: depth }; +} +function size(node: Node | null): number { + return node === null ? 0 : 1 + size(node.left) + size(node.right); +} +class Branch { + readonly left: Branch | null; + readonly right: Branch | null; + constructor(left: Branch | null, right: Branch | null) { + this.left = left; + this.right = right; + } +} +function branches(depth: number): Branch { + return depth === 0 ? new Branch(null, null) : new Branch(branches(depth - 1), branches(depth - 1)); +} +function count(branch: Branch | null): number { + return branch === null ? 0 : 1 + count(branch.left) + count(branch.right); +} +interface Cell { + next: Cell | null; + n: number; +} +function cells(n: number): Cell | null { + let head: Cell | null = null; + for (let i = 0; i < n; i++) head = { next: head, n: i }; + return head; +} +let total = 0; +let settled = 0; +for (let generation = 0; generation < 100; generation++) { + total += size(tree(16)) + count(branches(15)); + for (let c = cells(100000); c !== null; c = c.next) total += 1; + if (generation === 2) settled = process.memoryUsage.rss(); +} +const growth = Math.round((process.memoryUsage.rss() - settled) / (1024 * 1024)); +console.log(total, growth); +`; + +test.skipIf(sanitize)( + "rebuilding large trees and lists reuses every block their teardowns free", + { timeout: 300_000 }, + async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-segments-")); + try { + const source = join(dir, "churn.ts"); + await writeFile(source, PROGRAM); + const result = await compile(source, { + backend: "llvm", + outDir: dir, + outPath: join(dir, "churn"), + }); + expect(result.ok, JSON.stringify(result.diagnostics)).toBe(true); + if (!result.ok) return; + const { stdout } = await exec(result.binaryPath, [], { timeout: 120_000 }); + const [total, growth] = stdout.trim().split(" ").map(Number); + // 100 generations of 131071 + 65535 nodes and 100000 cells. + expect(total).toBe(100 * (131071 + 65535 + 100000)); + // One generation is about 10 MiB; a lost generation per round would be + // hundreds. + expect(growth).toBeLessThan(8); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }, +); From fd4f14936ff235d6ca9a2b6e587239b5a36b6c5b Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Thu, 22 Oct 2026 15:39:35 -0700 Subject: [PATCH 02/53] Check cycle bindings only where early code can reach the reading function The import-cycle analysis treated every deferred reference (a function body, parameter default or instance field) to a cycle binding as early once any user code could run before the declaration completed. With a static evaluation order it now walks the cycle's top-level code in evaluation order, following stored callables (function and class declarations, functions and methods stored by initializers) across the cycle's modules through import aliases, and counts a deferred reference as early only when that walk reaches it before the declaration ran. Top-level reads, unknown orders, top-level await and default-exported declarations keep the previous rule. This removes the SC1016 refusal on zod 4's `fn[CONSTANT_CATCH] = true` (a computed key folded from a const's literal type inside a function nothing calls during the cycle's evaluation), and drops initialization flags and checks that can never fire. Co-Authored-By: Claude Opus 5.5 --- .../compiler/src/frontend/program.test.ts | 30 +- packages/compiler/src/frontend/program.ts | 355 ++++++++++++++---- .../main.ts.json | 8 + .../main.ts.json | 8 + .../7380-cycle-deferred-folded-key/core.ts | 5 + .../7380-cycle-deferred-folded-key/main.ts | 10 + .../7380-cycle-deferred-folded-key/util.ts | 23 ++ .../7381-cycle-deferred-reached-tdz/alpha.ts | 13 + .../7381-cycle-deferred-reached-tdz/beta.ts | 12 + .../7381-cycle-deferred-reached-tdz/main.ts | 7 + .../main.ts.json | 1 + .../main.ts.json | 1 + 12 files changed, 407 insertions(+), 66 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7380-cycle-deferred-folded-key/main.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7381-cycle-deferred-reached-tdz/main.ts.json create mode 100644 tests/corpus/7380-cycle-deferred-folded-key/core.ts create mode 100644 tests/corpus/7380-cycle-deferred-folded-key/main.ts create mode 100644 tests/corpus/7380-cycle-deferred-folded-key/util.ts create mode 100644 tests/corpus/7381-cycle-deferred-reached-tdz/alpha.ts create mode 100644 tests/corpus/7381-cycle-deferred-reached-tdz/beta.ts create mode 100644 tests/corpus/7381-cycle-deferred-reached-tdz/main.ts create mode 100644 tests/harness/corpus-costs/7380-cycle-deferred-folded-key/main.ts.json create mode 100644 tests/harness/corpus-costs/7381-cycle-deferred-reached-tdz/main.ts.json diff --git a/packages/compiler/src/frontend/program.test.ts b/packages/compiler/src/frontend/program.test.ts index bdd6260ed..5eb3e363e 100644 --- a/packages/compiler/src/frontend/program.test.ts +++ b/packages/compiler/src/frontend/program.test.ts @@ -371,15 +371,39 @@ test("repeated cycle edges reuse a file's binding index", () => { test.for([ [ - // A shared instance built by a user constructor: the reads of it in - // the partner's function can run early; the class cannot. + // A shared instance built by a user constructor that calls the + // partner's function: its read of the instance can run early. + { + "main.ts": 'import { shared } from "./a.ts"; console.log(shared.n);', + "a.ts": + 'import { f } from "./b.ts"; export class C { n = 1; constructor() { console.log(f()); } } export const shared = new C();', + "b.ts": 'import { shared } from "./a.ts"; export function f(): number { return shared.n; }', + }, + ["shared"], + ], + [ + // User code runs while the instance is built, but nothing it reaches + // calls the partner's function before the declaration completed. { "main.ts": 'import { shared } from "./a.ts"; console.log(shared.n);', "a.ts": 'import { f } from "./b.ts"; export class C { n = 1; } export const shared = new C(); console.log(f());', "b.ts": 'import { shared } from "./a.ts"; export function f(): number { return shared.n; }', }, - ["shared"], + [], + ], + [ + // The partner's top level runs first and calls a hoisted function of + // the importer that reads a const declared later; a second function + // reads another const but nothing calls it early. + { + "main.ts": 'import { late } from "./a.ts"; console.log(late());', + "a.ts": + 'import { probe } from "./b.ts"; export function read(): string { return name; } export function late(): string { return other + probe; } export const name = "n"; export const other = "o";', + "b.ts": + 'import { read } from "./a.ts"; function attempt(): string { try { return read(); } catch { return "tdz"; } } export const probe = attempt();', + }, + ["name"], ], [ // The partner's top level reads the importer's const before it ran. diff --git a/packages/compiler/src/frontend/program.ts b/packages/compiler/src/frontend/program.ts index 507d3392d..f42bc426b 100644 --- a/packages/compiler/src/frontend/program.ts +++ b/packages/compiler/src/frontend/program.ts @@ -3760,22 +3760,7 @@ function computeCycleEarlyBindings7( const evaluation = known ? comp.slice().sort((a, b) => position.get(a)! - position.get(b)!) : comp.slice(); - interface Candidate { - symbol: ts.Symbol; - name: ts.Identifier; - kind: CycleEarlyBinding["kind"]; - module: ts.SourceFile; - /** User code can run between the cycle's start and the end of the - * declaration (always true when the order is unknown). */ - exposed: boolean; - rank: number; - start: number; - end: number; - reference: ts.Identifier | null; - folded: ts.Identifier | null; - localNames: Map; - } - const candidates = new Map(); + const candidates = new Map(); const names = new Set(); let dirty = !known; evaluation.forEach((m, rank) => { @@ -3812,6 +3797,7 @@ function computeCycleEarlyBindings7( start: ts.tokenStartOfNode(stmt), end: stmt.end, reference: null, + deferredReference: null, folded: null, localNames: new Map(), }); @@ -3880,15 +3866,27 @@ function computeCycleEarlyBindings7( ref.end <= c.end ) continue; + if (inDeferredPosition7(ref)) { + if (c.exposed && c.deferredReference === null) c.deferredReference = ref; + continue; + } let early: boolean; - if (inDeferredPosition7(ref)) early = c.exposed; - else if (sf === c.module) early = ts.tokenStartOfNode(ref) < c.start; + if (sf === c.module) early = ts.tokenStartOfNode(ref) < c.start; else if (!memberSet.has(sf) || !rankOf.has(sf)) early = false; else early = !known || rankOf.get(sf)! < c.rank; if (early) c.reference = ref; } } + // With a static evaluation order, a deferred reference only runs early + // when the code that runs before the declaration completes can reach + // the function holding it. + const reached = + known && [...candidates.values()].some((c) => c.reference === null && c.deferredReference) + ? cycleReachedDeferredReferences7(program, evaluation, candidates) + : null; for (const c of candidates.values()) { + if (c.reference === null && c.deferredReference !== null) + c.reference = reached === null ? c.deferredReference : (reached.get(c.symbol) ?? null); if (c.reference === null) continue; const settled = new Set(order.filter((sf) => !rankOf.has(sf))); if (known) for (const m of evaluation.slice(c.rank + 1)) settled.add(m); @@ -3907,6 +3905,280 @@ function computeCycleEarlyBindings7( return out; } +/** Collects into `units` the callables an initializer expression stores + * (see moduleEarlyBindingsOf7), recording in `immediate` the parts of a + * stored node that run where it is evaluated. */ +function storedCallables7( + checker: ts.TypeChecker, + e: ts.Expression, + units: ts.Node[], + immediate: Map, +): void { + while ( + ts.isParenthesizedExpression(e) || + ts.isAsExpression(e) || + ts.isSatisfiesExpression(e) || + ts.isNonNullExpression(e) + ) + e = e.expression; + if (ts.isArrowFunction(e) || ts.isFunctionExpression(e)) { + units.push(e); + } else if (ts.isClassExpression(e)) { + units.push(e); + immediate.set(e, classSyncParts7(e)); + } else if (ts.isObjectLiteralExpression(e)) { + for (const p of e.properties.nodes) { + if ( + ts.isMethodDeclaration(p) || + ts.isGetAccessorDeclaration(p) || + ts.isSetAccessorDeclaration(p) + ) { + // A computed key evaluates where the literal does; a plain + // member-chain key (`[Equal.symbol]`) runs no user code. + if (!ts.isComputedPropertyName(p.name)) units.push(p); + else if (memberChainKey7(p.name.expression)) { + units.push(p); + immediate.set(p, [p.name]); + } + } else if (ts.isPropertyAssignment(p)) + storedCallables7(checker, p.initializer, units, immediate); + } + } else if (ts.isArrayLiteralExpression(e)) { + for (const el of e.elements.nodes) + if (!ts.isSpreadElement(el)) storedCallables7(checker, el, units, immediate); + } else if (ts.isCallExpression(e)) { + // `const f = wrap(callback)` where wrap only returns closures over + // the callback: it runs when the stored result is called. + e.arguments.nodes.forEach((arg, index) => { + let fn: ts.Expression = arg; + while (ts.isParenthesizedExpression(fn)) fn = fn.expression; + if ( + (ts.isArrowFunction(fn) || ts.isFunctionExpression(fn)) && + returnsOnlyClosuresOver7(checker, e.expression, index) + ) + units.push(fn); + }); + } +} + +/** A binding of an import cycle and what the reference scan found. */ +interface CycleCandidate7 { + symbol: ts.Symbol; + name: ts.Identifier; + kind: CycleEarlyBinding["kind"]; + module: ts.SourceFile; + /** User code can run between the cycle's start and the end of the + * declaration (always true when the order is unknown). */ + exposed: boolean; + rank: number; + start: number; + end: number; + /** A reference read at top level before the declaration ran. */ + reference: ts.Identifier | null; + /** A deferred reference (function body, parameter default, instance + * field) when user code may run before the declaration completes. */ + deferredReference: ts.Identifier | null; + folded: ts.Identifier | null; + localNames: Map; +} + +/** For one import cycle with a static evaluation order: the first deferred + * reference to each candidate that code running before the candidate's + * declaration completes can reach, or null when the walk cannot be precise + * (top-level await, a default-exported function or class declaration). + * + * The cycle's top-level code runs in evaluation order. A stored callable + * (a function or class declaration, or a function, class or object-literal + * method stored by a variable initializer) becomes reachable once + * reachable code names its binding, through an import alias or a + * namespace member too; every other function-like in reachable code counts + * as reachable, since it may be handed to a callee. A namespace import of + * a cycle member used other than for a member read makes every stored + * callable reachable. moduleEarlyBindingsOf7 is the single-module form. */ +function cycleReachedDeferredReferences7( + program: ts.Program, + evaluation: readonly ts.SourceFile[], + candidates: ReadonlyMap, +): Map | null { + const checker = program.getTypeChecker(); + const members = new Set(evaluation); + for (const sf of evaluation) { + if (hasTopLevelAwait7(sf)) return null; + for (const stmt of sf.statements.nodes) { + if ( + (ts.isFunctionDeclaration(stmt) || ts.isClassDeclaration(stmt)) && + (ts.getCombinedModifierFlags(stmt as unknown as ts.Declaration) & + ts.ModifierFlags.Default) !== + 0 + ) + return null; + } + } + const stored = new Set(); + const immediate = new Map(); + const storedNames: ts.Identifier[] = []; + const unitsOfName = new Map(); + const store = (name: ts.Identifier, units: ts.Node[]): void => { + if (units.length === 0) return; + for (const u of units) stored.add(u); + storedNames.push(name); + unitsOfName.set(name, units); + }; + for (const sf of evaluation) { + for (const stmt of sf.statements.nodes) { + if ( + (ts.getCombinedModifierFlags(stmt as unknown as ts.Declaration) & + ts.ModifierFlags.Ambient) !== + 0 + ) + continue; + if (ts.isFunctionDeclaration(stmt) && stmt.name !== undefined && stmt.body !== undefined) { + store(stmt.name, [stmt]); + } else if (ts.isClassDeclaration(stmt) && stmt.name !== undefined) { + store(stmt.name, [stmt]); + } else if (ts.isVariableStatement(stmt)) { + for (const d of stmt.declarationList.declarations.nodes) { + if (ts.isIdentifier(d.name) && d.initializer !== undefined) { + const units: ts.Node[] = []; + storedCallables7(checker, d.initializer, units, immediate); + store(d.name, units); + } + } + } + } + } + const unitsBySymbol = new Map(); + const allUnits: ts.Node[] = []; + for (const name of storedNames) { + const symbol = checker.getSymbolAtLocation(name); + if (symbol === undefined) continue; + const units = unitsOfName.get(name)!; + unitsBySymbol.set(symbol, [...(unitsBySymbol.get(symbol) ?? []), ...units]); + allUnits.push(...units); + } + // The identifier texts worth resolving: declared names, and the local + // names of imports (named, default and namespace) in cycle members. + const names = new Set(); + for (const c of candidates.values()) names.add(c.name.text); + for (const name of storedNames) names.add(name.text); + for (const sf of evaluation) { + for (const stmt of sf.statements.nodes) { + if (!ts.isImportDeclaration(stmt) || stmt.importClause === undefined) continue; + const clause = stmt.importClause; + if (clause.name !== undefined) names.add(clause.name.text); + const bindings = clause.namedBindings; + if (bindings === undefined) continue; + if (ts.isNamedImports(bindings)) + for (const element of bindings.elements.nodes) names.add(element.name.text); + else names.add(bindings.name.text); + } + } + const idents: ts.Identifier[] = []; + for (const sf of evaluation) { + ts.walkPreorder(sf, (node) => { + if (ts.isImportDeclaration(node)) return "skip"; + if (ts.isIdentifier(node) && names.has(node.text)) idents.push(node); + return undefined; + }); + } + checker.prefetchSymbolNodesExact(idents); + const symbolOf = (id: ts.Identifier): ts.Symbol | undefined => { + const parent = id.parent; + let symbol = checker.getSymbolAtLocation(id); + if (parent !== undefined && ts.isShorthandPropertyAssignment(parent) && parent.name === id) { + symbol = checker.getShorthandAssignmentValueSymbol(parent) ?? symbol; + } + if (symbol !== undefined && symbol.flags & ts.SymbolFlags.Alias) { + symbol = checker.getAliasedSymbol(symbol); + } + return symbol; + }; + const isMemberNamespace = (symbol: ts.Symbol): boolean => + (symbol.flags & ts.SymbolFlags.ValueModule) !== 0 && + symbol.declarations.some((d) => ts.isSourceFile(d) && members.has(d)); + const declared = new Set(); + const reachedUnits = new Set(); + const out = new Map(); + const work: ts.Node[] = []; + const reach = (unit: ts.Node): void => { + if (reachedUnits.has(unit)) return; + reachedUnits.add(unit); + work.push(unit); + }; + const visit = (root: ts.Node): void => { + ts.walkPreorder(root, (node) => { + if (node !== root && stored.has(node)) { + for (const part of immediate.get(node) ?? []) visit(part); + return "skip"; + } + if (ts.isImportDeclaration(node)) return "skip"; + if (!ts.isIdentifier(node) || !names.has(node.text)) return undefined; + if (inTypePosition7(node)) return undefined; + const parent = node.parent; + if (parent !== undefined && ts.isExportSpecifier(parent)) return undefined; + if (parent !== undefined && declaredName7(parent) === node) return undefined; + const symbol = symbolOf(node); + if (symbol === undefined) return undefined; + if (isMemberNamespace(symbol)) { + // `ns.member` resolves through the member's own identifier; any + // other use can reach every export. + if ( + !( + parent !== undefined && + ts.isPropertyAccessExpression(parent) && + parent.expression === node + ) + ) + for (const unit of allUnits) reach(unit); + return undefined; + } + const c = candidates.get(symbol); + if ( + c !== undefined && + node !== c.name && + !declared.has(symbol) && + !out.has(symbol) && + inDeferredPosition7(node) && + !( + c.kind === "class" && + ts.getSourceFileOfNode(node) === c.module && + ts.tokenStartOfNode(node) >= c.start && + node.end <= c.end + ) + ) + out.set(symbol, node); + for (const unit of unitsBySymbol.get(symbol) ?? []) reach(unit); + return undefined; + }); + }; + const drain = (): void => { + while (work.length > 0) visit(work.pop()!); + }; + const declare = (name: ts.BindingName): void => { + for (const id of boundNames7(name)) { + const symbol = checker.getSymbolAtLocation(id); + if (symbol !== undefined) declared.add(symbol); + } + }; + for (const sf of evaluation) { + for (const stmt of sf.statements.nodes) { + if (ts.isClassDeclaration(stmt)) { + for (const part of classSyncParts7(stmt)) visit(part); + drain(); + if (stmt.name !== undefined) declare(stmt.name); + } else if (ts.isVariableStatement(stmt)) { + for (const d of stmt.declarationList.declarations.nodes) { + visit(d); + drain(); + declare(d.name); + } + } else if (!ts.isFunctionDeclaration(stmt)) visit(stmt); + drain(); + } + } + return out; +} + /** Top-level let/const and class bindings, in modules outside the * exactly-initialized cycles, that code running before their declaration * completes can read. @@ -3971,51 +4243,8 @@ function moduleEarlyBindingsOf7( storedBy.set(name.text, list); }; const checker = program.getTypeChecker(); - const storedIn = (e: ts.Expression, units: ts.Node[]): void => { - while ( - ts.isParenthesizedExpression(e) || - ts.isAsExpression(e) || - ts.isSatisfiesExpression(e) || - ts.isNonNullExpression(e) - ) - e = e.expression; - if (ts.isArrowFunction(e) || ts.isFunctionExpression(e)) { - units.push(e); - } else if (ts.isClassExpression(e)) { - units.push(e); - immediate.set(e, classSyncParts7(e)); - } else if (ts.isObjectLiteralExpression(e)) { - for (const p of e.properties.nodes) { - if ( - ts.isMethodDeclaration(p) || - ts.isGetAccessorDeclaration(p) || - ts.isSetAccessorDeclaration(p) - ) { - // A computed key evaluates where the literal does; a plain - // member-chain key (`[Equal.symbol]`) runs no user code. - if (!ts.isComputedPropertyName(p.name)) units.push(p); - else if (memberChainKey7(p.name.expression)) { - units.push(p); - immediate.set(p, [p.name]); - } - } else if (ts.isPropertyAssignment(p)) storedIn(p.initializer, units); - } - } else if (ts.isArrayLiteralExpression(e)) { - for (const el of e.elements.nodes) if (!ts.isSpreadElement(el)) storedIn(el, units); - } else if (ts.isCallExpression(e)) { - // `const f = wrap(callback)` where wrap only returns closures over - // the callback: it runs when the stored result is called. - e.arguments.nodes.forEach((arg, index) => { - let fn: ts.Expression = arg; - while (ts.isParenthesizedExpression(fn)) fn = fn.expression; - if ( - (ts.isArrowFunction(fn) || ts.isFunctionExpression(fn)) && - returnsOnlyClosuresOver7(checker, e.expression, index) - ) - units.push(fn); - }); - } - }; + const storedIn = (e: ts.Expression, units: ts.Node[]): void => + storedCallables7(checker, e, units, immediate); sf.statements.nodes.forEach((stmt, index) => { if (ambient(stmt)) return; if (ts.isFunctionDeclaration(stmt) && stmt.name !== undefined && stmt.body !== undefined) { diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7380-cycle-deferred-folded-key/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7380-cycle-deferred-folded-key/main.ts.json new file mode 100644 index 000000000..18ef785c5 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7380-cycle-deferred-folded-key/main.ts.json @@ -0,0 +1,8 @@ +{ + "order": [ + "/tests/corpus/7380-cycle-deferred-folded-key/util.ts", + "/tests/corpus/7380-cycle-deferred-folded-key/core.ts", + "/tests/corpus/7380-cycle-deferred-folded-key/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7381-cycle-deferred-reached-tdz/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7381-cycle-deferred-reached-tdz/main.ts.json new file mode 100644 index 000000000..65d9b19e9 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7381-cycle-deferred-reached-tdz/main.ts.json @@ -0,0 +1,8 @@ +{ + "order": [ + "/tests/corpus/7381-cycle-deferred-reached-tdz/beta.ts", + "/tests/corpus/7381-cycle-deferred-reached-tdz/alpha.ts", + "/tests/corpus/7381-cycle-deferred-reached-tdz/main.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7380-cycle-deferred-folded-key/core.ts b/tests/corpus/7380-cycle-deferred-folded-key/core.ts new file mode 100644 index 000000000..7f4f4ab14 --- /dev/null +++ b/tests/corpus/7380-cycle-deferred-folded-key/core.ts @@ -0,0 +1,5 @@ +import { describe, kind, mark } from "./util.ts"; + +export const version = 3; +export const marked = mark({}); +console.log("core body", kind(), describe()); diff --git a/tests/corpus/7380-cycle-deferred-folded-key/main.ts b/tests/corpus/7380-cycle-deferred-folded-key/main.ts new file mode 100644 index 000000000..92617eba3 --- /dev/null +++ b/tests/corpus/7380-cycle-deferred-folded-key/main.ts @@ -0,0 +1,10 @@ +// A cycle member runs user code before it declares a constant that only +// functions read, one of them as a computed key and one under `typeof` +// (both resolved from the constant's literal type). Nothing calls those +// functions before the declaration ran, so they read the constant +// normally once the other member's top level calls them. +import { marked } from "./core.ts"; +import { describe, kind, mark } from "./util.ts"; + +console.log(JSON.stringify(marked), kind()); +console.log(JSON.stringify(mark({ seen: false })), describe()); diff --git a/tests/corpus/7380-cycle-deferred-folded-key/util.ts b/tests/corpus/7380-cycle-deferred-folded-key/util.ts new file mode 100644 index 000000000..a038d9e63 --- /dev/null +++ b/tests/corpus/7380-cycle-deferred-folded-key/util.ts @@ -0,0 +1,23 @@ +import { version } from "./core.ts"; + +export function makeCounter(): () => number { + let n = 0; + return () => ++n; +} +const next = makeCounter(); +console.log("util body", next(), next()); + +export const TAG = "~tagged"; + +export function mark(box: Record): Record { + box[TAG] = true; + return box; +} + +export function kind(): string { + return typeof TAG; +} + +export function describe(): string { + return `core v${version}`; +} diff --git a/tests/corpus/7381-cycle-deferred-reached-tdz/alpha.ts b/tests/corpus/7381-cycle-deferred-reached-tdz/alpha.ts new file mode 100644 index 000000000..8dc002c27 --- /dev/null +++ b/tests/corpus/7381-cycle-deferred-reached-tdz/alpha.ts @@ -0,0 +1,13 @@ +import { probe } from "./beta.ts"; + +export function report(): string { + return `alpha ${NAME}`; +} + +export function late(): string { + return `${OTHER} after ${probe}`; +} + +export const NAME = "ready"; +export const OTHER = "other"; +console.log("alpha body", report()); diff --git a/tests/corpus/7381-cycle-deferred-reached-tdz/beta.ts b/tests/corpus/7381-cycle-deferred-reached-tdz/beta.ts new file mode 100644 index 000000000..c3d1551ff --- /dev/null +++ b/tests/corpus/7381-cycle-deferred-reached-tdz/beta.ts @@ -0,0 +1,12 @@ +import { report } from "./alpha.ts"; + +function attempt(): string { + try { + return report(); + } catch (error) { + return error instanceof ReferenceError ? `ReferenceError: ${error.message}` : "other error"; + } +} + +export const probe = attempt(); +console.log("beta body", probe); diff --git a/tests/corpus/7381-cycle-deferred-reached-tdz/main.ts b/tests/corpus/7381-cycle-deferred-reached-tdz/main.ts new file mode 100644 index 000000000..6a0b7e3ca --- /dev/null +++ b/tests/corpus/7381-cycle-deferred-reached-tdz/main.ts @@ -0,0 +1,7 @@ +// The first cycle member to run calls a function of the other member +// before that member declared the constant the function reads: Node's +// temporal-dead-zone ReferenceError. A second function of the same member +// reads another constant but runs only after the cycle finished. +import { late, report } from "./alpha.ts"; + +console.log(report(), late()); diff --git a/tests/harness/corpus-costs/7380-cycle-deferred-folded-key/main.ts.json b/tests/harness/corpus-costs/7380-cycle-deferred-folded-key/main.ts.json new file mode 100644 index 000000000..d5d1c5f65 --- /dev/null +++ b/tests/harness/corpus-costs/7380-cycle-deferred-folded-key/main.ts.json @@ -0,0 +1 @@ +{ "plain": 2500, "san": 3500 } diff --git a/tests/harness/corpus-costs/7381-cycle-deferred-reached-tdz/main.ts.json b/tests/harness/corpus-costs/7381-cycle-deferred-reached-tdz/main.ts.json new file mode 100644 index 000000000..b24c0c970 --- /dev/null +++ b/tests/harness/corpus-costs/7381-cycle-deferred-reached-tdz/main.ts.json @@ -0,0 +1 @@ +{ "plain": 1800, "san": 2600 } From 862d5e836d42da00b05dc948d77d9b42c688e5d6 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Thu, 22 Oct 2026 16:12:37 -0700 Subject: [PATCH 03/53] Give the tsgo/tsc-ts IR parity job 60 minutes Its release pass alone takes about 15 minutes, so the dev pass is regularly cancelled at the 30-minute limit on main and on PRs. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5a100e374..3b9ef44a5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -524,7 +524,7 @@ jobs: if: ${{ needs.validation_scope.outputs.full == 'true' && needs.lanes.outputs.linux == 'true' }} name: test (tsgo and tsc-ts give the same IR) runs-on: ubuntu-24.04 - timeout-minutes: 30 + timeout-minutes: 60 steps: - uses: actions/checkout@v4 - uses: pnpm/action-setup@v4 From 43917080c9f8e2af993a632a04178b9b720901c7 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 20:01:47 -0500 Subject: [PATCH 04/53] Reach a class's methods from 'this' in its static initializers The cycle walk resolved only identifiers, so a static field initializer or static block that called a method through 'this' did not reach the method, and its reads of a cycle binding declared later lost their temporal-dead-zone checks. 'this' in static code now reaches the class's stored unit, as naming the class does. --- .../compiler/src/frontend/program.test.ts | 21 ++++++++++ packages/compiler/src/frontend/program.ts | 41 +++++++++++++++++++ .../7382-cycle-static-this-tdz/main.ts.json | 8 ++++ .../7382-cycle-static-this-tdz/alpha.ts | 11 +++++ .../corpus/7382-cycle-static-this-tdz/beta.ts | 26 ++++++++++++ .../corpus/7382-cycle-static-this-tdz/main.ts | 8 ++++ .../7382-cycle-static-this-tdz/main.ts.json | 1 + 7 files changed, 116 insertions(+) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7382-cycle-static-this-tdz/main.ts.json create mode 100644 tests/corpus/7382-cycle-static-this-tdz/alpha.ts create mode 100644 tests/corpus/7382-cycle-static-this-tdz/beta.ts create mode 100644 tests/corpus/7382-cycle-static-this-tdz/main.ts create mode 100644 tests/harness/corpus-costs/7382-cycle-static-this-tdz/main.ts.json diff --git a/packages/compiler/src/frontend/program.test.ts b/packages/compiler/src/frontend/program.test.ts index 5eb3e363e..6a068353a 100644 --- a/packages/compiler/src/frontend/program.test.ts +++ b/packages/compiler/src/frontend/program.test.ts @@ -405,6 +405,27 @@ test.for([ }, ["name"], ], + [ + // The partner's class calls a method through `this` in a static + // initializer; the method reads a const of the importer declared later. + { + "main.ts": 'import { name } from "./a.ts"; console.log(name);', + "a.ts": 'import { Probe } from "./b.ts"; export const name = "n"; console.log(Probe.value);', + "b.ts": + 'import { name } from "./a.ts"; export class Probe { static read(): string { return name; } static value = this.read(); }', + }, + ["name"], + ], + [ + // The same through a static block of a class stored by a const. + { + "main.ts": 'import { name } from "./a.ts"; console.log(name);', + "a.ts": 'import { Box } from "./b.ts"; export const name = "n"; console.log(Box.value);', + "b.ts": + 'import { name } from "./a.ts"; export const Box = class { static value = ""; static read(): string { return name; } static { this.value = this.read(); } };', + }, + ["name"], + ], [ // The partner's top level reads the importer's const before it ran. { diff --git a/packages/compiler/src/frontend/program.ts b/packages/compiler/src/frontend/program.ts index f42bc426b..c8384ef65 100644 --- a/packages/compiler/src/frontend/program.ts +++ b/packages/compiler/src/frontend/program.ts @@ -4112,6 +4112,13 @@ function cycleReachedDeferredReferences7( return "skip"; } if (ts.isImportDeclaration(node)) return "skip"; + if (node.kind === ts.SyntaxKind.ThisKeyword) { + // `this` in a static initializer or block names the class, so it + // reaches the class's methods as naming the class does. + const owner = staticThisClass7(node); + if (owner !== undefined && stored.has(owner)) reach(owner); + return undefined; + } if (!ts.isIdentifier(node) || !names.has(node.text)) return undefined; if (inTypePosition7(node)) return undefined; const parent = node.parent; @@ -4497,6 +4504,40 @@ function returnsOnlyClosuresOver7( * expressions, decorators, computed member keys, static field * initializers and static blocks. Methods and instance fields wait for a * call or a construction. */ +/** The class whose static field initializer or static block evaluates `this` + * (through arrow functions, which keep the enclosing `this`), or undefined + * when another function or a class's instance side binds it. */ +function staticThisClass7(node: ts.Node): ts.ClassLikeDeclaration | undefined { + let child = node; + for (let n = node.parent; n !== undefined; child = n, n = n.parent) { + if (ts.isArrowFunction(n)) continue; + if (ts.isClassStaticBlockDeclaration(n)) return n.parent as ts.ClassLikeDeclaration; + if (ts.isPropertyDeclaration(n)) { + if (n.initializer !== child) return undefined; + return (ts.getCombinedModifierFlags(n) & ts.ModifierFlags.Static) !== 0 + ? (n.parent as ts.ClassLikeDeclaration) + : undefined; + } + if (ts.isComputedPropertyName(n)) { + // A computed key evaluates in the scope around the class. + const member = n.parent; + const cls = member?.parent; + if (cls === undefined) return undefined; + child = cls; + n = cls; + continue; + } + if ( + ts.isFunctionLike(n) || + ts.isClassDeclaration(n) || + ts.isClassExpression(n) || + ts.isSourceFile(n) + ) + return undefined; + } + return undefined; +} + function classSyncParts7(c: ts.ClassLikeDeclaration): ts.Node[] { const decorators = (n: ts.Node): ts.Node[] => ((n as { modifiers?: ts.NodeArray }).modifiers?.nodes ?? []).filter( diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7382-cycle-static-this-tdz/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7382-cycle-static-this-tdz/main.ts.json new file mode 100644 index 000000000..986683684 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7382-cycle-static-this-tdz/main.ts.json @@ -0,0 +1,8 @@ +{ + "order": [ + "/tests/corpus/7382-cycle-static-this-tdz/beta.ts", + "/tests/corpus/7382-cycle-static-this-tdz/alpha.ts", + "/tests/corpus/7382-cycle-static-this-tdz/main.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7382-cycle-static-this-tdz/alpha.ts b/tests/corpus/7382-cycle-static-this-tdz/alpha.ts new file mode 100644 index 000000000..ff8eb4a87 --- /dev/null +++ b/tests/corpus/7382-cycle-static-this-tdz/alpha.ts @@ -0,0 +1,11 @@ +import { Probe, Lazy } from "./beta.ts"; + +export { Probe }; +export const label = "ready"; +export const other = "other"; + +export function later(): string { + return `${Lazy.describe()} ${other}`; +} + +console.log("alpha body", Probe.result, Lazy.value); diff --git a/tests/corpus/7382-cycle-static-this-tdz/beta.ts b/tests/corpus/7382-cycle-static-this-tdz/beta.ts new file mode 100644 index 000000000..428907967 --- /dev/null +++ b/tests/corpus/7382-cycle-static-this-tdz/beta.ts @@ -0,0 +1,26 @@ +import { label, other } from "./alpha.ts"; + +function guard(read: () => string): string { + try { + return read(); + } catch (error) { + return error instanceof ReferenceError ? `ReferenceError: ${error.message}` : "other error"; + } +} + +export class Probe { + static helper(): string { + return guard(() => `label ${label}`); + } + static result = this.helper(); +} + +export class Lazy { + static read(): string { + return guard(() => `other ${other}`); + } + static describe(): string { + return `lazy ${other}`; + } + static value = [0].map(() => this.read()).join(""); +} diff --git a/tests/corpus/7382-cycle-static-this-tdz/main.ts b/tests/corpus/7382-cycle-static-this-tdz/main.ts new file mode 100644 index 000000000..e3bbf1581 --- /dev/null +++ b/tests/corpus/7382-cycle-static-this-tdz/main.ts @@ -0,0 +1,8 @@ +// The first cycle member to run builds classes whose static initializers +// call a method through `this`, directly and from an arrow function; both +// methods read constants of the other member before it declared them: +// Node's temporal-dead-zone ReferenceError. A third method reads a constant +// only after the cycle finished. +import { label, Probe, later } from "./alpha.ts"; + +console.log(label, Probe.result, later()); diff --git a/tests/harness/corpus-costs/7382-cycle-static-this-tdz/main.ts.json b/tests/harness/corpus-costs/7382-cycle-static-this-tdz/main.ts.json new file mode 100644 index 000000000..b24c0c970 --- /dev/null +++ b/tests/harness/corpus-costs/7382-cycle-static-this-tdz/main.ts.json @@ -0,0 +1 @@ +{ "plain": 1800, "san": 2600 } From da289bb593ef27c17e45c0121b63c61fdee1dbdb Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 23:13:49 -0500 Subject: [PATCH 05/53] Support writev on Transform and PassThrough, and runtime stream option values --- .../builtin-apis/streams-and-readline.mdx | 4 +- packages/cli/test/native-link-info.test.ts | 4 +- .../src/backend/llvm/expr-stream-callbacks.ts | 11 + .../compiler/src/backend/llvm/lib-shared.ts | 2 + packages/compiler/src/backend/runtime-abi.ts | 4 +- .../src/backend/runtime-pack-targets.test.ts | 2 +- .../compiler/src/backend/runtime-pack.test.ts | 2 +- .../src/frontend/lowering/lower-classes.ts | 6 +- .../src/frontend/lowering/lower-stream.ts | 230 ++++++++++++++++-- packages/compiler/src/ir/builtin-effects.ts | 1 + .../compiler/src/ir/builtin-signatures.ts | 5 + packages/compiler/src/ir/runtime-features.ts | 2 + packages/compiler/src/ir/validate.ts | 2 + .../test/native-codegen-integration.test.ts | 4 +- .../corpus/7460-transform-writev.mjs.json | 6 + .../corpus/7461-transform-writev.cjs.json | 6 + .../corpus/7462-transform-writev.ts.json | 6 + .../corpus/7463-stream-option-values.ts.json | 6 + .../corpus/7464-stream-option-values.mjs.json | 6 + .../7465-transform-callback-values.ts.json | 6 + .../runtime-darwin-arm64/scripts/build.mjs | 2 +- .../runtime-pack-common/scripts/build.mjs | 2 +- packages/runtime/src/scr_console.c | 2 +- packages/runtime/src/scr_core.h | 2 +- packages/runtime/src/scr_dyn_builtin_stream.c | 9 +- packages/runtime/src/scr_stream.c | 77 +++++- packages/runtime/src/scr_stream.h | 8 + tests/corpus/7460-transform-writev.mjs | 33 +++ tests/corpus/7461-transform-writev.cjs | 33 +++ tests/corpus/7462-transform-writev.ts | 109 +++++++++ tests/corpus/7463-stream-option-values.ts | 58 +++++ tests/corpus/7464-stream-option-values.mjs | 28 +++ .../corpus/7465-transform-callback-values.ts | 41 ++++ .../7460-transform-writev.mjs.json | 1 + .../7461-transform-writev.cjs.json | 1 + .../7462-transform-writev.ts.json | 1 + .../7463-stream-option-values.ts.json | 1 + .../7464-stream-option-values.mjs.json | 1 + .../7465-transform-callback-values.ts.json | 1 + .../self-hosting-native-toolchain.test.ts | 2 +- 40 files changed, 670 insertions(+), 57 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7460-transform-writev.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7461-transform-writev.cjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7462-transform-writev.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7463-stream-option-values.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7464-stream-option-values.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7465-transform-callback-values.ts.json create mode 100644 tests/corpus/7460-transform-writev.mjs create mode 100644 tests/corpus/7461-transform-writev.cjs create mode 100644 tests/corpus/7462-transform-writev.ts create mode 100644 tests/corpus/7463-stream-option-values.ts create mode 100644 tests/corpus/7464-stream-option-values.mjs create mode 100644 tests/corpus/7465-transform-callback-values.ts create mode 100644 tests/harness/corpus-costs/7460-transform-writev.mjs.json create mode 100644 tests/harness/corpus-costs/7461-transform-writev.cjs.json create mode 100644 tests/harness/corpus-costs/7462-transform-writev.ts.json create mode 100644 tests/harness/corpus-costs/7463-stream-option-values.ts.json create mode 100644 tests/harness/corpus-costs/7464-stream-option-values.mjs.json create mode 100644 tests/harness/corpus-costs/7465-transform-callback-values.ts.json diff --git a/docs/content/limitations/builtin-apis/streams-and-readline.mdx b/docs/content/limitations/builtin-apis/streams-and-readline.mdx index fe6bb2dfb..b88ead1f4 100644 --- a/docs/content/limitations/builtin-apis/streams-and-readline.mdx +++ b/docs/content/limitations/builtin-apis/streams-and-readline.mdx @@ -32,9 +32,9 @@ `readable.compose(stage, { signal })` is `stream.compose(readable, stage)` with the signal added; the options are an object literal whose `signal` is a variable or a property read. `pipeline` from `node:stream/promises` accepts a trailing `{ signal, end }` object, with or without function stages, and `finished` accepts `readable`, `writable`, `error`, `cleanup`, and `signal` options, with boolean literals for the flags. Stream constructors accept a `signal` option in an options object literal, and `stream.addAbortSignal` accepts any stream. - The `construct` option and `_construct` method, `decodeStrings`, and `defaultEncoding` work on every stream class, and the `writev` option and `_writev` method on Writable and Duplex; `writev` on a Transform or PassThrough, assigning `_construct` after construction, and a non-literal `decodeStrings` or `defaultEncoding` in an options object literal are unsupported. + The `construct` option and `_construct` method, `decodeStrings`, and `defaultEncoding` work on every stream class, and the `writev` option and `_writev` method on every class with a writable side. + In typed code, a `writev` entry's `encoding` declared as a string receives `""` where Node passes `undefined` or `null` (an object-mode write). `setDefaultEncoding()` takes any encoding Node knows. - A `transform` or `flush` option callback whose completion callback types `data` as `any`, as @types/node's `TransformCallback` does, may pass a Buffer, a string, or nothing; any other value throws a TypeError where the callback is called, instead of Node's `ERR_INVALID_ARG_TYPE` stream error. `node:readline` and `node:readline/promises` (also reached as `readline.promises`) interfaces, from `createInterface(options)` or `createInterface(input, output, completer, terminal)`, read `process.stdin` or a readable stream: `question`, `prompt`, `write` (with a key object literal of `name`, `ctrl`, `meta`, and `shift`), `pause`, `resume`, `'line'`/`'close'`/`'pause'`/`'resume'`/`'error'`/`'history'`/`'SIGINT'`/`'SIGTSTP'`/`'SIGCONT'` listeners, and `for await` compile. `on`, `once`, and `addListener` return the interface, so registrations chain. The `history`, `historySize`, `removeHistoryDuplicates`, `completer`, `tabSize`, and `escapeCodeTimeout` options are validated as in Node. diff --git a/packages/cli/test/native-link-info.test.ts b/packages/cli/test/native-link-info.test.ts index bf5b3be14..8715d9356 100644 --- a/packages/cli/test/native-link-info.test.ts +++ b/packages/cli/test/native-link-info.test.ts @@ -72,7 +72,7 @@ describe.runIf(supported)("macOS arm64 native link info", () => { minimum_os: "14.0", }, program: { object, entry_symbol: "main" }, - runtime_abi: { version: 33, marker: "scr_runtime_abi_v33" }, + runtime_abi: { version: 34, marker: "scr_runtime_abi_v34" }, runtime_pack: { kind: "precompiled", package: "@scriptc/runtime-darwin-arm64", @@ -165,6 +165,6 @@ describe.runIf(supported)("macOS arm64 native link info", () => { () => null, (failure: { stderr?: string }) => failure, ); - expect(error?.stderr).toContain("scr_runtime_abi_v33"); + expect(error?.stderr).toContain("scr_runtime_abi_v34"); }); }); diff --git a/packages/compiler/src/backend/llvm/expr-stream-callbacks.ts b/packages/compiler/src/backend/llvm/expr-stream-callbacks.ts index 14a2e3432..b368c91f4 100644 --- a/packages/compiler/src/backend/llvm/expr-stream-callbacks.ts +++ b/packages/compiler/src/backend/llvm/expr-stream-callbacks.ts @@ -727,6 +727,17 @@ export function streamCbThunkFor( const ty = host.llType(p); host.declare(`declare zeroext i1 @scr_exc_pending()`); if (hasThis) host.declare(`declare void @scr_stream_release_v(ptr)`); + // An entry `encoding` declared as a string (the declarations' + // `encoding: string`) receives "" where Node passes undefined or + // null (an object-mode write), as a write's encoding parameter does. + const entry = + p.kind === "array" && p.elem.kind === "record" + ? host.recordsById.get(p.elem.shapeId) + : undefined; + if (entry?.fields.some((f) => f.name === "encoding" && f.type.kind === "string")) { + host.declare(`declare void @scr_stream_writev_fill_encodings(ptr)`); + d.push(` call void @scr_stream_writev_fill_encodings(ptr %chunks)`); + } d.push( ` %vt${i} = call ${ty === "i1" ? "zeroext i1" : ty} @${helper}(ptr %chunks, ptr null)`, ` %vtbad${i} = call zeroext i1 @scr_exc_pending()`, diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index 054d9341b..548e8e597 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -1135,6 +1135,8 @@ export const LIB_FN_SYMS: Record = { "readable.pushEncoding": "scr_stream_set_push_encoding", "stream.objectMode": "scr_stream_object_mode", "stream.decodeStrings": "scr_stream_decode_strings", + "stream.optionValues": "scr_stream_option_values", + "stream.checkEncodingOption": "scr_stream_check_encoding_option", "stream.duplexFromGen": "scr_stream_duplex_from_gen", "stream.duplexFromDyn": "scr_stream_duplex_from_dyn", "stream.duplexifyReadable": "scr_stream_duplexify_readable", diff --git a/packages/compiler/src/backend/runtime-abi.ts b/packages/compiler/src/backend/runtime-abi.ts index 7d3dcbc02..173de6408 100644 --- a/packages/compiler/src/backend/runtime-abi.ts +++ b/packages/compiler/src/backend/runtime-abi.ts @@ -2,8 +2,8 @@ * of the link contract: a program object keeps this symbol undefined and the * matching runtime defines it, so incompatible manual links fail before the * program can start. */ -export const RUNTIME_ABI_VERSION = 33 as const; -export const RUNTIME_ABI_MARKER = "scr_runtime_abi_v33" as const; +export const RUNTIME_ABI_VERSION = 34 as const; +export const RUNTIME_ABI_MARKER = "scr_runtime_abi_v34" as const; /** Object consumption is useful today, but the complete scr_* surface may * still change before 1.0. The versioned marker prevents accidental mixing; diff --git a/packages/compiler/src/backend/runtime-pack-targets.test.ts b/packages/compiler/src/backend/runtime-pack-targets.test.ts index 532051b88..923b55c5b 100644 --- a/packages/compiler/src/backend/runtime-pack-targets.test.ts +++ b/packages/compiler/src/backend/runtime-pack-targets.test.ts @@ -52,7 +52,7 @@ async function fixture(target: NativeTargetSpec, workers = false) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 33, marker: "scr_runtime_abi_v33" }, + runtime_abi: { version: 34, marker: "scr_runtime_abi_v34" }, compiler: { command: "fixture", identity: "fixture", target: target.llvmTriple }, macros: { executable: [], excluded: ["SCR_LIB"], sanitizer: "external-toolchain-required" }, flavors: { diff --git a/packages/compiler/src/backend/runtime-pack.test.ts b/packages/compiler/src/backend/runtime-pack.test.ts index 18b04b9a6..681f86817 100644 --- a/packages/compiler/src/backend/runtime-pack.test.ts +++ b/packages/compiler/src/backend/runtime-pack.test.ts @@ -127,7 +127,7 @@ async function fixture(target: NativeTargetSpec = MACOS_ARM64_TARGET) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 33, marker: "scr_runtime_abi_v33" }, + runtime_abi: { version: 34, marker: "scr_runtime_abi_v34" }, compiler: { command: "clang", identity: "fixture clang", diff --git a/packages/compiler/src/frontend/lowering/lower-classes.ts b/packages/compiler/src/frontend/lowering/lower-classes.ts index 6dd0124bd..e3ab29dba 100644 --- a/packages/compiler/src/frontend/lowering/lower-classes.ts +++ b/packages/compiler/src/frontend/lowering/lower-classes.ts @@ -2734,8 +2734,8 @@ export function collectClassShapeInner( // state, so an override or shadowing field would split-brain. // Underscore methods are the SUPPORTED override form — but only // the ones the class's own base consumes (a `_read` on a - // Transform, or `_writev`/`_construct` anywhere, would be consumed - // by Node machinery that has no lowering here). + // Transform, or `_writev` on a Readable, would be consumed by Node + // machinery that has no lowering here). if (memberName && ts.isIdentifier(memberName)) { const streamBase = (() => { for (let c = base; c; c = c.base) if (c.builtinStream) return c; @@ -2754,7 +2754,7 @@ export function collectClassShapeInner( lowerer.unsupported( "SC1090", memberName, - `declaring '_writev' on a ${streamBase.def.name.slice(1)} subclass (batched writes are supported on Writable and Duplex subclasses)`, + `declaring '_writev' on a ${streamBase.def.name.slice(1)} subclass (a Readable has no writable side)`, ); } const accepted = streamCtorShape(streamBase.def.name).accepted; diff --git a/packages/compiler/src/frontend/lowering/lower-stream.ts b/packages/compiler/src/frontend/lowering/lower-stream.ts index c37ae9074..15afb741e 100644 --- a/packages/compiler/src/frontend/lowering/lower-stream.ts +++ b/packages/compiler/src/frontend/lowering/lower-stream.ts @@ -179,15 +179,15 @@ export const UNDERSCORE_METHODS: ReadonlyMap = new Map([ /** The option callbacks applied after construction rather than through * the constructor's callback slots (option name → method name): construct - * on every stream class, and writev on the writable classes whose - * machinery hands it batches (Writable and Duplex). */ + * on every stream class, and writev on every class with a writable side + * (a Transform's batches reach writev, not the transform, as in Node). */ export const STREAM_EXTRA_CALLBACKS: ReadonlyMap = new Map([ ["construct", "_construct"], ["writev", "_writev"], ]); export function streamExtraCallbacks(cls: string): readonly string[] { - return cls === "%Writable" || cls === "%Duplex" ? ["construct", "writev"] : ["construct"]; + return cls === "%Readable" ? ["construct"] : ["construct", "writev"]; } /** The completion callback's parameter position in each underscore @@ -442,6 +442,13 @@ interface StreamOptions { * right after construction (Node's constructors call addAbortSignal * once the state exists), or null when absent. */ signal: IrExpr | null; + /** Non-literal decodeStrings and defaultEncoding values: valueDecls + * evaluates them into locals before construction (the defaultEncoding + * validated there, as the state constructors throw before the stream + * exists), and stream.optionValues applies them to the fresh stream. */ + valueDecls: IrStmt[]; + decodeStringsValue: IrExpr | null; + defaultEncodingValue: IrExpr | null; /** The objectMode option's sides (a Duplex's readableObjectMode and * writableObjectMode set one side): applied with stream.objectMode * right after construction. */ @@ -732,6 +739,12 @@ function pinAnyDataCompletion( if (!symbol || lowerer.stageSourceTypes.has(symbol)) return null; const declared = lowerer.paramShape(param).type; if (declared.kind !== "func" || declared.rest || declared.params[1]?.kind !== "dyn") return null; + // Only when every use calls the callback with a Buffer, a string or + // nothing as `data`: another value (or a callback that escapes) keeps + // `data` checked-dynamic, so the runtime pushes it as push(data) does + // and reports ERR_INVALID_ARG_TYPE through 'error', as Node does. + if (fnNode.body === undefined || !completionDataIsChunk(lowerer, fnNode.body, symbol, param.name)) + return null; const data = unionOf(lowerer, [BYTES, STRING, { kind: "undefinedT" }]); lowerer.stageSourceTypes.set( symbol, @@ -740,6 +753,59 @@ function pinAnyDataCompletion( return symbol; } +/** Whether every reference to the completion callback `symbol` in `body` + * calls it with a `data` argument that is a Buffer, a string, null or + * undefined (or with no data). */ +function completionDataIsChunk( + lowerer: Lowerer, + body: ts.Node, + symbol: ts.Symbol, + declaration: ts.Node, +): boolean { + let ok = true; + const visit = (node: ts.Node): void => { + if (!ok) return; + if ( + ts.isIdentifier(node) && + node !== declaration && + lowerer.checker.getSymbolAtLocation(node) === symbol + ) { + const call = node.parent; + if (!call || !ts.isCallExpression(call) || call.expression !== node) { + ok = false; + return; + } + const data = call.arguments.nodes[1]; + if (data !== undefined) { + let t: IrType | null; + try { + t = lowerer.mapTypeOf(lowerer.typeOf(data)); + } catch { + t = null; + } + const arms = + t?.kind === "union" ? (lowerer.unions.get(t.unionId)?.arms ?? null) : t ? [t] : null; + if ( + arms === null || + !arms.every( + (a) => + (a.kind === "bytes" && a.elem === "u8") || + a.kind === "string" || + a.kind === "undefinedT" || + a.kind === "nullT", + ) + ) { + ok = false; + return; + } + } + } + ts.forEachChild(node, visit); + }; + visit(body); + return ok; +} + /** Parses the options object literal of a stream constructor. `which` * names the callbacks this class accepts, in canonical flag order. */ function parseStreamOptions( @@ -767,6 +833,9 @@ function parseStreamOptions( badEncoding: null, pushEncoding: null, signal: null, + valueDecls: [], + decodeStringsValue: null, + defaultEncodingValue: null, readableObject: false, writableObject: false, decodeStringsOff: false, @@ -802,6 +871,21 @@ function parseStreamOptions( optsLit !== undefined && ts.isObjectLiteralExpression(optsLit) && literalWritesValues(optsLit, ctorName !== "Readable" && ctorName !== "Writable"); + // In JavaScript a decodeStrings value known only at run time may let + // string writes through, so the write and transform callbacks' chunk + // parameters read as any (the declarations type it Buffer). + const runtimeDecodeStrings = + optsLit !== undefined && + isJsSourceFile(ts.getSourceFileOfNode(optsLit)) && + ts.isObjectLiteralExpression(optsLit) && + (optsLit.properties?.nodes ?? []).some( + (p) => + ts.isPropertyAssignment(p) && + (ts.isIdentifier(p.name) || ts.isStringLiteral(p.name)) && + p.name.text === "decodeStrings" && + p.initializer.kind !== ts.SyntaxKind.TrueKeyword && + p.initializer.kind !== ts.SyntaxKind.FalseKeyword, + ); if (out.readableObject) out.hwmR = numLit(16, loc); if (out.writableObject) out.hwmW = numLit(16, loc); const tuples = cbTuples(lowerer); @@ -851,6 +935,7 @@ function parseStreamOptions( } const full = tuples[name]!.map((t) => (t === "fn" ? null : t)); const pinned = out.readableObject ? null : pinAnyDataCompletion(lowerer, name, fnNode); + let anyChunk: ts.Symbol | undefined; try { // The completion-callback tail: typed by the checker contextually; // accept the user's declared func type as the tuple's entry so the @@ -867,12 +952,22 @@ function parseStreamOptions( } return declared ?? funcOf([], VOID); }); + const chunkParam = + runtimeDecodeStrings && (name === "write" || name === "transform") + ? fnNode.parameters.nodes[0] + : undefined; + anyChunk = + chunkParam !== undefined && ts.isIdentifier(chunkParam.name) && !chunkParam.type + ? lowerer.checker.getSymbolAtLocation(chunkParam.name) + : undefined; + if (anyChunk) lowerer.anySymbols.add(anyChunk); present.set( name, lowerStreamCallback(lowerer, name, fnNode, thisType, tuple, writesValues), ); } finally { if (pinned) lowerer.stageSourceTypes.delete(pinned); + if (anyChunk) lowerer.anySymbols.delete(anyChunk); } continue; } @@ -925,11 +1020,15 @@ function parseStreamOptions( if (writableClass) out.decodeStringsOff = true; break; } - lowerer.noLowering( - `${ctorName} with a runtime 'decodeStrings' value`, - prop, - "write 'decodeStrings: true' or 'decodeStrings: false'", - ); + // a runtime value: WritableState's `options.decodeStrings !== false` + const local = lowerer.declareHiddenLocal("%decodeStrings", DYN); + out.valueDecls.push({ + kind: "varDecl", + localId: local.id, + init: lowerer.lowerExprExpecting(value, DYN), + loc, + }); + if (writableClass) out.decodeStringsValue = varRef(local.id, DYN, loc); break; } case "encoding": { @@ -973,11 +1072,33 @@ function parseStreamOptions( break; } } - lowerer.noLowering( - `the ${ctorName} option 'defaultEncoding' with a runtime value`, - prop, - "write the encoding as a string literal", - ); + { + // a runtime value: null, undefined and utf8 are the default, + // another name Buffer.isEncoding knows is kept as given, and any + // other value throws ERR_UNKNOWN_ENCODING before construction + const local = lowerer.declareHiddenLocal("%defaultEncoding", DYN); + const ref = varRef(local.id, DYN, loc); + out.valueDecls.push( + { + kind: "varDecl", + localId: local.id, + init: lowerer.lowerExprExpecting(value, DYN), + loc, + }, + { + kind: "exprStmt", + expr: { + kind: "libCall", + fn: "stream.checkEncodingOption", + args: [ref], + type: VOID, + loc, + }, + loc, + }, + ); + out.defaultEncodingValue = ref; + } break; case "signal": { // `if (options.signal) addAbortSignal(options.signal, this)`: @@ -1041,9 +1162,7 @@ function parseStreamOptions( lowerer.noLowering( `the ${ctorName} option 'writev'`, prop, - ctorName === "Readable" - ? "a Readable has no writable side" - : "batched writes on a Transform would bypass its transform — writev is supported on Writable and Duplex", + "a Readable has no writable side", ); break; default: @@ -1083,6 +1202,9 @@ function streamCtorArgs( badEncoding: string | null; pushEncoding: string | null; signal: IrExpr | null; + valueDecls: IrStmt[]; + decodeStringsValue: IrExpr | null; + defaultEncodingValue: IrExpr | null; readableObject: boolean; writableObject: boolean; decodeStringsOff: boolean; @@ -1117,6 +1239,9 @@ function streamCtorArgs( badEncoding: o.badEncoding, pushEncoding: o.pushEncoding, signal: o.signal, + valueDecls: o.valueDecls, + decodeStringsValue: o.decodeStringsValue, + defaultEncodingValue: o.defaultEncodingValue, readableObject: o.readableObject, writableObject: o.writableObject, decodeStringsOff: o.decodeStringsOff, @@ -1237,6 +1362,18 @@ export function lowerStreamNew(lowerer: Lowerer, expr: ts.NewExpression, info: C loc, }; } + const values = optionValuesArgs(parsed, loc); + if (values) { + // runtime decodeStrings and defaultEncoding values (answers the + // receiver +1) + built = { + kind: "libCall", + fn: "stream.optionValues", + args: [built, ...values], + type: thisType, + loc, + }; + } if (parsed.pushEncoding !== null) { // { defaultEncoding }: chain the push-side decode default (the // setEncoding chaining shape — answers the receiver +1). @@ -1269,8 +1406,11 @@ export function lowerStreamNew(lowerer: Lowerer, expr: ts.NewExpression, info: C loc, }; } - if (!parsed.decodeStringsOff && parsed.writeEncoding === null && parsed.extras.length === 0) - return built; + if (!parsed.decodeStringsOff && parsed.writeEncoding === null && parsed.extras.length === 0) { + return parsed.valueDecls.length > 0 + ? { kind: "seqExpr", stmts: parsed.valueDecls, result: built, type: thisType, loc } + : built; + } const fresh = lowerer.declareHiddenLocal("%stream", thisType); const followUps = streamOptionFollowUps( parsed, @@ -1280,13 +1420,30 @@ export function lowerStreamNew(lowerer: Lowerer, expr: ts.NewExpression, info: C ); return { kind: "seqExpr", - stmts: [{ kind: "varDecl", localId: fresh.id, init: built, loc }, ...followUps], + stmts: [ + ...parsed.valueDecls, + { kind: "varDecl", localId: fresh.id, init: built, loc }, + ...followUps, + ], result: varRef(fresh.id, thisType, loc), type: thisType, loc, }; } +/** stream.optionValues' (decodeStrings, defaultEncoding) arguments, or + * null when both options are literals or absent. */ +function optionValuesArgs( + parsed: { decodeStringsValue: IrExpr | null; defaultEncodingValue: IrExpr | null }, + loc: SrcLoc, +): IrExpr[] | null { + if (parsed.decodeStringsValue === null && parsed.defaultEncodingValue === null) return null; + return [ + parsed.decodeStringsValue ?? dynUndefinedExpr(loc), + parsed.defaultEncodingValue ?? dynUndefinedExpr(loc), + ]; +} + /** True when a value of Node's argument type `passed` widens implicitly * into the declared parameter type `declared` (one of its union arms, or a * sub-union of it). */ @@ -1589,6 +1746,7 @@ export function lowerStreamSuperCall( ]; } const out: IrStmt[] = [ + ...parsed.valueDecls, { kind: "exprStmt", expr: { @@ -1614,6 +1772,20 @@ export function lowerStreamSuperCall( loc, }); } + const values = optionValuesArgs(parsed, loc); + if (values) { + out.push({ + kind: "exprStmt", + expr: { + kind: "libCall", + fn: "stream.optionValues", + args: [thisRef(), ...values], + type: thisType, + loc, + }, + loc, + }); + } if (parsed.pushEncoding !== null) { out.push({ kind: "exprStmt", @@ -1699,12 +1871,20 @@ export function lowerStreamUnderscoreAssign( const clsName = cls.slice(1); if (methodName === "_construct") { // Node's constructor reads _construct once; a later assignment has no - // effect there - lowerer.noLowering( - `assigning '${methodName}' on a ${clsName}`, - expr.left, - "declare _construct on the class or pass the construct option", - ); + // effect on the stream, so only the receiver and the value are + // evaluated + const value = lowerer.lowerExpr(expr.right); + return { + kind: "exprStmt", + expr: { + kind: "seqExpr", + stmts: [{ kind: "exprStmt", expr: recv, loc }], + result: value, + type: value.type, + loc, + }, + loc, + }; } const which = methodName === "_writev" ? "writev" : optionByMethod.get(methodName)!; const { accepted } = streamCtorShape(cls); diff --git a/packages/compiler/src/ir/builtin-effects.ts b/packages/compiler/src/ir/builtin-effects.ts index d8e78be08..e786fa9a8 100644 --- a/packages/compiler/src/ir/builtin-effects.ts +++ b/packages/compiler/src/ir/builtin-effects.ts @@ -362,6 +362,7 @@ export const MAY_THROW_LIB_FNS: ReadonlySet = new Set([ "transform.newDyn", "passthrough.newDyn", "readable.initDyn", + "stream.checkEncodingOption", "writable.initDyn", "duplex.initDyn", "transform.initDyn", diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index 7cb666ccf..ed135a800 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -4103,6 +4103,11 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ * after construction; answers the receiver (+1, the chaining shape). */ "stream.objectMode": { argTypes: [null, BOOL, BOOL], result: VOID }, "stream.decodeStrings": { argTypes: [null, BOOL], result: VOID }, + /** Runtime decodeStrings and defaultEncoding option values, applied + * after construction (answers the receiver +1); the encoding was + * validated by stream.checkEncodingOption before construction. */ + "stream.optionValues": { argTypes: [null, DYN, DYN], result: VOID }, + "stream.checkEncodingOption": { argTypes: [DYN], result: VOID }, "stream.duplexFromGen": { argTypes: [null, null, null, null, null, null], result: VOID }, "stream.duplexFromDyn": { argTypes: [DYN], result: VOID }, /** Duplex.from over a lone Readable or Writable, or a { readable, diff --git a/packages/compiler/src/ir/runtime-features.ts b/packages/compiler/src/ir/runtime-features.ts index 85910369a..bd4727b95 100644 --- a/packages/compiler/src/ir/runtime-features.ts +++ b/packages/compiler/src/ir/runtime-features.ts @@ -219,6 +219,8 @@ const STREAM_LIB_FNS: ReadonlySet = new Set([ "stream.chunkDone", "stream.objectMode", "stream.decodeStrings", + "stream.optionValues", + "stream.checkEncodingOption", "stream.duplexFromGen", "stream.duplexFromDyn", "stream.duplexFromPromise", diff --git a/packages/compiler/src/ir/validate.ts b/packages/compiler/src/ir/validate.ts index 770fee6c6..1f3692c9c 100644 --- a/packages/compiler/src/ir/validate.ts +++ b/packages/compiler/src/ir/validate.ts @@ -5495,6 +5495,7 @@ function validateFunction( e.fn === "stream.destroyErr" || e.fn === "stream.objectMode" || e.fn === "stream.decodeStrings" || + e.fn === "stream.optionValues" || e.fn === "stream.duplexFromGen" || e.fn === "stream.duplexFromDyn" || e.fn === "stream.duplexifyReadable" || @@ -5867,6 +5868,7 @@ function validateFunction( e.fn === "readable.pushEncoding" || e.fn === "stream.objectMode" || e.fn === "stream.decodeStrings" || + e.fn === "stream.optionValues" || e.fn === "writable.writeEncoding" || e.fn === "writable.setDefaultEncoding" || e.fn === "stream.destroy" || diff --git a/packages/compiler/test/native-codegen-integration.test.ts b/packages/compiler/test/native-codegen-integration.test.ts index 0fd5249d3..3a1f8fdee 100644 --- a/packages/compiler/test/native-codegen-integration.test.ts +++ b/packages/compiler/test/native-codegen-integration.test.ts @@ -378,7 +378,7 @@ done: expect(await symbols(helperObject, ["-u"])).toEqual(await symbols(clangObject, ["-u"])); expect(await symbols(helperObject, ["-gU"])).toEqual(await symbols(clangObject, ["-gU"])); expect(await symbols(helperObject, ["-gU"])).toEqual(["0000000000000000 T _main"]); - expect(await symbols(helperObject, ["-u"])).toContain("_scr_runtime_abi_v33"); + expect(await symbols(helperObject, ["-u"])).toContain("_scr_runtime_abi_v34"); const clangExe = join(dir, "clang-program"); const linkDriver = join(dir, "clang-driver.c"); @@ -712,7 +712,7 @@ done: .split("\n"); expect(undefinedSymbols).toContain("_sf_scale"); expect(undefinedSymbols).toContain("_sf_callback_mix"); - expect(undefinedSymbols).toContain("_scr_runtime_abi_v33"); + expect(undefinedSymbols).toContain("_scr_runtime_abi_v34"); }); test("helper and clang object paths retain the same outbound FFI C ABI", async () => { diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7460-transform-writev.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7460-transform-writev.mjs.json new file mode 100644 index 000000000..1015e1354 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7460-transform-writev.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7460-transform-writev.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7461-transform-writev.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7461-transform-writev.cjs.json new file mode 100644 index 000000000..4d3c86215 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7461-transform-writev.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7461-transform-writev.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7462-transform-writev.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7462-transform-writev.ts.json new file mode 100644 index 000000000..9b42f2d60 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7462-transform-writev.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7462-transform-writev.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7463-stream-option-values.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7463-stream-option-values.ts.json new file mode 100644 index 000000000..5aa543dea --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7463-stream-option-values.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7463-stream-option-values.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7464-stream-option-values.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7464-stream-option-values.mjs.json new file mode 100644 index 000000000..2a438d418 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7464-stream-option-values.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7464-stream-option-values.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7465-transform-callback-values.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7465-transform-callback-values.ts.json new file mode 100644 index 000000000..17633d208 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7465-transform-callback-values.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7465-transform-callback-values.ts" + ], + "diags": [] +} diff --git a/packages/runtime-darwin-arm64/scripts/build.mjs b/packages/runtime-darwin-arm64/scripts/build.mjs index e17312713..9f856b376 100644 --- a/packages/runtime-darwin-arm64/scripts/build.mjs +++ b/packages/runtime-darwin-arm64/scripts/build.mjs @@ -340,7 +340,7 @@ async function build() { package: packageManifest.name, version: packageManifest.version, target: RUNTIME_PACK_MATRIX.target, - runtime_abi: { version: 33, marker: "scr_runtime_abi_v33" }, + runtime_abi: { version: 34, marker: "scr_runtime_abi_v34" }, compiler: { command: compiler, identity: compilerVersion, diff --git a/packages/runtime-pack-common/scripts/build.mjs b/packages/runtime-pack-common/scripts/build.mjs index 60ef9dc49..50da4a78e 100644 --- a/packages/runtime-pack-common/scripts/build.mjs +++ b/packages/runtime-pack-common/scripts/build.mjs @@ -437,7 +437,7 @@ async function build() { package: packageManifest.name, version: packageManifest.version, target: matrix.target, - runtime_abi: { version: 33, marker: "scr_runtime_abi_v33" }, + runtime_abi: { version: 34, marker: "scr_runtime_abi_v34" }, compiler: { command: compiler, identity: compilerVersion, diff --git a/packages/runtime/src/scr_console.c b/packages/runtime/src/scr_console.c index f8dd9a0c0..193352424 100644 --- a/packages/runtime/src/scr_console.c +++ b/packages/runtime/src/scr_console.c @@ -148,7 +148,7 @@ static void scr_enable_utf8_console_output(void) { } #endif -void scr_runtime_abi_v33(void) {} +void scr_runtime_abi_v34(void) {} void scr_init(void) { #ifdef SCR_WORKERS diff --git a/packages/runtime/src/scr_core.h b/packages/runtime/src/scr_core.h index d716f0de3..1b5e28545 100644 --- a/packages/runtime/src/scr_core.h +++ b/packages/runtime/src/scr_core.h @@ -81,7 +81,7 @@ void scr_init(void); /* Program objects emitted by the bundled LLVM helper reference this symbol. * Its versioned spelling makes a mismatched manual runtime link fail before * the program can start. */ -void scr_runtime_abi_v33(void); +void scr_runtime_abi_v34(void); void scr_runtime_workers_v8(void); /* ── the trap funnel (scr_console.c; scr_library.c under -DSCR_LIB) ────── diff --git a/packages/runtime/src/scr_dyn_builtin_stream.c b/packages/runtime/src/scr_dyn_builtin_stream.c index 926f85eea..2875cfa06 100644 --- a/packages/runtime/src/scr_dyn_builtin_stream.c +++ b/packages/runtime/src/scr_dyn_builtin_stream.c @@ -34,7 +34,7 @@ * receiver without a reference; the hook closures outlive the record. * * Not supported here (each throws the "not supported yet" Error where it - * runs): the options scr_stream.c's dyn option walk refuses, `_writev` and `_construct`, symbol event names, listeners() / + * runs): the options scr_stream.c's dyn option walk refuses, symbol event names, listeners() / * rawListeners(), `wrap()`, and pipes into values that are neither runtime * streams nor constructor-function streams. */ #include "scr_runtime.h" @@ -898,20 +898,15 @@ static ScrDyn *bcs_construct(int kind, ScrDyn *self, ScrDyn *const *args, size_t } bool failed = false; bool writes = kind != BCS_READABLE; - bool transformish = kind == BCS_TRANSFORM || kind == BCS_PASSTHROUGH; ScrDyn *opts = bcs_arg(args, argc, 0); /* Node's constructors assign options.writev and options.construct to * the instance (this._writev = options.writev), then read _writev and * _construct, so an option shadows the prototype's method. */ if (opts->kind == SCR_DYN_OBJ) { - if (writes && !transformish && !bcs_copy_option(self, opts, "writev", "_writev")) return NULL; + if (writes && !bcs_copy_option(self, opts, "writev", "_writev")) return NULL; if (!bcs_copy_option(self, opts, "construct", "_construct")) return NULL; } bool has_writev = writes && bcs_is_function(self, "_writev", &failed); - if (has_writev && transformish) { - bcs_unsupported("_writev on a constructor-function Transform is not supported yet"); - return NULL; - } bool has_construct = !failed && bcs_is_function(self, "_construct", &failed); if (failed) return NULL; ScrStream *s = kind == BCS_READABLE ? scr_stream_new_readable_dyn(opts) diff --git a/packages/runtime/src/scr_stream.c b/packages/runtime/src/scr_stream.c index 275c03039..4b962675a 100644 --- a/packages/runtime/src/scr_stream.c +++ b/packages/runtime/src/scr_stream.c @@ -2157,6 +2157,24 @@ static ScrDyn *scr_stream_wentry_encoding(const ScrStreamState *st, ScrDyn *enc) return scr_dyn_retain(enc ? enc : scr_dyn_undefined()); } +/* A compiled writev whose entries declare `encoding` a string: an entry + * without one (an object-mode write passes undefined or null) gets "". + * The batch array is the one this call receives. */ +void scr_stream_writev_fill_encodings(ScrDyn *chunks) { + if (!chunks || chunks->kind != SCR_DYN_ARR) return; + double n = scr_dyn_arr_len(chunks); + for (double i = 0; i < n; i++) { + ScrDyn *entry = scr_dyn_arr_at(chunks, i); /* +1 */ + ScrDyn *enc = entry->kind == SCR_DYN_OBJ ? scr_dyn_obj_get(entry, "encoding", 8) : NULL; + if (entry->kind == SCR_DYN_OBJ && (!enc || enc->kind == SCR_DYN_UNDEF || enc->kind == SCR_DYN_NULL)) { + ScrStr *empty = scr_str_new("", 0); + scr_dyn_obj_set(entry, "encoding", 8, scr_dyn_new_str(empty)); + scr_str_release(empty); + } + scr_dyn_release(entry); + } +} + /* A batch entry's callback: the write's callback, or Node's nop. */ static ScrDyn *scr_stream_writev_entry_cb_call(ScrClosure *clo, ScrDyn *const *args, size_t argc) { (void)args; @@ -2675,9 +2693,8 @@ void *scr_stream_native(ScrStream *s, void (*release)(void *)) { * way the compiler-emitted dyn thunks do (chunk as Buffer-flavored bytes, * encoding 'buffer', the error via the boundary encoding, the completion * callback as a callable dyn over the *_done glue). Options that Node's - * base WOULD consume but have no lowering here (objectMode truthy, - * decodeStrings:false, non-utf8 defaultEncoding, signal, construct, - * writev, and read/write/final on the transformish bases) throw the + * base WOULD consume but have no lowering here (signal, and + * read/write/final on the transformish bases) throw the * "not supported yet" Error — the compile-time fence's runtime twin. */ /* The boxed dyn callable (+1 — scr_box_get_ref retains; callers release @@ -2903,11 +2920,8 @@ static bool scr_stream_parse_dyn_opts(const ScrDyn *opts, char base, ScrStreamDy } } out->construct = scr_stream_dynopt_fn(opts, "construct", 9); + /* a Transform's batches reach writev, not the transform, as in Node */ if (base != 'r') out->writev = scr_stream_dynopt_fn(opts, "writev", 6); - if (transformish && out->writev != NULL) { - static const char m[] = "the stream option 'writev' on a Transform is not supported yet (batches would bypass the transform)"; - return scr_stream_dynopt_unsupported(m, sizeof m - 1); - } { /* a known encoding or ERR_UNKNOWN_ENCODING (the states' constructors) */ ScrDyn *v = scr_dyn_obj_get((ScrDyn *)opts, "defaultEncoding", 15); @@ -6296,6 +6310,55 @@ ScrStream *scr_stream_set_default_encoding(ScrStream *s, ScrDyn *enc) { return scr_stream_retain(s); } +/* Whether a defaultEncoding option value is the default: null, undefined, + * 'utf8' or 'utf-8' (the state constructors' kDefaultUTF8Encoding). */ +static bool scr_stream_default_encoding_value(const ScrDyn *v) { + if (!v || v->kind == SCR_DYN_UNDEF || v->kind == SCR_DYN_NULL) return true; + return v->kind == SCR_DYN_STR && ((v->v.str->len == 4 && memcmp(v->v.str->data, "utf8", 4) == 0) || + (v->v.str->len == 5 && memcmp(v->v.str->data, "utf-8", 5) == 0)); +} + +/* A runtime defaultEncoding option value, checked before construction as + * the state constructors check it: the default, a name Buffer.isEncoding + * knows, or ERR_UNKNOWN_ENCODING (thrown). Borrows v. */ +void scr_stream_check_encoding_option(ScrDyn *v) { + if (scr_stream_default_encoding_value(v)) return; + if (v->kind == SCR_DYN_STR) { + ScrStr *canonical = scr_bytes_normalize_encoding(v->v.str); + if (canonical) scr_str_release(canonical); + return; + } + ScrStr *shown = scr_dyn_string_coerce_js(v); + if (!shown) return; + char msg[128]; + int len = snprintf(msg, sizeof msg, "Unknown encoding: %.*s", (int)(shown->len < 64 ? shown->len : 64), + shown->data); + scr_str_release(shown); + scr_throw_error_msg_code(SCR_ERR_TYPE, msg, (size_t)(len < 0 ? 0 : len), "ERR_UNKNOWN_ENCODING"); +} + +/* Runtime decodeStrings and defaultEncoding option values over a fresh + * stream (after its objectMode): decodeStrings === false keeps string + * writes, and a known defaultEncoding is the write side's default as + * given and how a byte-mode readable side decodes push(string). Borrows + * both; answers the receiver (+1). */ +ScrStream *scr_stream_option_values(ScrStream *s, ScrDyn *decode_strings, ScrDyn *default_encoding) { + ScrStreamState *st = s->st; + if (st->has_w && decode_strings && decode_strings->kind == SCR_DYN_BOOL && !decode_strings->v.b) + scr_stream_set_decode_strings(s, false); + if (!scr_stream_default_encoding_value(default_encoding) && default_encoding->kind == SCR_DYN_STR) { + ScrStr *canonical = scr_bytes_normalize_encoding(default_encoding->v.str); + if (canonical) { + scr_stream_set_write_encoding(s, default_encoding->v.str); + if (st->has_r && !st->r.object_entries) scr_stream_release(scr_stream_set_push_encoding(s, canonical)); + scr_str_release(canonical); + } else if (scr_exc_pending()) { + scr_exc_clear(); /* checked before construction */ + } + } + return scr_stream_retain(s); +} + /* stream.objectMode: scr_stream_set_object_mode, answering the receiver * (+1). */ ScrStream *scr_stream_object_mode(ScrStream *s, bool readable, bool writable) { diff --git a/packages/runtime/src/scr_stream.h b/packages/runtime/src/scr_stream.h index d26077b5c..e368ec7ff 100644 --- a/packages/runtime/src/scr_stream.h +++ b/packages/runtime/src/scr_stream.h @@ -563,6 +563,9 @@ void scr_stream_set_writev_concat(ScrStream *s); void scr_stream_set_construct(ScrStream *s, ScrClosure *cb, ScrStreamPlainInv inv); void scr_stream_set_writev(ScrStream *s, ScrClosure *cb, ScrStreamWritevInv inv); void scr_stream_set_writev_default(ScrStream *s, ScrClosure *cb, ScrStreamWritevInv inv); +/* A compiled writev whose entries declare `encoding` a string: entries + * without an encoding (object mode) get "" before the conversion. */ +void scr_stream_writev_fill_encodings(struct ScrDyn *chunks); /* Duplex.from (duplexify.js): over an async generator function (the * compiled stage takes the source generator that the compiled source * makes from next() and ack(); all borrowed, helpers too), and over a @@ -605,6 +608,11 @@ ScrStream *scr_stream_duplex_from_promise(struct ScrPromise *p, ScrClosure *box) * forms answer the stream (+1). */ void scr_stream_set_decode_strings(ScrStream *s, bool decode); ScrStream *scr_stream_decode_strings(ScrStream *s, bool decode); +/* Runtime decodeStrings/defaultEncoding option values: checked before + * construction (ERR_UNKNOWN_ENCODING thrown), then applied to the fresh + * stream, answering it (+1). Values borrowed. */ +void scr_stream_check_encoding_option(struct ScrDyn *v); +ScrStream *scr_stream_option_values(ScrStream *s, struct ScrDyn *decode_strings, struct ScrDyn *default_encoding); void scr_stream_set_write_encoding(ScrStream *s, ScrStr *enc); ScrStream *scr_stream_write_encoding(ScrStream *s, ScrStr *enc); ScrStream *scr_stream_set_default_encoding(ScrStream *s, struct ScrDyn *enc); diff --git a/tests/corpus/7460-transform-writev.mjs b/tests/corpus/7460-transform-writev.mjs new file mode 100644 index 000000000..d78bdfc77 --- /dev/null +++ b/tests/corpus/7460-transform-writev.mjs @@ -0,0 +1,33 @@ +// writev on a Transform and a PassThrough (the writev option): a batch of +// buffered writes reaches writev instead of the transform, a lone write +// still reaches the transform, and the readable side gets what writev +// pushes. +import { Transform, PassThrough } from "node:stream"; +const t = new Transform({ + transform(chunk, enc, cb) { console.log("transform", String(chunk), enc); cb(null, chunk); }, + writev(chunks, cb) { + console.log("writev", chunks.map((c) => `${c.chunk}/${c.encoding}`).join(",")); + for (const { chunk } of chunks) this.push(String(chunk).toUpperCase()); + cb(); + }, +}); +t.on("data", (d) => console.log("data", String(d))); +t.on("finish", () => console.log("finish")); +t.on("end", () => console.log("end")); +t.write("a"); +t.cork(); +t.write("b"); +t.write("c", "latin1"); +process.nextTick(() => { t.uncork(); t.end("d"); }); + +const p = new PassThrough({ + objectMode: true, + writev(chunks, cb) { console.log("pt writev", JSON.stringify(chunks.map((c) => c.chunk))); cb(); }, +}); +p.on("data", (d) => console.log("pt data", JSON.stringify(d))); +p.cork(); +p.write({ x: 1 }); +p.write([2]); +p.uncork(); +p.write("solo"); +p.end(); diff --git a/tests/corpus/7461-transform-writev.cjs b/tests/corpus/7461-transform-writev.cjs new file mode 100644 index 000000000..dd45cf09d --- /dev/null +++ b/tests/corpus/7461-transform-writev.cjs @@ -0,0 +1,33 @@ +// Constructor-function Transform and PassThrough streams with _writev: +// the prototype's _writev takes the batches, and a writev option shadows +// it with the stream as this. +'use strict'; +const { Transform, PassThrough } = require('node:stream'); +function T(opts) { Transform.call(this, opts); } +Object.setPrototypeOf(T.prototype, Transform.prototype); +Object.setPrototypeOf(T, Transform); +T.prototype._transform = function (chunk, enc, cb) { cb(null, 'T(' + chunk + ')'); }; +T.prototype._writev = function (chunks, cb) { + this.push('V(' + chunks.map((c) => String(c.chunk)).join('+') + ')'); + cb(); +}; +const t = Object.create(T.prototype); +T.call(t, {}); +t.on('data', (d) => console.log('t data', String(d))); +t.on('end', () => console.log('t end')); +t.write('a'); +t.cork(); +t.write('b'); +t.write('c'); +process.nextTick(() => { t.uncork(); t.end(); }); +function P(opts) { PassThrough.call(this, opts); } +Object.setPrototypeOf(P.prototype, PassThrough.prototype); +Object.setPrototypeOf(P, PassThrough); +const p = Object.create(P.prototype); +P.call(p, { writev(chunks, cb) { console.log('p writev', chunks.length, this === p); cb(); } }); +p.on('data', (d) => console.log('p data', String(d))); +p.cork(); +p.write('x'); +p.write('y'); +p.uncork(); +p.write('z'); diff --git a/tests/corpus/7462-transform-writev.ts b/tests/corpus/7462-transform-writev.ts new file mode 100644 index 000000000..ca079b3c9 --- /dev/null +++ b/tests/corpus/7462-transform-writev.ts @@ -0,0 +1,109 @@ +// Typed writev on Transform and PassThrough: an options writev with +// decodeStrings: false and flush, a Transform subclass's _writev, a +// batch completing with an error, and object-mode entries whose +// encoding is declared as a string or as Node's string | null | +// undefined. +import { Transform, PassThrough, Writable } from "node:stream"; +type WritevChunks = Array<{ chunk: any; encoding: BufferEncoding }>; +type Cb = (error?: Error | null) => void; +type TransformCallback = (error?: Error | null, data?: any) => void; +const log: string[] = []; +const t = new Transform({ + decodeStrings: false, + transform(chunk: string, enc: BufferEncoding, cb: TransformCallback): void { + log.push(`transform ${chunk} ${enc}`); + cb(null, chunk); + }, + flush(cb: TransformCallback): void { + log.push("flush"); + cb(null, "!"); + }, + writev(chunks: WritevChunks, cb: Cb): void { + log.push(`writev ${chunks.map((c) => `${c.chunk}/${c.encoding}`).join(",")}`); + this.push(chunks.map((c) => String(c.chunk)).join("")); + cb(); + }, +}); +t.on("data", (d: Buffer) => log.push(`data ${d.toString()}`)); +t.on("end", () => log.push("end")); +t.on("finish", () => log.push("finish")); +t.on("close", () => { + console.log(log.join("\n")); + next(); +}); +t.write("one"); +t.cork(); +t.write("two", "latin1"); +t.write("three"); +t.uncork(); +t.end("four"); + +class Batcher extends Transform { + _transform(chunk: Buffer, _enc: BufferEncoding, cb: TransformCallback): void { + console.log("batcher transform", chunk.toString()); + cb(null, chunk); + } + _writev(chunks: Array<{ chunk: any; encoding: BufferEncoding }>, cb: Cb): void { + console.log("batcher writev", chunks.length); + this.push(Buffer.concat(chunks.map((c) => c.chunk as Buffer))); + cb(); + } +} + +function next(): void { + const b = new Batcher(); + b.on("data", (d: Buffer) => console.log("batcher data", d.toString())); + b.write("x"); + b.cork(); + b.write("y"); + b.write("z"); + process.nextTick(() => { + b.uncork(); + b.end(); + }); + b.on("finish", () => { + const p = new PassThrough({ + writev(chunks: WritevChunks, cb: Cb): void { + console.log("pt writev", chunks.length, String(chunks[0]!.chunk)); + cb(new Error("batch failed")); + }, + }); + p.on("error", (e: Error) => { + console.log("pt error", e.message); + objectEntries(); + }); + p.cork(); + p.write("1"); + p.write("2"); + p.uncork(); + }); +} + +function objectEntries(): void { + const declared = new Writable({ + objectMode: true, + write(_chunk: any, _enc: BufferEncoding, cb: Cb): void { cb(); }, + writev(chunks: WritevChunks, cb: Cb): void { + console.log("string encodings", JSON.stringify(chunks.map((c) => c.chunk))); + cb(); + }, + }); + declared.write({ n: 0 }); + declared.cork(); + declared.write({ n: 1 }); + declared.write({ n: 2 }, () => {}); + declared.uncork(); + const nodeTyped = new Writable({ + objectMode: true, + write(_chunk: any, _enc: BufferEncoding, cb: Cb): void { cb(); }, + writev(chunks: Array<{ chunk: any; encoding: string | null | undefined }>, cb: Cb): void { + console.log("node encodings", JSON.stringify(chunks.map((c) => [c.chunk, c.encoding === undefined ? "undefined" : c.encoding]))); + cb(); + }, + }); + nodeTyped.write({ n: 0 }); + nodeTyped.cork(); + nodeTyped.write({ n: 1 }); + nodeTyped.write({ n: 2 }, () => {}); + nodeTyped.uncork(); +} diff --git a/tests/corpus/7463-stream-option-values.ts b/tests/corpus/7463-stream-option-values.ts new file mode 100644 index 000000000..04f497b6b --- /dev/null +++ b/tests/corpus/7463-stream-option-values.ts @@ -0,0 +1,58 @@ +// Typed decodeStrings and defaultEncoding options whose values are known +// only at run time: decodeStrings === false keeps string writes, a known +// defaultEncoding is the write default as given and how push(string) +// decodes, null and utf8 are the default, an unknown name throws +// ERR_UNKNOWN_ENCODING before construction, and a subclass's super() +// options take them the same way. +import { Writable, Readable, Duplex } from "node:stream"; +type Cb = (error?: Error | null) => void; +const flags: boolean[] = JSON.parse("[false, true]"); +const encodings: Array = JSON.parse('["hex", "latin1", null, "UTF8"]'); +for (const decode of flags) { + const w = new Writable({ + decodeStrings: decode, + defaultEncoding: encodings[0] as BufferEncoding, + write(chunk: any, enc: string, cb: Cb): void { + console.log("write", decode, typeof chunk, enc, String(chunk)); + cb(); + }, + }); + w.write("6869"); +} +const r = new Readable({ defaultEncoding: encodings[1] as BufferEncoding, read(): void {} }); +r.push("é"); +r.push(null); +r.on("data", (d: Buffer) => console.log("readable", d.toString("hex"))); +const dflt = new Writable({ + defaultEncoding: encodings[2] as any, + write(chunk: Buffer, enc: string, cb: Cb): void { + console.log("null default", enc, chunk.toString()); + cb(); + }, +}); +dflt.write("ok"); +const upper = new Duplex({ + defaultEncoding: encodings[3] as BufferEncoding, + decodeStrings: flags[0], + read(): void {}, + write(chunk: any, enc: string, cb: Cb): void { + console.log("duplex", typeof chunk, enc); + cb(); + }, +}); +upper.write("x"); +try { + new Writable({ defaultEncoding: JSON.parse('"nope"') }); +} catch (e) { + console.log("bad", (e as Error & { code?: string }).code, (e as Error).message); +} +class Sink extends Writable { + constructor(decode: boolean, encoding: BufferEncoding) { + super({ decodeStrings: decode, defaultEncoding: encoding }); + } + _write(chunk: any, enc: string, cb: Cb): void { + console.log("sink", typeof chunk, enc); + cb(); + } +} +new Sink(flags[0]!, encodings[1] as BufferEncoding).write("y"); diff --git a/tests/corpus/7464-stream-option-values.mjs b/tests/corpus/7464-stream-option-values.mjs new file mode 100644 index 000000000..dfeb7a447 --- /dev/null +++ b/tests/corpus/7464-stream-option-values.mjs @@ -0,0 +1,28 @@ +// Option values known only at run time in JavaScript: decodeStrings and +// defaultEncoding read from an object, a write callback whose chunk is +// then a string, decodeStrings: 0 (only false turns decoding off), a +// defaultEncoding that is not a string, and a _construct assigned after +// construction, which Node's constructor never reads. +import { Writable, Readable } from "node:stream"; +const opts = { decode: false, enc: "base64" }; +const w = new Writable({ + decodeStrings: opts.decode, + defaultEncoding: opts.enc, + write(c, e, cb) { console.log("js", typeof c, e, String(c)); cb(); }, +}); +w.write("aGk="); +try { new Writable({ defaultEncoding: 5 }); } catch (e) { console.log("bad", e.code, e.message); } +const zero = new Writable({ decodeStrings: 0, write(c, e, cb) { console.log("zero", Buffer.isBuffer(c), e); cb(); } }); +zero.write("z"); +const late = new Writable({ write(chunk, enc, cb) { console.log("write", chunk.toString()); cb(); } }); +let ran = false; +late._construct = (cb) => { ran = true; cb(); }; +late.write("a"); +late.end(() => console.log("ended, construct ran:", ran)); +const r = new Readable({ read() {} }); +const order = []; +(order.push("receiver"), r)._construct = (order.push("value"), function (cb) { console.log("never"); cb(); }); +console.log(order.join(",")); +r.push("x"); +r.push(null); +r.on("data", (d) => console.log("data", d.toString())); diff --git a/tests/corpus/7465-transform-callback-values.ts b/tests/corpus/7465-transform-callback-values.ts new file mode 100644 index 000000000..e7003e894 --- /dev/null +++ b/tests/corpus/7465-transform-callback-values.ts @@ -0,0 +1,41 @@ +// A transform or flush callback passing a value that is not a Buffer or a +// string: the push reports Node's ERR_INVALID_ARG_TYPE through 'error' +// and destroys the stream, for option callbacks and _transform methods, +// while Buffer and string values take the typed path. +import { Transform } from "node:stream"; +type TransformCallback = (error?: Error | null, data?: any) => void; +const t = new Transform({ + transform(chunk: Buffer, _enc: BufferEncoding, cb: TransformCallback): void { + console.log("transform", chunk.toString()); + cb(null, 42); + }, +}); +t.on("error", (e: Error & { code?: string }) => console.log("error", e.name, e.code, e.message)); +t.on("data", (d: Buffer) => console.log("data", d.toString())); +t.on("close", () => console.log("close", t.destroyed)); +console.log("write", t.write("a")); +const f = new Transform({ + transform(chunk: Buffer, _enc: BufferEncoding, cb: TransformCallback): void { cb(null, chunk); }, + flush(cb: TransformCallback): void { cb(null, { tail: true }); }, +}); +f.on("error", (e: Error & { code?: string }) => console.log("flush error", e.code, e.message)); +f.on("data", (d: Buffer) => console.log("flush data", d.toString())); +f.on("close", () => console.log("flush close")); +f.end("z"); +class Bad extends Transform { + _transform(chunk: Buffer, _enc: BufferEncoding, cb: TransformCallback): void { + cb(null, chunk.length); + } +} +const bad = new Bad(); +bad.on("error", (e: Error & { code?: string }) => console.log("error", e.code, e.message)); +bad.on("close", () => console.log("close")); +bad.write("abc"); +class Good extends Transform { + _transform(chunk: Buffer, _enc: BufferEncoding, cb: TransformCallback): void { + cb(null, chunk.toString().toUpperCase()); + } +} +const good = new Good(); +good.on("data", (d: Buffer) => console.log("good", d.toString())); +good.end("xyz"); diff --git a/tests/harness/corpus-costs/7460-transform-writev.mjs.json b/tests/harness/corpus-costs/7460-transform-writev.mjs.json new file mode 100644 index 000000000..f6c21fd3d --- /dev/null +++ b/tests/harness/corpus-costs/7460-transform-writev.mjs.json @@ -0,0 +1 @@ +{ "plain": 2266, "san": 2991 } diff --git a/tests/harness/corpus-costs/7461-transform-writev.cjs.json b/tests/harness/corpus-costs/7461-transform-writev.cjs.json new file mode 100644 index 000000000..b5868c90b --- /dev/null +++ b/tests/harness/corpus-costs/7461-transform-writev.cjs.json @@ -0,0 +1 @@ +{ "plain": 1828, "san": 2413 } diff --git a/tests/harness/corpus-costs/7462-transform-writev.ts.json b/tests/harness/corpus-costs/7462-transform-writev.ts.json new file mode 100644 index 000000000..f8fd66c2a --- /dev/null +++ b/tests/harness/corpus-costs/7462-transform-writev.ts.json @@ -0,0 +1 @@ +{ "plain": 2178, "san": 2875 } diff --git a/tests/harness/corpus-costs/7463-stream-option-values.ts.json b/tests/harness/corpus-costs/7463-stream-option-values.ts.json new file mode 100644 index 000000000..0f66a4266 --- /dev/null +++ b/tests/harness/corpus-costs/7463-stream-option-values.ts.json @@ -0,0 +1 @@ +{ "plain": 1786, "san": 4192 } diff --git a/tests/harness/corpus-costs/7464-stream-option-values.mjs.json b/tests/harness/corpus-costs/7464-stream-option-values.mjs.json new file mode 100644 index 000000000..b609b128f --- /dev/null +++ b/tests/harness/corpus-costs/7464-stream-option-values.mjs.json @@ -0,0 +1 @@ +{ "plain": 1678, "san": 4502 } diff --git a/tests/harness/corpus-costs/7465-transform-callback-values.ts.json b/tests/harness/corpus-costs/7465-transform-callback-values.ts.json new file mode 100644 index 000000000..1c67bb43f --- /dev/null +++ b/tests/harness/corpus-costs/7465-transform-callback-values.ts.json @@ -0,0 +1 @@ +{ "plain": 1571, "san": 2074 } diff --git a/tests/harness/self-hosting-native-toolchain.test.ts b/tests/harness/self-hosting-native-toolchain.test.ts index 4d5546ab6..669a3a0d6 100644 --- a/tests/harness/self-hosting-native-toolchain.test.ts +++ b/tests/harness/self-hosting-native-toolchain.test.ts @@ -108,7 +108,7 @@ for (const backend of ["llvm"] as const) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 33, marker: "scr_runtime_abi_v33" }, + runtime_abi: { version: 34, marker: "scr_runtime_abi_v34" }, compiler: { command: "fixture", identity: "fixture", target: target.llvmTriple }, macros: { executable: [], excluded: [], sanitizer: "external-toolchain-required" }, flavors: { From 2ebb89b0ef90150b2f5015732b31fe64c90c9908 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 01:24:41 -0500 Subject: [PATCH 06/53] Store number arrays of proven int32s in 4-byte slots across direct calls An int32 array candidate can now be a module global or a parameter, not only a local. A number[] parameter qualifies when its function is reached by direct calls alone (not as a value, a method or a constructor, and not through a result-slot entry the backend calls in its place), it is never written, and every call passes a candidate at its position; a local, global or parameter may then be passed there as well. A global qualifies when every function reads it only in candidate positions and every write stores a fresh array. moduleInt32ArrayCandidates finds them by dropping, pass after pass, the ones a function or call disqualifies. The candidates one array can reach through calls form a group (int32ArrayGroups): the int32 slot analysis keeps or drops a group as one, since a store through any member writes the same array. A module with a module namespace object keeps its globals in doubles, as member reads can reach any global by name. --- packages/compiler/src/backend/llvm/emitter.ts | 22 +- packages/compiler/src/ir/int32-arrays.ts | 361 ++++++++++++++++-- packages/compiler/src/ir/int32-slots.ts | 102 +++-- .../test/int32-element-array-emission.test.ts | 39 ++ .../7580-int32-arrays-across-calls.ts.json | 6 + .../corpus/7580-int32-arrays-across-calls.ts | 146 +++++++ .../7580-int32-arrays-across-calls.ts.json | 1 + 7 files changed, 608 insertions(+), 69 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7580-int32-arrays-across-calls.ts.json create mode 100644 tests/corpus/7580-int32-arrays-across-calls.ts create mode 100644 tests/harness/corpus-costs/7580-int32-arrays-across-calls.ts.json diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index c0cc9fef2..bfa02dfca 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -68,7 +68,7 @@ import { type StringSplit, } from "./split-loops.js"; import { scalarizeNumericRecords } from "../../ir/scalar-records.js"; -import { addRecordResultEntries } from "./record-results.js"; +import { addRecordResultEntries, resultEntryName } from "./record-results.js"; import { specializeCallbackCalls } from "../../ir/callback-specialization.js"; import { type ChunkResult, @@ -82,6 +82,7 @@ import { versionLoopBoundParameters } from "../../ir/parameter-versions.js"; import { allocatedLiteral, fillAllocationLength, + int32ArrayKey, selectInt32Arrays, } from "../../ir/int32-arrays.js"; import { everyExpr, everyExprChild, everyStmtChild, everyStmtList } from "../../ir/traverse.js"; @@ -509,6 +510,15 @@ function provenInt32Arrays(mod: IrModule): ReadonlySet { return new Set([...slots.int32Arrays, ...slots.checkedInt32Arrays]); } +/** The functions whose calls the backend may emit as calls of their + * result-slot entries (record-results.ts). */ +function resultEntryTargets(mod: IrModule): ReadonlySet { + const names = new Set(mod.functions.map((fn) => fn.name)); + const targets = new Set(); + for (const name of names) if (names.has(resultEntryName(name))) targets.add(name); + return targets; +} + /** The emitter for `input` after the module passes every emission runs. */ function moduleEmitter(input: IrModule, options: LlvmTargetOptions): LlEmitter { // Facts are keyed by node identity: emit the same module whether it was @@ -544,6 +554,7 @@ function moduleEmitter(input: IrModule, options: LlvmTargetOptions): LlEmitter { ), ), provenInt32Arrays, + resultEntryTargets, ) : mod, options, @@ -780,7 +791,7 @@ export class LlEmitter { everyStmtList(fn.body, { stmt: (s) => { if (s.kind !== "varDecl" && s.kind !== "assign") return true; - const key = `${fn.name}\0${s.localId}`; + const key = int32ArrayKey(fn.name, s.localId); const checked = checkedInt32Arrays.has(key); if (!checked && !int32Arrays.has(key)) return true; const value = s.kind === "varDecl" ? s.init : s.value; @@ -798,15 +809,16 @@ export class LlEmitter { }); } - /** Whether `arr` is one of the current function's int32 arrays - * (ir/int32-arrays.ts, kept by the int32 slot analysis). */ + /** Whether `arr` reads one of the int32 arrays (ir/int32-arrays.ts, kept + * by the int32 slot analysis): a local or parameter of the current + * function, or a module global. */ int32Array(arr: IrExpr): boolean { if (arr.kind !== "varRef") return false; if (this.int32KindArray(arr)) return true; if (this.int32Slots.int32Arrays.size === 0) return false; return ( this.int32ArrayTemps.has(arr.localId) || - this.int32Slots.int32Arrays.has(`${this.currentFunctionName}\0${arr.localId}`) + this.int32Slots.int32Arrays.has(int32ArrayKey(this.currentFunctionName, arr.localId)) ); } diff --git a/packages/compiler/src/ir/int32-arrays.ts b/packages/compiler/src/ir/int32-arrays.ts index 02de7268f..0872fcddb 100644 --- a/packages/compiler/src/ir/int32-arrays.ts +++ b/packages/compiler/src/ir/int32-arrays.ts @@ -1,3 +1,4 @@ +import { exprFunctionReference, moduleFunctionReferences } from "./function-references.js"; import { F64, type IrExpr, type IrFunction, type IrModule, type IrStmt } from "./ir.js"; type SwitchCase = Extract["cases"][number]; @@ -12,16 +13,21 @@ import { type IrTransform, } from "./traverse.js"; -/** Local number arrays the backend may store as int32 slots. +/** Number arrays the backend may store as int32 slots. * - * A candidate is a local `number[]` of a synchronous function that no - * other binding can hold and only compiled element operations reach: - * every write of the local stores a fresh array (an array literal, or - * `new Array(n).fill(v)`), and every read of it is the receiver of an - * element read, write, state test, `length`, `push` or `pop` (a discarded - * `pop` included), or the array of an element-reader helper call. The - * array never reaches a runtime function that reads its slots, a library - * call, a closure, a return or untyped code. + * A candidate is a `number[]` binding that only compiled element + * operations reach: a local of a synchronous function, a module global, or + * a parameter. Every write of a local or a global stores a fresh array (an + * array literal, or `new Array(n).fill(v)`); a parameter is never written. + * Every read of a candidate is the receiver of an element read, write, + * state test, `length`, `push` or `pop` (a discarded `pop` included), the + * array of an element-reader helper call, or the argument of a direct call + * whose parameter at that position is a candidate itself. A parameter is a + * candidate only when its function is reached by direct calls alone and + * every call passes a candidate there, so every binding that can hold one + * of these arrays is a candidate: the array never reaches a runtime + * function that reads its slots, a library call, a closure, a return or + * untyped code. * * Candidates are only proposed here. The int32 slot analysis * (int32-slots.ts) keeps the ones whose every stored element is a proven @@ -33,6 +39,18 @@ import { * calls are inlined first (rewriteCandidates), so no shared helper ever * receives an int32 array. */ +/** The candidate key of the binding `localId` as `fnName` reads it: + * `function\0local` for a local or parameter, `\0global` for a module + * global (`%g.` ids), which every function reads alike. */ +export function int32ArrayKey(fnName: string, localId: string): string { + return localId.startsWith("%g.") ? `\0${localId}` : `${fnName}\0${localId}`; +} + +/** The int32 element slot (int32-slots.ts) of a candidate key. */ +export function int32ArraySlot(key: string): string { + return key.startsWith("\0") ? `g:${key.slice(1)}` : `l:${key}`; +} + /** Array methods a candidate may be the receiver of: the runtime paths * behind them read and write int32 slots through the double-valued slot * accessors. */ @@ -202,10 +220,51 @@ function writesLocal(e: IrExpr, id: string): boolean { return !expr(e); } -/** The candidate locals of `fn` (see the module comment). */ -export function int32ArrayCandidates(fn: IrFunction, readers: ReadonlySet): Set { +/** The parameters and globals a module's functions share as candidates, + * still assumed to qualify. */ +export interface SharedCandidates { + /** Candidate parameters, as `function\0local` keys. */ + params: ReadonlySet; + /** Candidate globals, as `%g.` ids. */ + globals: ReadonlySet; + /** Whether the parameter at `index` of `callee` is a candidate. */ + paramAt: (callee: string, index: number) => boolean; +} + +const NO_SHARED: SharedCandidates = { + params: new Set(), + globals: new Set(), + paramAt: () => false, +}; + +/** The candidates of `fn` (see the module comment): its own locals, its + * parameters in `shared.params`, and the globals of `shared.globals` that + * it reads only in candidate positions. `written` receives the globals it + * assigns fresh arrays. */ +export function int32ArrayCandidates( + fn: IrFunction, + readers: ReadonlySet, + shared: SharedCandidates = NO_SHARED, + written: Set = new Set(), +): Set { const result = new Set(); - if (fn.async || fn.generator) return result; + if (fn.async || fn.generator) { + // Globals read here leave the candidates. + for (const id of shared.globals) result.add(id); + everyStmtList(fn.body, { + expr: (e) => { + if ((e.kind === "varRef" || e.kind === "assignExpr") && result.has(e.localId)) + result.delete(e.localId); + return true; + }, + stmt: (s) => { + if ((s.kind === "varDecl" || s.kind === "assign") && result.has(s.localId)) + result.delete(s.localId); + return true; + }, + }); + return result; + } const captured = new Set( [...(fn.captures ?? []), ...(fn.classCaptures ?? [])].map((c) => c.localId), ); @@ -217,11 +276,12 @@ export function int32ArrayCandidates(fn: IrFunction, readers: ReadonlySet(); + const allocated = new Set(); // A receiver position: the candidate is read there without escaping. const receiver = (e: IrExpr): void => { if (e.kind !== "varRef") expr(e); @@ -255,13 +315,22 @@ export function int32ArrayCandidates(fn: IrFunction, readers: ReadonlySet { + if (shared.paramAt(e.callee, index)) receiver(arg); + else expr(arg); + }); return true; } } @@ -272,7 +341,8 @@ export function int32ArrayCandidates(fn: IrFunction, readers: ReadonlySet | undefined, live: ReadonlySet): Set { + const result = new Set(); + if (ids) for (const id of ids) if (live.has(id)) result.add(id); + return result; +} + +/** The candidates of every function of `mod` that has any, only those + * whose keys are in `keep` when given. Parameters and globals are shared: + * starting from every one that may qualify, each pass drops the ones a + * function reads outside candidate positions, the parameters some call + * passes anything but a candidate, and the globals never assigned a fresh + * array, until a pass drops nothing. Functions in `redirected` keep their + * parameters: some of their calls reach another function. */ +export function moduleInt32ArrayCandidates( + mod: IrModule, + readers: ReadonlySet, + keep: ReadonlySet | null = null, + redirected: ReadonlySet = new Set(), +): Map> { + const kept = (key: string): boolean => keep === null || keep.has(key); + const arrayParams = new Map(); + for (const fn of mod.functions) { + // Class methods, constructors and runtime helpers (`%` names) keep + // their parameters. + if (fn.name.startsWith("%") || fn.async || fn.generator || redirected.has(fn.name)) continue; + const indexes: number[] = []; + fn.params.forEach((p, i) => { + if (p.type.kind === "array" && p.type.elem.kind === "f64") indexes.push(i); + }); + if (indexes.length > 0) arrayParams.set(fn.name, indexes); + } + let globals = new Set(); + for (const g of mod.globals ?? []) + if ( + g.type.kind === "array" && + g.type.elem.kind === "f64" && + g.initFlag === undefined && + kept(`\0${g.id}`) + ) + globals.add(g.id); + const result = new Map>(); + if (arrayParams.size === 0 && globals.size === 0) { + for (const fn of mod.functions) { + const candidates = int32ArrayCandidates(fn, readers); + for (const id of candidates) if (!kept(`${fn.name}\0${id}`)) candidates.delete(id); + if (candidates.size > 0) result.set(fn.name, candidates); + } + return result; + } + // One walk: the functions reached other than by direct calls, the direct + // calls of functions with number[] parameters, the globals each function + // reads or writes, and whether a module namespace object exists (its + // member reads can reach any global by name). + const reached = new Set(moduleFunctionReferences(mod)); + const calls: { caller: string; call: IrExpr & { kind: "call" } }[] = []; + const globalsIn = new Map>(); + let namespaces = false; + for (const fn of mod.functions) { + const used = new Set(); + const note = (id: string): void => { + if (globals.has(id)) used.add(id); + }; + everyStmtList(fn.body, { + expr: (e) => { + const name = exprFunctionReference(e); + if (name !== undefined) reached.add(name); + if (e.kind === "moduleNsRef") namespaces = true; + if (e.kind === "call" && arrayParams.has(e.callee)) + calls.push({ caller: fn.name, call: e }); + if (e.kind === "varRef" || e.kind === "assignExpr") note(e.localId); + return true; + }, + stmt: (s) => { + if (s.kind === "varDecl" || s.kind === "assign") note(s.localId); + return true; + }, + }); + if (used.size > 0) globalsIn.set(fn.name, used); + } + if (namespaces) { + globals = new Set(); + globalsIn.clear(); + } + const paramIds = new Map(); + let params = new Set(); + for (const fn of mod.functions) { + const indexes = arrayParams.get(fn.name); + if (!indexes || reached.has(fn.name)) continue; + paramIds.set( + fn.name, + fn.params.map((p) => p.localId), + ); + for (const i of indexes) { + const key = `${fn.name}\0${fn.params[i]!.localId}`; + if (kept(key)) params.add(key); + } + } + for (;;) { + const current = params; + const written = new Set(); + const liveGlobals = new Set(globals); + result.clear(); + for (const fn of mod.functions) { + const shared: SharedCandidates = { + params: current, + globals: liveIn(globalsIn.get(fn.name), globals), + paramAt: (callee, index) => { + const id = paramIds.get(callee)?.[index]; + return id !== undefined && current.has(`${callee}\0${id}`); + }, + }; + const candidates = int32ArrayCandidates(fn, readers, shared, written); + for (const id of shared.globals) if (!candidates.has(id)) liveGlobals.delete(id); + for (const id of candidates) + if (!shared.globals.has(id) && !kept(`${fn.name}\0${id}`)) candidates.delete(id); + if (candidates.size > 0) result.set(fn.name, candidates); + } + for (const id of liveGlobals) if (!written.has(id)) liveGlobals.delete(id); + const live = new Set( + [...current].filter((key) => { + const cut = key.indexOf("\0"); + return result.get(key.slice(0, cut))?.has(key.slice(cut + 1)) === true; + }), + ); + // A parameter needs a candidate at that position of every call; a call + // with fewer arguments passes undefined. + for (const { caller, call } of calls) { + const ids = paramIds.get(call.callee); + if (!ids) continue; + ids.forEach((id, i) => { + const key = `${call.callee}\0${id}`; + if (!live.has(key)) return; + const arg = call.args[i]; + const ok = + arg?.kind === "varRef" && + (arg.localId.startsWith("%g.") + ? liveGlobals.has(arg.localId) + : result.get(caller)?.has(arg.localId) === true); + if (!ok) live.delete(key); + }); + } + if (live.size === current.size && liveGlobals.size === globals.size) break; + params = live; + globals = liveGlobals; + } + return result; +} + +/** The candidates one array can reach together: a direct call's argument + * and the parameter it binds hold the same array, so their element facts + * stand or fall together. `group` maps each of `keys` to its group's keys + * (itself alone for most), and `globalReaders` each candidate global's + * key to the functions that read or write it. */ +export function int32ArrayGroups( + mod: IrModule, + keys: ReadonlySet, +): { group: Map; globalReaders: Map> } { + const parent = new Map(); + const root = (key: string): string => { + let r = key; + while (parent.has(r)) r = parent.get(r)!; + if (r !== key) parent.set(key, r); + return r; + }; + const globalReaders = new Map>(); + const params = new Map(mod.functions.map((fn) => [fn.name, fn.params])); + const paramKeys = new Set( + mod.functions.flatMap((fn) => fn.params.map((p) => `${fn.name}\0${p.localId}`)), + ); + if ([...keys].some((key) => key.startsWith("\0") || paramKeys.has(key))) { + for (const fn of mod.functions) { + const touch = (id: string): void => { + const key = int32ArrayKey(fn.name, id); + if (!key.startsWith("\0") || !keys.has(key)) return; + const set = globalReaders.get(key) ?? new Set(); + set.add(fn.name); + globalReaders.set(key, set); + }; + everyStmtList(fn.body, { + expr: (e) => { + if (e.kind === "varRef" || e.kind === "assignExpr") touch(e.localId); + if (e.kind === "call") + e.args.forEach((arg, i) => { + const param = params.get(e.callee)?.[i]; + if (arg.kind !== "varRef" || !param) return; + const a = int32ArrayKey(fn.name, arg.localId), + p = `${e.callee}\0${param.localId}`; + if (!keys.has(a) || !keys.has(p)) return; + const ra = root(a), + rp = root(p); + if (ra !== rp) parent.set(ra, rp); + }); + return true; + }, + stmt: (s) => { + if (s.kind === "varDecl" || s.kind === "assign") touch(s.localId); + return true; + }, + }); + } + } + const members = new Map(); + for (const key of keys) { + const r = root(key); + const list = members.get(r); + if (list) list.push(key); + else members.set(r, [key]); + } + const group = new Map(); + for (const key of keys) group.set(key, members.get(root(key))!); + return { group, globalReaders }; +} + /** The element values stored into each candidate: element writes, pushed * values, literal elements and fill values. */ export function int32ArrayStores( @@ -471,40 +759,45 @@ function rewriteCandidates( return { ...fn, body, locals: locals.filter((local) => !popLocals.has(local.id)) }; } -/** Propose the module's int32 array candidates (`function\0local` keys in +/** Propose the module's int32 array candidates (int32ArrayKey keys in * `int32ArrayCandidates`), only those in `keep` when given, and rewrite * their functions. */ export function markInt32ArrayCandidates( mod: IrModule, keep: ReadonlySet | null = null, + redirected: ReadonlySet = new Set(), ): IrModule { if (mod.lib !== undefined) return mod; const readers = new Map(); for (const fn of mod.functions) if (isElementReader(fn)) readers.set(fn.name, fn); - const names = new Set(readers.keys()); - const keys: string[] = []; + const all = moduleInt32ArrayCandidates(mod, new Set(readers.keys()), keep, redirected); + if (all.size === 0) return mod; + const keys = new Set(); const functions = mod.functions.map((fn) => { - const candidates = int32ArrayCandidates(fn, names); - for (const id of candidates) - if (keep === null || keep.has(`${fn.name}\0${id}`)) keys.push(`${fn.name}\0${id}`); - else candidates.delete(id); - return candidates.size === 0 ? fn : rewriteCandidates(fn, candidates, readers); + const candidates = all.get(fn.name); + if (!candidates) return fn; + for (const id of candidates) keys.add(int32ArrayKey(fn.name, id)); + return rewriteCandidates(fn, candidates, readers); }); - return keys.length === 0 ? mod : { ...mod, functions, int32ArrayCandidates: keys }; + return { ...mod, functions, int32ArrayCandidates: [...keys] }; } /** The module the backend emits: rewritten for exactly the candidates the * int32 slot analysis (`prove`, returning its int32Arrays) keeps, so a * module without int32 arrays is `mod` itself. A dropped candidate's * rewrite is undone by marking again without it, until every candidate - * marked is kept. */ + * marked is kept. `redirected` names the functions some of whose calls the + * backend emits as calls of another function (a result-slot entry): their + * parameters keep doubles. */ export function selectInt32Arrays( mod: IrModule, prove: (marked: IrModule) => ReadonlySet, + redirected: (mod: IrModule) => ReadonlySet = () => new Set(), ): IrModule { let keep: ReadonlySet | null = null; + const others = redirected(mod); for (let round = 0; round < 4; round++) { - const marked = markInt32ArrayCandidates(mod, keep); + const marked = markInt32ArrayCandidates(mod, keep, others); if (marked === mod) return mod; const kept = prove(marked); const proposed = marked.int32ArrayCandidates ?? []; diff --git a/packages/compiler/src/ir/int32-slots.ts b/packages/compiler/src/ir/int32-slots.ts index 4a2abc5b0..a8e91c86e 100644 --- a/packages/compiler/src/ir/int32-slots.ts +++ b/packages/compiler/src/ir/int32-slots.ts @@ -10,7 +10,13 @@ import { } from "./integer-ranges.js"; import { exprFunctionReference, moduleFunctionReferences } from "./function-references.js"; import { findInitializerBindings, withInitializerBindings } from "./initializer-bindings.js"; -import { checkableInt32Allocations, int32ArrayStores } from "./int32-arrays.js"; +import { + checkableInt32Allocations, + int32ArrayGroups, + int32ArrayKey, + int32ArraySlot, + int32ArrayStores, +} from "./int32-arrays.js"; import { everyExprChild, everyStmtChild, everyStmtList, everyTypeChild } from "./traverse.js"; /** The storage slot an array-valued expression reads, as an @@ -247,9 +253,10 @@ export class Int32Slots { /** Whether the module has int32 array candidates (the narrowedReads * range fact). */ narrowedReads = false; - /** Local number arrays (`function\0local`) stored as int32 slots: the - * module's int32 array candidates whose every stored element the - * fixpoint proved an int32. */ + /** Number arrays stored as int32 slots (int32ArrayKey keys: locals and + * parameters as `function\0local`, globals as `\0global`): the module's + * int32 array candidates whose every stored element, through every + * binding that can hold the same array, the fixpoint proved an int32. */ readonly int32Arrays = new Set(); /** Local number arrays that may hold int32 slots with checked stores: the * candidates whose element stores the fixpoint proved exact integers but @@ -779,17 +786,29 @@ export function analyzeInt32Slots(mod: IrModule, inputs: Int32SlotInputs = {}): for (const key of contracts?.int32Locals ?? []) slots.addInt32Local(key); for (const key of contracts?.int32Elements ?? []) slots.addInt32Elements(key); // Int32 array candidates (ir/int32-arrays.ts): assumed to hold int32 - // elements until a store the fixpoint cannot prove drops one. + // elements until a store the fixpoint cannot prove drops one, with every + // candidate the same array can reach (its group). const arrayCandidates = new Map>(); - slots.narrowedReads = (mod.int32ArrayCandidates?.length ?? 0) > 0; - for (const key of mod.int32ArrayCandidates ?? []) { + const globalArrays = new Set(); + const arrayKeys = new Set(mod.int32ArrayCandidates ?? []); + slots.narrowedReads = arrayKeys.size > 0; + for (const key of arrayKeys) { const cut = key.indexOf("\0"); - const fnName = key.slice(0, cut); - const set = arrayCandidates.get(fnName) ?? new Set(); - set.add(key.slice(cut + 1)); - arrayCandidates.set(fnName, set); - slots.addInt32Elements(`l:${key}`); + if (cut === 0) globalArrays.add(key.slice(1)); + else { + const fnName = key.slice(0, cut); + const set = arrayCandidates.get(fnName) ?? new Set(); + set.add(key.slice(cut + 1)); + arrayCandidates.set(fnName, set); + } + slots.addInt32Elements(int32ArraySlot(key)); } + const arrayGroups = arrayKeys.size > 0 ? int32ArrayGroups(mod, arrayKeys) : null; + // Functions that read or write a candidate global. + const globalArrayFunctions = new Set(); + if (arrayGroups) + for (const names of arrayGroups.globalReaders.values()) + for (const name of names) globalArrayFunctions.add(name); /* ── candidate fields ── */ const children = new Map(); @@ -1070,6 +1089,7 @@ export function analyzeInt32Slots(mod: IrModule, inputs: Int32SlotInputs = {}): }; const hasCandidateWrite = (fn: IrFunction): boolean => { if (arrayCandidates.has(fn.name)) return true; + if (globalArrayFunctions.has(fn.name)) return true; const box = (localId: string): boolean => slots.hasBox(slots.boxGroup(fn.name, localId)); const returns = slots.hasReturn(fn.name); return !everyStmtList(fn.body, { @@ -1124,25 +1144,49 @@ export function analyzeInt32Slots(mod: IrModule, inputs: Int32SlotInputs = {}): // (checkableInt32Allocations), it keeps int32 slots with checked stores // (checkedInt32Arrays), which widen it to doubles in place at the first // store that is not an int32. - const arrays = arrayCandidates.get(fn.name); - if (arrays) { - let dropped = false; + const locals = arrayCandidates.get(fn.name); + let arrays = locals; + if (globalArrayFunctions.has(fn.name)) { + const both = new Set(); + if (locals) for (const id of locals) both.add(id); + for (const id of globalArrays) both.add(id); + arrays = both; + } + if (arrays && arrayGroups) { + const requeue = (name: string): void => { + if (queued.has(name)) return; + queued.add(name); + queue.push(name); + }; + const params = new Set(fn.params.map((param) => param.localId)); for (const [id, values] of int32ArrayStores(fn, arrays)) { - const key = `l:${fn.name}\0${id}`; - if (!slots.hasInt32ElementSlot(key) || values.every(int32)) continue; - slots.dropInt32Elements(key); + const key = int32ArrayKey(fn.name, id); + if (!slots.hasInt32ElementSlot(int32ArraySlot(key)) || values.every(int32)) continue; + // The array's group drops together: every function reading a + // member is analyzed again without its facts. + const group = arrayGroups.group.get(key) ?? [key]; + for (const member of group) { + slots.dropInt32Elements(int32ArraySlot(member)); + if (member.startsWith("\0")) { + const names = arrayGroups.globalReaders.get(member); + if (names) for (const name of names) requeue(name); + } else requeue(member.slice(0, member.indexOf("\0"))); + } + // Checked stores serve a lone local: its allocations are the only + // arrays it holds. const checkable = (value: IrExpr): boolean => { const range = ranges.get(value); return int32(value) || range != null; }; - if (values.every(checkable) && checkableInt32Allocations(fn, id)) - slots.checkedInt32Arrays.add(`${fn.name}\0${id}`); - else reason(`array:${fn.name}\0${id}`, "an element store is not int32"); - dropped = true; - } - if (dropped && !queued.has(fn.name)) { - queued.add(fn.name); - queue.push(fn.name); + if ( + group.length === 1 && + !key.startsWith("\0") && + !params.has(id) && + values.every(checkable) && + checkableInt32Allocations(fn, id) + ) + slots.checkedInt32Arrays.add(key); + else for (const member of group) reason(`array:${member}`, "an element store is not int32"); } } // An int32 written into a slot still assumed non-negative: widen the @@ -1213,10 +1257,8 @@ export function analyzeInt32Slots(mod: IrModule, inputs: Int32SlotInputs = {}): } for (const proven of provenByFunction.values()) for (const value of proven) slots.provenWrites.add(value); - for (const [fnName, ids] of arrayCandidates) - for (const id of ids) - if (slots.hasInt32ElementSlot(`l:${fnName}\0${id}`)) - slots.int32Arrays.add(`${fnName}\0${id}`); + for (const key of arrayKeys) + if (slots.hasInt32ElementSlot(int32ArraySlot(key))) slots.int32Arrays.add(key); const ranged = rangedParameters(slots, functions, retry, numeric, (callee) => readers.get(`ret:${callee}`), ); diff --git a/packages/compiler/test/int32-element-array-emission.test.ts b/packages/compiler/test/int32-element-array-emission.test.ts index 0b68a5d3e..cd62278db 100644 --- a/packages/compiler/test/int32-element-array-emission.test.ts +++ b/packages/compiler/test/int32-element-array-emission.test.ts @@ -196,3 +196,42 @@ test("only the candidates the proof keeps are rewritten", async () => { for (const fn of selected.functions) if (fn.name !== "compose") expect(fn).toBe(mod.functions.find((f) => f.name === fn.name)); }); + +const sharedSource = ` +const data: number[] = []; +for (let i = 0; i < 64; i++) data.push((i * 7) & 255); +function fnv(values: number[]): number { + let h = 2166136261; + for (let i = 0; i < values.length; i++) h = Math.imul(h ^ values[i]!, 16777619) >>> 0; + return h; +} +function mean(values: number[]): number { + let s = 0; + for (let i = 0; i < values.length; i++) s += values[i]!; + return s / values.length; +} +function first(values: number[]): number { + return values[0]!; +} +const ints = [1, 2, 3]; +const halves = [0.5, 1.5]; +const pick: (values: number[]) => number = first; +console.log(fnv(data), mean(ints), mean(halves), pick([4])); +`; + +test("arrays passed to helpers and held in module globals get 4-byte slots", async () => { + const mod = await lowerIr(sharedSource); + const proposed = markInt32ArrayCandidates(mod).int32ArrayCandidates ?? []; + // A global and the parameter every call passes it to; a function used as + // a value keeps its parameter. + expect(proposed).toEqual(expect.arrayContaining(["\0%g.e.data", "fnv\0values.0"])); + expect(proposed.some((key) => key.startsWith("first\0"))).toBe(false); + const llvm = await emitLlvm(sharedSource); + // The helper reads int32 slots; `mean`, also called with fractions, + // reads doubles. + expect(functions(llvm, "fnv")).toMatch( + /getelementptr inbounds i32, ptr %\w+, i64 %\w+\n\s+%\w+ = load i32/, + ); + expect(functions(llvm, "mean")).not.toMatch(/getelementptr inbounds i32, ptr/); + expect(llvm).toMatch(/@scr_arr_new(_small)?\(i32 7/); +}); diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7580-int32-arrays-across-calls.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7580-int32-arrays-across-calls.ts.json new file mode 100644 index 000000000..33282756c --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7580-int32-arrays-across-calls.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7580-int32-arrays-across-calls.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7580-int32-arrays-across-calls.ts b/tests/corpus/7580-int32-arrays-across-calls.ts new file mode 100644 index 000000000..13df54609 --- /dev/null +++ b/tests/corpus/7580-int32-arrays-across-calls.ts @@ -0,0 +1,146 @@ +// Number arrays of proven int32s that reach other functions: passed to +// helpers called directly, along chains of helpers and recursion, and held +// in module globals that several functions read. A helper that also +// receives an array of other numbers or one that is not a fresh array, +// stores a fraction through its parameter, or escapes as a value keeps +// every array it can reach in doubles, as does a global an async function +// reads. Each case uses its own arrays and helpers. + +const data: number[] = []; +let seed = 7; +for (let i = 0; i < 2000; i++) { + seed = (Math.imul(seed, 48271) + 11) & 0x7fffffff; + data.push(seed % 1000); +} + +function fnv(values: number[]): number { + let h = 2166136261; + for (let i = 0; i < values.length; i++) { + h ^= values[i]! & 0xff; + h = Math.imul(h, 16777619) >>> 0; + } + return h; +} + +function histogram(values: number[], buckets: number): number[] { + const counts: number[] = new Array(buckets).fill(0); + for (let i = 0; i < values.length; i++) { + const b = values[i]! % buckets; + counts[b] = counts[b]! + 1; + } + return counts; +} + +// A helper that writes through its parameter and calls another helper. +function scale(values: number[], factor: number): number { + for (let i = 0; i < values.length; i++) values[i] = (values[i]! * factor) & 0xffff; + return sum(values); +} + +function sum(values: number[]): number { + let s = 0; + for (let i = 0; i < values.length; i++) s = (s + values[i]!) | 0; + return s; +} + +// Recursion passes the array on, and the elements index another array. +function chase(next: number[], at: number, steps: number): number { + if (steps === 0) return at; + return chase(next, next[at]!, steps - 1); +} + +function pushAll(target: number[], count: number): void { + for (let i = 0; i < count; i++) target.push((i * 3) & 0xffff); +} + +// `n` is a power of two: the masked elements are indexes of `perm`. +function locals(n: number): string { + const perm: number[] = []; + for (let i = 0; i < n; i++) perm.push((i * 37 + 11) & (n - 1)); + const copy: number[] = []; + pushAll(copy, n); + const s1 = scale(copy, 5); + return `locals ${chase(perm, 0, 1000)} ${s1} ${copy[n - 1]} ${copy.length} ${sum(perm)}`; +} + +// Read only through the element reader (`a[i] ?? 0`) and states. +function readSparse(values: number[], at: number): string { + return `${values[at] ?? -1} ${at in values} ${values.length}`; +} + +// The same helper receives an array of other numbers too: the arrays it +// can reach keep doubles. +function mean(values: number[]): number { + let s = 0; + for (let i = 0; i < values.length; i++) s += values[i]!; + return s / values.length; +} + +// A store through the parameter that is not an int32. +function halve(values: number[]): void { + for (let i = 0; i < values.length; i++) values[i] = values[i]! / 2; +} + +// A function used as a value keeps its parameters. +function first(values: number[]): number { + return values[0]!; +} + +let reassigned: number[] = [1, 2, 3]; +function regrow(n: number): number { + reassigned = []; + for (let i = 0; i < n; i++) reassigned.push(i & 15); + let s = 0; + for (let i = 0; i < reassigned.length; i++) s += reassigned[i]!; + return s; +} + +// Not a fresh array: the result of a call. +function total(values: number[]): number { + let s = 0; + for (let i = 0; i < values.length; i++) s += values[i]!; + return s; +} + +const awaited: number[] = [4, 5, 6]; +async function later(): Promise { + return awaited[0]! + awaited.length; +} + +// A helper whose record result its callers only project: the backend may +// call its result-slot entry instead, so its parameter keeps doubles. +interface Span { + lo: number; + hi: number; +} +function spanAt(values: number[], i: number): Span { + return { lo: values[i]!, hi: values[i + 1]! }; +} +const spans: number[] = [5, 9, 2, 7, 11]; +function spanTotal(): number { + let total = 0; + for (let i = 0; i < 4; i++) { + const span = spanAt(spans, i); + total += span.hi * 10 - span.lo; + } + return total; +} + +const ints: number[] = [3, 4, 5]; +const fractions: number[] = [0.5, 1.5, 2.5]; +const halved: number[] = [8, 6, 4]; +halve(halved); +const pick: (values: number[]) => number = first; +const viaValue: number[] = [9, 8, 7]; + +console.log(`fnv ${fnv(data)} ${fnv(data)}`); +const h = histogram(data, 10); +console.log(`hist ${h.length} ${h[0]} ${h[9]} ${total(h)}`); +console.log(locals(128)); +console.log(readSparse(data, 5), readSparse(data, 5000)); +console.log(`mean ${mean(ints)} ${mean(fractions)}`); +console.log(`halved ${halved[0]} ${halved[2]} ${halved.length}`); +console.log(`value ${pick(viaValue)} ${viaValue.length}`); +console.log(`regrow ${regrow(40)} ${reassigned.length} ${reassigned[39]} ${regrow(5)}`); +console.log(`span ${spanTotal()}`); +void later().then((v) => console.log(`later ${v}`)); diff --git a/tests/harness/corpus-costs/7580-int32-arrays-across-calls.ts.json b/tests/harness/corpus-costs/7580-int32-arrays-across-calls.ts.json new file mode 100644 index 000000000..7c9d20632 --- /dev/null +++ b/tests/harness/corpus-costs/7580-int32-arrays-across-calls.ts.json @@ -0,0 +1 @@ +{ "plain": 3100, "san": 3500 } From b6d6eb60aed3849954d9f69cc0ff4607d63a7504 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 01:34:19 -0500 Subject: [PATCH 07/53] Replace a scheduled _construct hook, and keep runtime option values in order Node's constructNT reads _construct on the construction tick, so an assignment after the constructor and before that tick replaces a scheduled construct hook. The assignment was dropped, so the original hook ran. It now lowers to stream.replaceConstruct, which swaps the hook while its tick has not run and has no effect after it or on a stream without a hook. Runtime decodeStrings and defaultEncoding values are read into locals before the constructor evaluates the other option members, while Node evaluates the members in order. A runtime value after a member whose value has effects (a call or an assignment) now refuses instead of reading a value that member could still change. --- .../builtin-apis/streams-and-readline.mdx | 1 + .../compiler/src/backend/llvm/lib-shared.ts | 1 + .../compiler/src/backend/llvm/lib-stream.ts | 3 + .../src/frontend/lowering/lower-stream.ts | 63 +++++++++++++------ .../compiler/src/ir/builtin-signatures.ts | 1 + packages/compiler/src/ir/validate.ts | 1 + .../7466-stream-construct-assign.mjs.json | 6 ++ .../stream-runtime-option-order.ts.json | 6 ++ packages/runtime/src/scr_stream.c | 13 ++++ packages/runtime/src/scr_stream.h | 4 ++ tests/corpus/7466-stream-construct-assign.mjs | 46 ++++++++++++++ .../stream-runtime-option-order.ts | 28 +++++++++ .../stream-runtime-option-order.ts.txt | 17 +++++ .../7466-stream-construct-assign.mjs.json | 1 + 14 files changed, 173 insertions(+), 18 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7466-stream-construct-assign.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/stream-runtime-option-order.ts.json create mode 100644 tests/corpus/7466-stream-construct-assign.mjs create mode 100644 tests/diagnostics/stream-runtime-option-order.ts create mode 100644 tests/harness/__snapshots__/stream-runtime-option-order.ts.txt create mode 100644 tests/harness/corpus-costs/7466-stream-construct-assign.mjs.json diff --git a/docs/content/limitations/builtin-apis/streams-and-readline.mdx b/docs/content/limitations/builtin-apis/streams-and-readline.mdx index b88ead1f4..166fc3402 100644 --- a/docs/content/limitations/builtin-apis/streams-and-readline.mdx +++ b/docs/content/limitations/builtin-apis/streams-and-readline.mdx @@ -33,6 +33,7 @@ `pipeline` from `node:stream/promises` accepts a trailing `{ signal, end }` object, with or without function stages, and `finished` accepts `readable`, `writable`, `error`, `cleanup`, and `signal` options, with boolean literals for the flags. Stream constructors accept a `signal` option in an options object literal, and `stream.addAbortSignal` accepts any stream. The `construct` option and `_construct` method, `decodeStrings`, and `defaultEncoding` work on every stream class, and the `writev` option and `_writev` method on every class with a writable side. + A `decodeStrings` or `defaultEncoding` value known only at run time is unsupported after an option member whose value has effects (a call or an assignment). In typed code, a `writev` entry's `encoding` declared as a string receives `""` where Node passes `undefined` or `null` (an object-mode write). `setDefaultEncoding()` takes any encoding Node knows. `node:readline` and `node:readline/promises` (also reached as `readline.promises`) interfaces, from `createInterface(options)` or `createInterface(input, output, completer, terminal)`, read `process.stdin` or a readable stream: `question`, `prompt`, `write` (with a key object literal of `name`, `ctrl`, `meta`, and `shift`), `pause`, `resume`, `'line'`/`'close'`/`'pause'`/`'resume'`/`'error'`/`'history'`/`'SIGINT'`/`'SIGTSTP'`/`'SIGCONT'` listeners, and `for await` compile. diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index f31906354..25af12ad6 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -1768,6 +1768,7 @@ export const USES_TIMERS_LIB_FNS = new Set([ "stream.setTransform", "stream.setFlush", "stream.setConstruct", + "stream.replaceConstruct", "stream.duplexFromGen", "stream.duplexFromDyn", "stream.duplexifyReadable", diff --git a/packages/compiler/src/backend/llvm/lib-stream.ts b/packages/compiler/src/backend/llvm/lib-stream.ts index 5fd4af1a9..fdbf13d4f 100644 --- a/packages/compiler/src/backend/llvm/lib-stream.ts +++ b/packages/compiler/src/backend/llvm/lib-stream.ts @@ -189,6 +189,7 @@ export function emitStreamLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlV e.fn === "stream.setTransform" || e.fn === "stream.setFlush" || e.fn === "stream.setConstruct" || + e.fn === "stream.replaceConstruct" || e.fn === "stream.setWritev" || e.fn === "stream.setWritevDefault" ) { @@ -203,6 +204,7 @@ export function emitStreamLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlV "stream.setTransform": "t", "stream.setFlush": "l", "stream.setConstruct": "c", + "stream.replaceConstruct": "c", "stream.setWritev": "v", "stream.setWritevDefault": "v", }; @@ -214,6 +216,7 @@ export function emitStreamLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlV "stream.setTransform": "scr_stream_set_transform", "stream.setFlush": "scr_stream_set_flush", "stream.setConstruct": "scr_stream_set_construct", + "stream.replaceConstruct": "scr_stream_replace_construct", "stream.setWritev": "scr_stream_set_writev", "stream.setWritevDefault": "scr_stream_set_writev_default", }; diff --git a/packages/compiler/src/frontend/lowering/lower-stream.ts b/packages/compiler/src/frontend/lowering/lower-stream.ts index 15afb741e..cf9f19b9d 100644 --- a/packages/compiler/src/frontend/lowering/lower-stream.ts +++ b/packages/compiler/src/frontend/lowering/lower-stream.ts @@ -106,6 +106,21 @@ function pureRead(node: ts.Expression): boolean { return false; } +/** Whether every option member before `prop` evaluates without effects. + * Runtime decodeStrings and defaultEncoding values are read into locals + * before the constructor evaluates the other members, while Node evaluates + * the members in order: an earlier member with effects could change them. */ +function earlierOptionsPure(options: ts.ObjectLiteralExpression, prop: ts.Node): boolean { + for (const member of options.properties.nodes) { + if (member === prop) return true; + if (ts.isPropertyAssignment(member)) { + if (!sideEffectFreeOptionValue(member.initializer) && !pureRead(member.initializer)) + return false; + } else if (ts.isSpreadAssignment(member)) return false; + } + return true; +} + /** The stream sides of a receiver class: the nearest stream-class * ancestor's, or null off the stream hierarchy. */ export function streamSidesOf( @@ -1021,6 +1036,16 @@ function parseStreamOptions( break; } // a runtime value: WritableState's `options.decodeStrings !== false` + if ( + optsLit !== undefined && + ts.isObjectLiteralExpression(optsLit) && + !earlierOptionsPure(optsLit, prop) + ) + lowerer.noLowering( + `the ${ctorName} option 'decodeStrings' with a runtime value after an option value with effects`, + prop, + "compute the earlier option values into consts before the options object (the runtime value is read before the other options run)", + ); const local = lowerer.declareHiddenLocal("%decodeStrings", DYN); out.valueDecls.push({ kind: "varDecl", @@ -1076,6 +1101,16 @@ function parseStreamOptions( // a runtime value: null, undefined and utf8 are the default, // another name Buffer.isEncoding knows is kept as given, and any // other value throws ERR_UNKNOWN_ENCODING before construction + if ( + optsLit !== undefined && + ts.isObjectLiteralExpression(optsLit) && + !earlierOptionsPure(optsLit, prop) + ) + lowerer.noLowering( + `the ${ctorName} option 'defaultEncoding' with a runtime value after an option value with effects`, + prop, + "compute the earlier option values into consts before the options object (the runtime value is read before the other options run)", + ); const local = lowerer.declareHiddenLocal("%defaultEncoding", DYN); const ref = varRef(local.id, DYN, loc); out.valueDecls.push( @@ -1869,24 +1904,15 @@ export function lowerStreamUnderscoreAssign( const loc = locOf(expr); const cls = base.def.name; const clsName = cls.slice(1); - if (methodName === "_construct") { - // Node's constructor reads _construct once; a later assignment has no - // effect on the stream, so only the receiver and the value are - // evaluated - const value = lowerer.lowerExpr(expr.right); - return { - kind: "exprStmt", - expr: { - kind: "seqExpr", - stmts: [{ kind: "exprStmt", expr: recv, loc }], - result: value, - type: value.type, - loc, - }, - loc, - }; - } - const which = methodName === "_writev" ? "writev" : optionByMethod.get(methodName)!; + // `_construct`: Node's constructNT reads it on the construction tick, so + // an assignment before that tick replaces a scheduled hook + // (stream.replaceConstruct) and has no effect after it or without one. + const which = + methodName === "_writev" + ? "writev" + : methodName === "_construct" + ? "construct" + : optionByMethod.get(methodName)!; const { accepted } = streamCtorShape(cls); if (!accepted.includes(which) && !streamExtraCallbacks(cls).includes(which)) { lowerer.noLowering( @@ -1925,6 +1951,7 @@ export function lowerStreamUnderscoreAssign( transform: "stream.setTransform", flush: "stream.setFlush", writev: "stream.setWritev", + construct: "stream.replaceConstruct", }; return { kind: "exprStmt", diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index aa31105e7..ca91b399c 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -3975,6 +3975,7 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ "stream.setTransform": { argTypes: [null, null], result: VOID }, "stream.setFlush": { argTypes: [null, null], result: VOID }, "stream.setConstruct": { argTypes: [null, null], result: VOID }, + "stream.replaceConstruct": { argTypes: [null, null], result: VOID }, "stream.setWritev": { argTypes: [null, null], result: VOID }, "stream.setWritevDefault": { argTypes: [null, null], result: VOID }, /** The dyn-options twins (a checked-dynamic options record — the JS diff --git a/packages/compiler/src/ir/validate.ts b/packages/compiler/src/ir/validate.ts index 44c032620..8b3b6c048 100644 --- a/packages/compiler/src/ir/validate.ts +++ b/packages/compiler/src/ir/validate.ts @@ -5508,6 +5508,7 @@ function validateFunction( e.fn === "stream.duplexFromGenSignal" || e.fn === "stream.duplexFromAsyncFnSignal" || e.fn === "stream.setConstruct" || + e.fn === "stream.replaceConstruct" || e.fn === "stream.setWritev" || e.fn === "stream.setWritevDefault" || e.fn === "stream.iteratorClose" || diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7466-stream-construct-assign.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7466-stream-construct-assign.mjs.json new file mode 100644 index 000000000..f5bb52517 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7466-stream-construct-assign.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7466-stream-construct-assign.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/stream-runtime-option-order.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/stream-runtime-option-order.ts.json new file mode 100644 index 000000000..40b64edac --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/stream-runtime-option-order.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/diagnostics/stream-runtime-option-order.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_stream.c b/packages/runtime/src/scr_stream.c index 4b962675a..cd95e12cd 100644 --- a/packages/runtime/src/scr_stream.c +++ b/packages/runtime/src/scr_stream.c @@ -269,6 +269,7 @@ struct ScrStreamState { * destroyed and runs its _destroy once construction completes. */ bool constructing, destroy_deferred; bool construct_finished; /* onConstruct ran: a second callback is ERR_MULTIPLE_CALLBACK */ + bool construct_called; /* the construct tick called the hook */ ScrError *destroy_err; /* owned: the deferred destroy's error */ ScrClosure *construct_cb; ScrStreamPlainInv construct_inv; @@ -7523,6 +7524,17 @@ void scr_stream_set_construct(ScrStream *s, ScrClosure *cb /*moves*/, ScrStreamP scr_stream_construct(s, cb, inv); } +void scr_stream_replace_construct(ScrStream *s, ScrClosure *cb /*moves*/, ScrStreamPlainInv inv) { + ScrStreamState *st = s->st; + if (!st->constructing || st->construct_called || !st->construct_cb) { + scr_closure_release(cb); + return; + } + scr_closure_release(st->construct_cb); + st->construct_cb = cb; + st->construct_inv = inv; +} + /* A class's _writev under options that may carry their own writev (the * option, an instance property in Node, wins). */ void scr_stream_set_writev_default(ScrStream *s, ScrClosure *cb /*moves*/, ScrStreamWritevInv inv) { @@ -7791,6 +7803,7 @@ static void scr_stream_run_tick(ScrStreamTick *t) { scr_stream_error_buffer(s); break; case SCR_ST_CONSTRUCT: { + st->construct_called = true; if (st->construct_cb) { st->construct_inv(st->construct_cb, s); /* a throw inside _construct reaches onConstruct on a tick */ diff --git a/packages/runtime/src/scr_stream.h b/packages/runtime/src/scr_stream.h index e368ec7ff..a4798c629 100644 --- a/packages/runtime/src/scr_stream.h +++ b/packages/runtime/src/scr_stream.h @@ -561,6 +561,10 @@ void scr_stream_set_writev_concat(ScrStream *s); /* The user construct option / _construct method (scheduled now; cb * moves), and the writev option / _writev method (cb moves). */ void scr_stream_set_construct(ScrStream *s, ScrClosure *cb, ScrStreamPlainInv inv); +/* `s._construct = fn` after construction: replaces a construct hook whose + * tick has not run yet (Node's constructNT reads _construct then), and has + * no effect otherwise (cb moves). */ +void scr_stream_replace_construct(ScrStream *s, ScrClosure *cb, ScrStreamPlainInv inv); void scr_stream_set_writev(ScrStream *s, ScrClosure *cb, ScrStreamWritevInv inv); void scr_stream_set_writev_default(ScrStream *s, ScrClosure *cb, ScrStreamWritevInv inv); /* A compiled writev whose entries declare `encoding` a string: entries diff --git a/tests/corpus/7466-stream-construct-assign.mjs b/tests/corpus/7466-stream-construct-assign.mjs new file mode 100644 index 000000000..37684afcd --- /dev/null +++ b/tests/corpus/7466-stream-construct-assign.mjs @@ -0,0 +1,46 @@ +// Assigning _construct after construction: Node reads it on the +// construction tick, so an assignment before that tick replaces the +// scheduled hook, and one after it, or on a stream without a hook, has no +// effect. +import { Readable, Writable } from "node:stream"; + +const w = new Writable({ + construct(cb) { + console.log("original construct"); + cb(); + }, + write(chunk, _enc, cb) { + console.log("write", chunk.toString()); + cb(); + }, +}); +w._construct = function (cb) { + console.log("replacement construct"); + cb(); +}; +w.write("a"); +w.end(); +w.on("finish", () => { + w._construct = function (cb) { + console.log("late construct"); + cb(); + }; + console.log("finish"); + readable(); +}); + +// A stream without a construct hook: the assignment has no effect. +function readable() { + const r = new Readable({ + read() { + this.push("x"); + this.push(null); + }, + }); + r._construct = function (cb) { + console.log("never"); + cb(); + }; + r.on("data", (d) => console.log("data", d.toString())); + r.on("end", () => console.log("end")); +} diff --git a/tests/diagnostics/stream-runtime-option-order.ts b/tests/diagnostics/stream-runtime-option-order.ts new file mode 100644 index 000000000..b3854e328 --- /dev/null +++ b/tests/diagnostics/stream-runtime-option-order.ts @@ -0,0 +1,28 @@ +import { Writable } from "node:stream"; + +// Runtime decodeStrings and defaultEncoding values are read before the +// other option members are evaluated, while Node evaluates the members in +// order: a value after a member with effects refuses. +let encoding = (process.env["STREAM_ENCODING"] ?? "utf8") as BufferEncoding; +let decode = process.env["STREAM_DECODE"] !== "0"; +function bump(): number { + encoding = "hex"; + decode = false; + return 16; +} +const sink = new Writable({ + highWaterMark: bump(), + defaultEncoding: encoding, + write(_chunk, _encoding, done) { + done(); + }, +}); +const other = new Writable({ + highWaterMark: bump(), + decodeStrings: decode, + write(_chunk, _encoding, done) { + done(); + }, +}); +sink.end(); +other.end(); diff --git a/tests/harness/__snapshots__/stream-runtime-option-order.ts.txt b/tests/harness/__snapshots__/stream-runtime-option-order.ts.txt new file mode 100644 index 000000000..8e08dfbf0 --- /dev/null +++ b/tests/harness/__snapshots__/stream-runtime-option-order.ts.txt @@ -0,0 +1,17 @@ +stream-runtime-option-order.ts:15:3 - error SC2020: 'the Writable option 'defaultEncoding' with a runtime value after an option value with effects' is part of the standard library types but has no scriptc lowering yet + + 14 | highWaterMark: bump(), + 15 | defaultEncoding: encoding, + | ^~~~~~~~~~~~~~~~~~~~~~~~~ + 16 | write(_chunk, _encoding, done) { + + hint: compute the earlier option values into consts before the options object (the runtime value is read before the other options run) + +stream-runtime-option-order.ts:22:3 - error SC2020: 'the Writable option 'decodeStrings' with a runtime value after an option value with effects' is part of the standard library types but has no scriptc lowering yet + + 21 | highWaterMark: bump(), + 22 | decodeStrings: decode, + | ^~~~~~~~~~~~~~~~~~~~~ + 23 | write(_chunk, _encoding, done) { + + hint: compute the earlier option values into consts before the options object (the runtime value is read before the other options run) \ No newline at end of file diff --git a/tests/harness/corpus-costs/7466-stream-construct-assign.mjs.json b/tests/harness/corpus-costs/7466-stream-construct-assign.mjs.json new file mode 100644 index 000000000..1c67bb43f --- /dev/null +++ b/tests/harness/corpus-costs/7466-stream-construct-assign.mjs.json @@ -0,0 +1 @@ +{ "plain": 1571, "san": 2074 } From 2f88f352816c22ac570af7720509cf2240efe1ff Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 01:56:34 -0500 Subject: [PATCH 08/53] Freeze records that other references still hold Object.freeze of a record that another reference may still hold stopped at compile time with SC2020 ("possibly-aliased value"), and freezing a record held in an untyped value threw SC2020 at run time. Node freezes the object itself: every later write through any reference throws a TypeError in strict mode code and leaves the object unchanged in sloppy-mode code. - A record a program may freeze keeps a frozen bit in its reference count word. A shape is freezable when a typed Object.freeze names it, or, in a program that freezes untyped values, when a value of it can be held as a live untyped reference. Only those shapes change: their release ignores the bit, and every write to them, keyed writes and deletes included, checks it first. - Object.freeze of a record held in an untyped value sets the bit and freezes the record's checked view, so Object.isFrozen, Object.isSealed and Object.isExtensible through that value answer for the frozen record. A typed Object.isFrozen of a record reads the bit. - Programs that never freeze a record compile to the same IR and LLVM. Corpus 7404 covers typed and untyped freezes, a recursive freeze through unknown, keyed writes and deletes, optional-field deletes, Object.assign, tuples, and frozen records that are released or collected as cycles, in strict mode code. 7405 covers the same writes in sloppy-mode code. Both match Node. Refs #1374 Co-authored-by: blurname <50226652+blurname@users.noreply.github.com> --- .../compiler/src/backend/llvm/lib-dispatch.ts | 13 + .../compiler/src/backend/llvm/lib-shared.ts | 2 + .../compiler/src/backend/llvm/shapes.test.ts | 37 ++ packages/compiler/src/backend/llvm/shapes.ts | 35 +- .../src/backend/llvm/stack-objects.ts | 7 +- .../lowering/expressions/element-access.ts | 2 +- .../src/frontend/lowering/frozen-records.ts | 328 ++++++++++++++++++ .../src/frontend/lowering/lower-calls.ts | 24 +- .../src/frontend/lowering/lower-stmts.ts | 28 +- .../compiler/src/frontend/lowering/lowerer.ts | 10 +- packages/compiler/src/ir/builtin-effects.ts | 1 + .../compiler/src/ir/builtin-signatures.ts | 12 + packages/compiler/src/ir/ir.ts | 4 + .../tests/corpus/7404-frozen-records.ts.json | 6 + .../7405-frozen-records-sloppy.cjs.json | 6 + packages/runtime/src/scr_cycle.c | 22 +- packages/runtime/src/scr_dyn.h | 7 + packages/runtime/src/scr_json.c | 71 +++- packages/runtime/src/scr_json.test.c | 13 + packages/runtime/src/scr_rc.h | 26 +- tests/corpus/7404-frozen-records.ts | 124 +++++++ tests/corpus/7405-frozen-records-sloppy.cjs | 34 ++ .../corpus-costs/7404-frozen-records.ts.json | 1 + .../7405-frozen-records-sloppy.cjs.json | 1 + 24 files changed, 776 insertions(+), 38 deletions(-) create mode 100644 packages/compiler/src/frontend/lowering/frozen-records.ts create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7404-frozen-records.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7405-frozen-records-sloppy.cjs.json create mode 100644 tests/corpus/7404-frozen-records.ts create mode 100644 tests/corpus/7405-frozen-records-sloppy.cjs create mode 100644 tests/harness/corpus-costs/7404-frozen-records.ts.json create mode 100644 tests/harness/corpus-costs/7405-frozen-records-sloppy.cjs.json diff --git a/packages/compiler/src/backend/llvm/lib-dispatch.ts b/packages/compiler/src/backend/llvm/lib-dispatch.ts index cebe661d4..4f1a4fef8 100644 --- a/packages/compiler/src/backend/llvm/lib-dispatch.ts +++ b/packages/compiler/src/backend/llvm/lib-dispatch.ts @@ -770,6 +770,19 @@ export function emitLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlValue { return { name: t, type: e.type }; } if (e.fn === "bytes.absent") return host.own({ name: "null", type: e.type }); + if (e.fn === "record.isFrozen") { + // The frozen bit of the record's rc word (scr_rc.h SCR_RC_FROZEN) is + // its second-highest bit; an immortal word sets the highest one too. + const value = host.emitReadReceiver(e.args[0]!); + const S = host.sizeType; + const rc = host.B.tmp(), + top = host.B.tmp(), + t = host.B.tmp(); + host.B.line(`${rc} = load ${S}, ptr ${value.name}`); + host.B.line(`${top} = lshr ${S} ${rc}, ${S === "i64" ? 62 : 30}`); + host.B.line(`${t} = icmp eq ${S} ${top}, 1`); + return { name: t, type: e.type }; + } const prefix = e.fn.slice(0, e.fn.indexOf(".")) as LibCallPrefix; // events.once builds typed listener adapters beside the emitter calls. if (e.fn === "events.once" || e.fn === "events.onceSignal") diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index 4a9578e5e..6fbad7e11 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -1014,6 +1014,8 @@ export const LIB_FN_SYMS: Record = { "class.descriptorHasValue": "scr_class_descriptor_has_value", "class.throwReadOnly": "scr_class_throw_read_only", "class.throwNotExtensible": "scr_class_throw_not_extensible", + "record.freeze": "scr_record_freeze", + "record.throwFrozen": "scr_record_throw_frozen", "dyn.nativeSetNew": "scr_dyn_native_set_new", "dyn.nativeSetIs": "scr_dyn_native_set_is", "dyn.nativeMapIs": "scr_dyn_native_map_is", diff --git a/packages/compiler/src/backend/llvm/shapes.test.ts b/packages/compiler/src/backend/llvm/shapes.test.ts index f7327a688..7af16ab83 100644 --- a/packages/compiler/src/backend/llvm/shapes.test.ts +++ b/packages/compiler/src/backend/llvm/shapes.test.ts @@ -14,6 +14,7 @@ import { type ShapeHost, vAdapters, } from "./shapes.js"; +import { mangleRecordRelease } from "../mangle.js"; import { NullableUnions } from "./nullable-unions.js"; import { InlineUnionFields } from "./inline-unions.js"; @@ -315,6 +316,42 @@ describe("emitted object releases", () => { root.indexOf("call void @scr_cyc_on_release(ptr %o)"), ); }); + + test("freezable records count their references without the frozen bit", () => { + const mod: IrModule = { + irVersion: 17, + sourceFile: "frozen.ts", + entry: "main", + functions: [], + records: [ + { id: "plain", fields: [{ name: "n", type: F64 }] }, + { id: "frozen", fields: [{ name: "n", type: F64 }], freezable: true }, + ], + }; + const body = (defs: string, name: string): string => + new RegExp(`^define internal void @${name}\\([^]*?^}`, "m").exec(defs)?.[0] ?? ""; + for (const [pointerBits, notFrozen, sharedBelow] of [ + [64, "-4611686018427387905", "-4611686018427387907"], + [32, "-1073741825", "-1073741827"], + ] as const) { + const { host } = declarationHost(pointerBits); + const defs = emitRecordShapes({ ...host, objectAudit: false }, mod).defs.join("\n"); + const S = pointerBits === 64 ? "i64" : "i32"; + const plain = body(defs, mangleRecordRelease("plain")); + expect(plain).toContain(`%shared = icmp ult ${S} %rcm2, -3`); + expect(plain).not.toContain("%refs"); + // The immortal word's count is ~FROZEN, which the compare sends to the + // immortal test, like counts 0 and 1. + const fast = body(defs, mangleRecordRelease("frozen")); + expect(fast).toContain(`%refs = and ${S} %rc, ${notFrozen}`); + expect(fast).toContain(`%rcm2 = sub ${S} %refs, 2`); + expect(fast).toContain(`%shared = icmp ult ${S} %rcm2, ${sharedBelow}`); + expect(fast).toContain(`%imm = icmp eq ${S} %rc, -1`); + const slow = body(defs, `${mangleRecordRelease("frozen")}_slow`); + expect(slow).toContain(`%refs = and ${S} %n, ${notFrozen}`); + expect(slow).toContain(`%dead = icmp eq ${S} %refs, 0`); + } + }); }); describe("live-object audit notes", () => { diff --git a/packages/compiler/src/backend/llvm/shapes.ts b/packages/compiler/src/backend/llvm/shapes.ts index d13f80a78..0997d0ace 100644 --- a/packages/compiler/src/backend/llvm/shapes.ts +++ b/packages/compiler/src/backend/llvm/shapes.ts @@ -1527,6 +1527,17 @@ export function destroyDefinitions( ]; } +/** The frozen bit of a freezable record's rc word (scr_rc.h SCR_RC_FROZEN): + * bit 62 on 64-bit targets, bit 30 on 32-bit ones. */ +function recordFrozenBit(sizeType: RcCount): bigint { + return 1n << (sizeType === "i64" ? 62n : 30n); +} + +/** `value` as a signed LLVM integer literal of `sizeType`. */ +function sizeLiteral(sizeType: RcCount, value: bigint): string { + return BigInt.asIntN(sizeType === "i64" ? 64 : 32, value).toString(); +} + /** Common NULL/immortal/decrement skeleton for ordinary object releases. * `freeBody` owns the zero-ref teardown and must leave the current block at * its end; traced objects also get the possible-cycle-root branch. @@ -1543,13 +1554,16 @@ export function releaseBody( bounded = true, immortal = false, chained = false, + frozen = false, ): string[] { const S = host.sizeType; if (bounded) host.declare(`declare void @scr_rc_destroy(ptr, ptr)`); const destroy = `${fnName}_destroy`; + // A freezable record's count is the word without its frozen bit. + const notFrozen = sizeLiteral(S, ~recordFrozenBit(S)); const lines = [ ...(bounded ? destroyDefinitions(destroy, freeBody, chained) : []), - ...releaseFastPath(host, fnName, `${fnName}_slow`, traced, comment, immortal), + ...releaseFastPath(host, fnName, `${fnName}_slow`, traced, comment, immortal, S, frozen), `define internal void @${fnName}_slow(ptr %o) noinline ${FN_ATTRS} {${comment ? ` ; ${comment}` : ""}`, `entry:`, ` %isnull = icmp eq ptr %o, null`, @@ -1561,7 +1575,9 @@ export function releaseBody( `dec:`, ` %n = sub ${S} %rc, 1`, ` store ${S} %n, ptr %o`, - ` %dead = icmp eq ${S} %n, 0`, + ...(frozen + ? [` %refs = and ${S} %n, ${notFrozen}`, ` %dead = icmp eq ${S} %refs, 0`] + : [` %dead = icmp eq ${S} %n, 0`]), ` br i1 %dead, label %free, label %${traced ? "root" : "done"}`, `free:`, ...(traced ? cycleDeadLines(host, "teardown") : []), @@ -1591,9 +1607,14 @@ export function releaseFastPath( comment = "", immortal = false, count: RcCount = host.sizeType, + frozen = false, ): string[] { const S = count; if (immortal) return immortalRcBody(host, fnName, "void", comment); + // A freezable record compares its count without the frozen bit: an + // immortal word's count is then ~FROZEN, which the same compare sends to + // `slow`. + const notFrozen = ~recordFrozenBit(S); return [ `define internal void @${fnName}(ptr %o) ${FN_ATTRS} {${comment ? ` ; ${comment} (fast path)` : ""}`, `entry:`, @@ -1601,8 +1622,13 @@ export function releaseFastPath( ` br i1 %isnull, label %done, label %check`, `check:`, ` %rc = load ${S}, ptr %o`, - ` %rcm2 = sub ${S} %rc, 2`, - ` %shared = icmp ult ${S} %rcm2, -3`, + ...(frozen + ? [ + ` %refs = and ${S} %rc, ${sizeLiteral(S, notFrozen)}`, + ` %rcm2 = sub ${S} %refs, 2`, + ` %shared = icmp ult ${S} %rcm2, ${sizeLiteral(S, notFrozen - 2n)}`, + ] + : [` %rcm2 = sub ${S} %rc, 2`, ` %shared = icmp ult ${S} %rcm2, -3`]), ` br i1 %shared, label %dec, label %slow`, `dec:`, ` %n = sub ${S} %rc, 1`, @@ -1810,6 +1836,7 @@ export function emitRecordShapes( refMembers.some((member) => needsBoundedRelease(member.type)), false, chained, + shape.freezable === true, ), ``, ); diff --git a/packages/compiler/src/backend/llvm/stack-objects.ts b/packages/compiler/src/backend/llvm/stack-objects.ts index c7a58a7ea..ae2b904db 100644 --- a/packages/compiler/src/backend/llvm/stack-objects.ts +++ b/packages/compiler/src/backend/llvm/stack-objects.ts @@ -44,8 +44,9 @@ export type StackObjectFieldOwner = (fields: readonly StackObjectField[]) => voi * standalone scalar class whose constructor never lets `this` escape * (record-results.ts). The caller proves that every use of the value only * projects it. Record fields may own references; the frame releases them - * (stackObjectFields). Published shapes keep heap storage: their field - * guards treat the immortal header as a frozen object. */ + * (stackObjectFields). Published and freezable shapes keep heap storage: + * their field guards treat the immortal header as a frozen object, and a + * frozen record keeps its frozen bit in a counted header. */ export function canStackObject(host: LlvmEmitterContext, e: IrExpr): e is StackObjectSource { // A ThreadSlots key keeps its slot index for its lifetime and has its // values released when it is freed: it lives on the heap. @@ -53,6 +54,8 @@ export function canStackObject(host: LlvmEmitterContext, e: IrExpr): e is StackO return false; if (e.type.kind === "record" && host.publishedTypes?.recordGuarded(e.type.shapeId) === true) return false; + if (e.type.kind === "record" && host.recordsById.get(e.type.shapeId)?.freezable === true) + return false; // A producer with a result-slot entry builds its fresh result in the // caller's storage. if (e.kind === "call") return host.fnByName.has(resultEntryName(e.callee)); diff --git a/packages/compiler/src/frontend/lowering/expressions/element-access.ts b/packages/compiler/src/frontend/lowering/expressions/element-access.ts index 0be9be81a..7a8fd646e 100644 --- a/packages/compiler/src/frontend/lowering/expressions/element-access.ts +++ b/packages/compiler/src/frontend/lowering/expressions/element-access.ts @@ -1022,7 +1022,7 @@ function lowerGuardedRecordWrite( const value = stabilized(lowerer, lowerer.lowerExpr(expr.right), "%recordValue", receiver.stmts); const write = lowerer.withExpressionOverride(target.expression, receiver.present, () => lowerer.withExpressionOverride(target.argumentExpression, key, () => - lowerer.withExpressionOverride(expr.right, value, () => lowerElementWrite(lowerer, expr)), + lowerer.withExpressionOverride(expr.right, value, () => lowerer.lowerElementWrite(expr)), ), ); return { diff --git a/packages/compiler/src/frontend/lowering/frozen-records.ts b/packages/compiler/src/frontend/lowering/frozen-records.ts new file mode 100644 index 000000000..f61de39ec --- /dev/null +++ b/packages/compiler/src/frontend/lowering/frozen-records.ts @@ -0,0 +1,328 @@ +import * as ts from "../ts7/adapter.js"; +import { + BOOL, + VOID, + shapeHasAccessorSlots, + type IrExpr, + type IrFunction, + type IrStmt, + type IrType, + type SrcLoc, +} from "../../ir/ir.js"; +import { numLit, strLit, varRef } from "../../ir/build.js"; +import { everyStmtList, transformStmtList } from "../../ir/traverse.js"; +import type { Lowerer } from "./lowerer.js"; +import { recordHasOwnHelper } from "./lower-calls.js"; +import { isStrictCode } from "./strict-mode.js"; + +/* Object.freeze of records that other references can still write. Only the + * shapes a program may freeze pay for it: their values keep a frozen bit in + * the rc word (scr_rc.h SCR_RC_FROZEN), their release ignores the bit, and + * every write to them checks it first (guardFrozenRecordWrites). A shape is + * freezable when a static Object.freeze names it, or, in a program that + * freezes untyped values, when a value of it or of a type reaching it can be + * held as a live untyped reference. */ + +type RecordType = Extract; +type RecordWrite = Extract; + +/** Object.freeze of a record: sets its frozen bit and answers the record. */ +export function lowerRecordFreeze( + lowerer: Lowerer, + target: IrExpr, + type: RecordType, + loc: SrcLoc, +): IrExpr { + const slot = lowerer.declareHiddenLocal("%freezeTarget", type); + const receiver = (): IrExpr => varRef(slot.id, type, loc); + return { + kind: "seqExpr", + stmts: [ + { kind: "varDecl", localId: slot.id, init: target, loc }, + { + kind: "exprStmt", + expr: { kind: "libCall", fn: "record.freeze", args: [receiver()], type: VOID, loc }, + loc, + }, + ], + result: receiver(), + type, + loc, + }; +} + +function isRecordWrite(s: IrStmt): s is RecordWrite { + return s.kind === "recordSet" || s.kind === "recordKeySet" || s.kind === "recordKeyDelete"; +} + +/** The key of a record write's source range: strict mode is lexical, so + * every write lowered from the same range shares its mode. */ +function writeKey(loc: SrcLoc): string { + return `${loc.file}\0${loc.start}\0${loc.end}`; +} + +/** Records the record writes of `stmt`, lowered from `node`, as sloppy-mode + * writes when `node` is sloppy-mode code: a write to a frozen record then + * leaves it unchanged instead of throwing. `deletes` says `stmt` is the + * absent-state write of `delete r.f`, which Node refuses only while the + * field is present. */ +export function markSloppyRecordWrites( + lowerer: Lowerer, + stmt: IrStmt, + node: ts.Node, + deletes = false, +): IrStmt { + if (deletes) lowerer.recordFieldDeletes.add(writeKey(stmt.loc)); + if (isStrictCode(node)) return stmt; + everyStmtList([stmt], { + stmt: (s) => { + if (s.kind === "recordSet" || s.kind === "recordKeySet" || s.kind === "recordKeyDelete") + lowerer.sloppyRecordWrites.add(writeKey(s.loc)); + return true; + }, + expr: () => true, + }); + return stmt; +} + +/** Whether a field write's value can be an optional field's absent state, + * which only a field write itself can hold. */ +function holdsAbsentState(e: IrExpr): boolean { + if (e.kind === "fieldAbsent") return true; + return e.kind === "ternary" && (holdsAbsentState(e.then) || holdsAbsentState(e.else_)); +} + +/** The shapes whose values the program may freeze. */ +function freezableShapes(lowerer: Lowerer, functions: IrFunction[]): Set { + const shapes = new Set(); + const live: IrType[] = []; + let dynFreeze = false; + for (const fn of functions) + everyStmtList(fn.body, { + expr: (e) => { + if (e.kind === "libCall" && e.fn === "record.freeze") { + const target = e.args[0]; + if (target && target.type.kind === "record") shapes.add(target.type.shapeId); + } else if (e.kind === "libCall" && e.fn === "dyn.freeze") dynFreeze = true; + else if (e.kind === "dynFrom") live.push(e.value.type); + return true; + }, + stmt: () => true, + }); + if (!dynFreeze) return shapes; + // An untyped reference to a class instance, or to a record or array + // boxed live, reaches the values its fields and elements hold as further + // live references. Class instances also cross into untyped code through + // compiled callbacks, so every class is a root. + const stack = [...live]; + for (const info of lowerer.classes.values()) + for (const field of info.def.fields) stack.push(field.type); + const seen = new Set(); + while (stack.length > 0) { + const type = stack.pop()!; + switch (type.kind) { + case "record": { + if (seen.has(`record:${type.shapeId}`)) break; + seen.add(`record:${type.shapeId}`); + shapes.add(type.shapeId); + const shape = lowerer.shapes.get(type.shapeId); + for (const field of shape?.fields ?? []) stack.push(field.type); + if (shape?.indexValue) stack.push(shape.indexValue); + break; + } + case "union": + if (seen.has(`union:${type.unionId}`)) break; + seen.add(`union:${type.unionId}`); + for (const arm of lowerer.unions.get(type.unionId)?.arms ?? []) stack.push(arm); + break; + case "array": + case "set": + stack.push(type.elem); + break; + case "map": + stack.push(type.key, type.value); + break; + case "promise": + stack.push(type.inner); + break; + case "generator": + stack.push(type.yieldT, type.retT, type.nextT); + break; + case "func": + for (const param of type.params) stack.push(param); + stack.push(type.ret); + break; + default: + break; + } + } + return shapes; +} + +/** The guarded form of a write to a freezable record: the receiver and the + * operands are evaluated first, in source order, then the frozen bit decides + * between the write and `reject`. */ +function guardedWrite( + receiver: IrExpr, + decls: IrStmt[], + reject: IrStmt[], + store: IrStmt, + loc: SrcLoc, +): IrStmt { + return { + kind: "block", + body: [ + ...decls, + { + kind: "if", + cond: { kind: "libCall", fn: "record.isFrozen", args: [receiver], type: BOOL, loc }, + then: reject, + else_: [store], + loc, + }, + ], + loc, + }; +} + +/** Checks the frozen bit before every write to a record of a freezable + * shape. A write to a frozen record throws Node's TypeError in strict mode + * code; in sloppy-mode code it leaves the record unchanged. Deleting a key + * the record does not have succeeds either way. */ +export function guardFrozenRecordWrites(lowerer: Lowerer, functions: IrFunction[]): void { + const freezable = freezableShapes(lowerer, functions); + for (const id of freezable) { + const shape = lowerer.shapes.get(id); + if (shape) shape.freezable = true; + } + if (freezable.size === 0) return; + const lifted = lowerer.liftedFns.length; + for (const fn of functions) { + const local = (type: IrType, loc: SrcLoc): Extract => { + const id = `%frozenWrite.${fn.locals.length}`; + fn.locals.push({ id, name: id, type, mutable: false }); + return { kind: "varRef", localId: id, type, loc }; + }; + const guarded = new Set(); + fn.body = transformStmtList(fn.body, { + stmt: (statement) => { + if (!isRecordWrite(statement) || guarded.has(statement)) return statement; + if (!freezable.has(statement.shapeId)) return statement; + const sloppy = lowerer.sloppyRecordWrites.has(writeKey(statement.loc)); + const deletes = + statement.kind === "recordSet" && lowerer.recordFieldDeletes.has(writeKey(statement.loc)); + const loc = statement.loc; + const shapeId = statement.shapeId; + const receiver = local(statement.obj.type, loc); + const decls: IrStmt[] = [ + { kind: "varDecl", localId: receiver.localId, init: statement.obj, loc }, + ]; + const owner = strLit( + lowerer.shapes.get(shapeId)?.tuple ? "[object Array]" : "#", + loc, + ); + const thrown = (key: IrExpr, operation: number): IrStmt => ({ + kind: "exprStmt", + expr: { + kind: "libCall", + fn: "record.throwFrozen", + args: [key, owner, numLit(operation, loc)], + type: VOID, + loc, + }, + loc, + }); + if (statement.kind === "recordSet") { + let value = statement.value; + if (!holdsAbsentState(value)) { + const slot = local(value.type, loc); + decls.push({ kind: "varDecl", localId: slot.localId, init: value, loc }); + value = slot; + } + const store: IrStmt = { + kind: "recordSet", + obj: receiver, + shapeId, + field: statement.field, + value, + loc, + }; + guarded.add(store); + const field = strLit(statement.field, loc); + // `delete r.f` of an optional field writes its absent state: Node + // only refuses it when the field is present. + const reject: IrStmt[] = sloppy + ? [] + : deletes + ? [ + { + kind: "if", + cond: lowerer.recordFieldPresent(receiver, shapeId, statement.field, loc), + then: [thrown(field, 2)], + else_: null, + loc, + }, + ] + : [thrown(field, 0)]; + return guardedWrite(receiver, decls, reject, store, loc); + } + const key = local(statement.key.type, loc); + decls.push({ kind: "varDecl", localId: key.localId, init: statement.key, loc }); + const named = key.type.kind === "string"; + const present: IrExpr = named + ? { + kind: "call", + callee: recordHasOwnHelper(lowerer, shapeId, loc), + args: [receiver, key], + type: BOOL, + loc, + } + : { kind: "boolLit", value: true, type: BOOL, loc }; + const text: IrExpr = named ? key : strLit("[key]", loc); + if (statement.kind === "recordKeySet") { + const value = local(statement.value.type, loc); + decls.push({ kind: "varDecl", localId: value.localId, init: statement.value, loc }); + const store: IrStmt = { + kind: "recordKeySet", + obj: receiver, + shapeId, + key, + value, + ...(statement.overflowOnly ? { overflowOnly: true as const } : {}), + loc, + }; + guarded.add(store); + const reject: IrStmt[] = sloppy + ? [] + : [ + { + kind: "if", + cond: present, + then: [thrown(text, 0)], + else_: [thrown(text, 1)], + loc, + }, + ]; + return guardedWrite(receiver, decls, reject, store, loc); + } + const store: IrStmt = { kind: "recordKeyDelete", obj: receiver, shapeId, key, loc }; + guarded.add(store); + const reject: IrStmt[] = sloppy + ? [] + : [{ kind: "if", cond: present, then: [thrown(text, 2)], else_: null, loc }]; + return guardedWrite(receiver, decls, reject, store, loc); + }, + expr: (expression) => expression, + }); + } + // The key-presence helpers the guards call. + for (const helper of lowerer.liftedFns.slice(lifted)) functions.push(helper); +} + +/** Whether a statically typed Object.freeze can freeze a value of `type`: + * a record whose shape has no accessor properties. */ +export function freezableRecordType(lowerer: Lowerer, type: IrType): type is RecordType { + if (type.kind !== "record") return false; + const shape = lowerer.shapes.get(type.shapeId); + return shape !== undefined && !shapeHasAccessorSlots(shape); +} diff --git a/packages/compiler/src/frontend/lowering/lower-calls.ts b/packages/compiler/src/frontend/lowering/lower-calls.ts index b595aaa43..e2ed5f9fe 100644 --- a/packages/compiler/src/frontend/lowering/lower-calls.ts +++ b/packages/compiler/src/frontend/lowering/lower-calls.ts @@ -245,6 +245,7 @@ import { import { fenceSymbolFieldCopy } from "./symbol-fields.js"; import { lowerClassDataDescriptor, lowerClassDescriptorRead } from "./class-descriptors.js"; import { lowerClassFieldDefine, lowerClassIntegrity } from "./class-attributes.js"; +import { freezableRecordType, lowerRecordFreeze } from "./frozen-records.js"; import { classStaticDataFor } from "./class-static-data.js"; import { lowerUrlFactory } from "./lower-url.js"; import { nullishValueUnitOf } from "./binding-analysis.js"; @@ -7206,7 +7207,7 @@ export function recordKeysArrayCall( * (a key is own exactly when Object.keys would list it — the two share * the test), everything else true. Unmatched keys probe the overflow * map when present, so explicit undefined values remain own keys. */ -function recordHasOwnHelper(lowerer: Lowerer, shapeId: string, loc: SrcLoc): string { +export function recordHasOwnHelper(lowerer: Lowerer, shapeId: string, loc: SrcLoc): string { const key = `obj.hasOwn:${shapeId}`; const existing = lowerer.arrHofHelpers.get(key); if (existing) return existing; @@ -8455,10 +8456,23 @@ function lowerObjectStaticCall( // the argument — no alias exists, so the frozen bit is unobservable // (writes through the Readonly result are compile errors, and no // other reference can write). Primitives pass through per ES2015. - // Aliased objects keep a fence: a later write through the original - // reference would need the runtime frozen bit (strict mode throws). + // Aliased records get the frozen bit of their rc word (frozen-records.ts); + // other aliased objects keep a fence: a later write through the original + // reference would need a runtime frozen bit (strict mode throws). if (member === "isFrozen" && call.arguments.nodes.length === 1) { - const value = lowerer.lowerExprExpecting(call.arguments.nodes[0]!, DYN); + const argNode = call.arguments.nodes[0]!; + if (tryLowerExpression(lowerer, argNode)?.type.kind === "record") { + const record = lowerer.lowerExpr(argNode); + if (record.type.kind === "record") + return { + kind: "libCall", + fn: "record.isFrozen", + args: [record], + type: BOOL, + loc: locOf(call), + }; + } + const value = lowerer.lowerExprExpecting(argNode, DYN); return { kind: "libCall", fn: "dyn.isFrozen", args: [value], type: BOOL, loc: locOf(call) }; } if (member === "freeze") { @@ -8508,6 +8522,8 @@ function lowerObjectStaticCall( if (fnIntegrity) return fnIntegrity; const integrity = lowerClassIntegrity(lowerer, call, "freeze", value); if (integrity) return integrity; + if (freezableRecordType(lowerer, value.type)) + return lowerRecordFreeze(lowerer, value, value.type, locOf(call)); lowerer.noLowering( "Object.freeze of a possibly-aliased value", call, diff --git a/packages/compiler/src/frontend/lowering/lower-stmts.ts b/packages/compiler/src/frontend/lowering/lower-stmts.ts index 66ecea602..e66d0e855 100644 --- a/packages/compiler/src/frontend/lowering/lower-stmts.ts +++ b/packages/compiler/src/frontend/lowering/lower-stmts.ts @@ -105,6 +105,7 @@ import { requireSpecOf, } from "../program.js"; import { isStrictCode } from "./strict-mode.js"; +import { markSloppyRecordWrites } from "./frozen-records.js"; import { COMPOUND_ASSIGN_OPS, type CompoundOp, @@ -7160,7 +7161,11 @@ function lowerDeleteStatement(lowerer: Lowerer, expr: ts.DeleteExpression): IrSt if (obj.type.kind === "record") { const shape = lowerer.shapes.get(obj.type.shapeId); if (shape?.indexValue && shape.fields.length === 0 && !shape.tuple) { - return { kind: "recordKeyDelete", obj, shapeId: obj.type.shapeId, key: lowerKey(), loc }; + return markSloppyRecordWrites( + lowerer, + { kind: "recordKeyDelete", obj, shapeId: obj.type.shapeId, key: lowerKey(), loc }, + expr, + ); } // `delete r.f` of a declared OPTIONAL field (undefined-armed slot) // writes the field's ABSENT state: a monomorphic shape cannot remove @@ -7181,14 +7186,19 @@ function lowerDeleteStatement(lowerer: Lowerer, expr: ts.DeleteExpression): IrSt if (field) { const absent = lowerer.absentFieldValue(field.type, loc); if (absent) { - return { - kind: "recordSet", - obj, - shapeId: obj.type.shapeId, - field: field.name, - value: absent, - loc, - }; + return markSloppyRecordWrites( + lowerer, + { + kind: "recordSet", + obj, + shapeId: obj.type.shapeId, + field: field.name, + value: absent, + loc, + }, + expr, + true, + ); } } if (shape?.indexValue) { diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index a765c91ee..ba6d1214f 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -150,6 +150,7 @@ import { defaultAfterUndefined, lowerStaticallyUndefinedArgument } from "./optio import { type ProtocolDispatch } from "./class-protocol-dispatch.js"; import { ClassDynamicDispatch, trackClassFieldCreation } from "./class-dynamic-dispatch.js"; import { guardClassAttributeWrites } from "./class-attributes.js"; +import { guardFrozenRecordWrites, markSloppyRecordWrites } from "./frozen-records.js"; import { applyHandleProperties } from "./handle-properties.js"; import { ClassProtocols } from "./class-protocols.js"; import { earlyReadVarSymbols } from "./var-hoisting.js"; @@ -3241,6 +3242,10 @@ export class Lowerer { // Stack of function contexts (bottom = the function being declared at // top level, top = the innermost nested function currently lowering). fnStack: FnCtx[] = []; + /** Source ranges of record writes in sloppy-mode code, and of the + * absent-state writes of `delete r.f` (markSloppyRecordWrites). */ + readonly sloppyRecordWrites = new Set(); + readonly recordFieldDeletes = new Set(); readonly liftedFns: IrFunction[] = []; /** The first lifted function of each name (liftedFnNamed), for the * liftedFns indexed so far. liftedFns only grows. */ @@ -7689,6 +7694,7 @@ export class Lowerer { finalizePrivateBrands(this, functions); trackClassFieldCreation(this, functions); guardClassAttributeWrites(this, functions); + guardFrozenRecordWrites(this, functions); applyHandleProperties(this, functions); // Globals typed by a class that never REGISTERED (a JS class whose @@ -12978,7 +12984,7 @@ export class Lowerer { } lowerElementWrite(expr: ts.BinaryExpression): IrStmt { - return lowerElementWrite(this, expr); + return markSloppyRecordWrites(this, lowerElementWrite(this, expr), expr); } lowerLiveWebValue(node: ts.Expression): IrExpr { @@ -13375,7 +13381,7 @@ export class Lowerer { } fieldSetStmt(target: FieldTarget, value: IrExpr, loc: SrcLoc, blame: ts.Node): IrStmt { - return fieldSetStmt(this, target, value, loc, blame); + return markSloppyRecordWrites(this, fieldSetStmt(this, target, value, loc, blame), blame); } lowerFieldCompound( diff --git a/packages/compiler/src/ir/builtin-effects.ts b/packages/compiler/src/ir/builtin-effects.ts index 9be819dd3..0505129fb 100644 --- a/packages/compiler/src/ir/builtin-effects.ts +++ b/packages/compiler/src/ir/builtin-effects.ts @@ -687,6 +687,7 @@ export const MAY_THROW_LIB_FNS: ReadonlySet = new Set([ "class.defineField", "class.throwReadOnly", "class.throwNotExtensible", + "record.throwFrozen", "dyn.toString", "dyn.valueOf", "string.fromCodePoint", diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index e60a6600b..1b2c1082d 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -271,6 +271,18 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ /** Throws Node's TypeError for a class field initializer on a * non-extensible instance: the field name. */ "class.throwNotExtensible": { argTypes: [STRING], result: VOID }, + /** Object.freeze of a record whose shape is freezable (frozen-records.ts): + * sets the frozen bit in its rc word (scr_record_freeze). The receiver is + * any record type. */ + "record.freeze": { argTypes: [null], result: VOID }, + /** Whether a record is frozen: its rc word's frozen bit, set by + * record.freeze and dyn.freeze of a capsule. Emitted inline. The receiver + * is any record type. */ + "record.isFrozen": { argTypes: [null], result: BOOL }, + /** Throws Node's TypeError for a write to a frozen record: the key, the + * object's text in the message ("#") and the operation (0 assign, + * 1 add, 2 delete). */ + "record.throwFrozen": { argTypes: [STRING, STRING, F64], result: VOID }, "dyn.nativeSetNew": { argTypes: [DYN], result: DYN }, "dyn.nativeSetIs": { argTypes: [DYN], result: BOOL }, "dyn.nativeMapIs": { argTypes: [DYN], result: BOOL }, diff --git a/packages/compiler/src/ir/ir.ts b/packages/compiler/src/ir/ir.ts index 63286ff54..3211e48f3 100644 --- a/packages/compiler/src/ir/ir.ts +++ b/packages/compiler/src/ir/ir.ts @@ -1685,6 +1685,10 @@ export interface IrRecordShape { * dynCheck), and JSON serialization appends overflow entries after the * declared fields in insertion order. */ indexValue?: IrType; + /** Values of this shape may be frozen (frozen-records.ts): their rc word + * can carry the frozen bit, which their release ignores, and every write + * to them checks it. */ + freezable?: true; /** Field names in FIRST-SEEN declaration order (the checker's property * order of the first ts.Type interned to this shape) — metadata, NOT * part of the interned identity: a later structurally-equal type with a diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7404-frozen-records.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7404-frozen-records.ts.json new file mode 100644 index 000000000..d85f2aab5 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7404-frozen-records.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7404-frozen-records.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7405-frozen-records-sloppy.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7405-frozen-records-sloppy.cjs.json new file mode 100644 index 000000000..3d87d019b --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7405-frozen-records-sloppy.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7405-frozen-records-sloppy.cjs" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_cycle.c b/packages/runtime/src/scr_cycle.c index 6dcd546b0..241164705 100644 --- a/packages/runtime/src/scr_cycle.c +++ b/packages/runtime/src/scr_cycle.c @@ -170,7 +170,8 @@ void scr_weak_dispose(void *object) { #include /* Every heap object's first member is `size_t rc`; the collector works on - * its count (SCR_RC_COUNT). */ + * its count (SCR_RC_COUNT), and tests it without a frozen record's bit + * (SCR_RC_REFS). */ #define SCR_RC(obj) SCR_RC_COUNT(obj) static void scr_cyc_oom(void) { @@ -1308,7 +1309,8 @@ static void scr_mg_visit(void *child, void *ctx) { scr_mg_skip_edge(child); return; } - size_t rc = --SCR_RC(child); + --SCR_RC(child); + size_t rc = SCR_RC_REFS(child); if (scr_cyc_color(h) == SCR_CYC_GRAY) { if (rc == 0) { scr_gray_referenced--; @@ -1336,7 +1338,7 @@ static inline __attribute__((always_inline)) void scr_mark_gray(void *obj, uintp )) h->buf = scr_gray_comp; if (scr_white.n < SCR_CYC_GRAY_LIST_MAX) scr_cyc_push_gray(&scr_white, obj); - if (SCR_RC(obj) > 0) { + if (SCR_RC_REFS(obj) > 0) { scr_gray_referenced++; scr_comp_referenced++; } @@ -1408,7 +1410,7 @@ static void scr_scan_visit(void *child, void *ctx) { static void scr_scan(void *obj, uintptr_t depth) { ScrCycHdr *h = scr_cyc_hdr(obj); if (scr_cyc_color(h) != SCR_CYC_GRAY) return; - if (SCR_RC(obj) > 0) { + if (SCR_RC_REFS(obj) > 0) { scr_restore_from(obj); return; } @@ -1476,7 +1478,7 @@ static void scr_xg_destroy(void *obj) { } static SCR_CYC_COMPACT void scr_xg_release(void *obj) { - if (SCR_RC(obj) > 1) { + if (SCR_RC_REFS(obj) > 1) { SCR_RC(obj) -= 1; scr_cyc_on_release(obj); /* lost a reference: a possible cycle root */ return; @@ -1768,14 +1770,14 @@ static __attribute__((noinline)) size_t scr_yin_collect(void) { if (scr_cyc_gen(h) != SCR_CYC_YOUNG || !scr_cyc_is_buffered(h) || scr_cyc_buf_index(h) != k || scr_cyc_color(h) != SCR_CYC_BLACK) scr_trap("scriptc: corrupt young nursery entry\n"); - if (SCR_RC(v[k]) == 0 || SCR_RC(v[k]) < scr_yin_counts[k]) + if (SCR_RC_REFS(v[k]) == 0 || SCR_RC_REFS(v[k]) < scr_yin_counts[k]) scr_trap("scriptc: young object with uncounted references\n"); } #endif for (size_t k = 0; k < n; k++) { void *obj = v[k]; ScrCycHdr *h = scr_cyc_hdr(obj); - if (scr_cyc_color(h) == SCR_CYC_GRAY || SCR_RC(obj) <= scr_yin_counts[k]) continue; + if (scr_cyc_color(h) == SCR_CYC_GRAY || SCR_RC_REFS(obj) <= scr_yin_counts[k]) continue; scr_cyc_set_color(h, SCR_CYC_GRAY); scr_yin_push(&scr_yin_stack, obj); while (scr_yin_stack.n) { @@ -1799,7 +1801,7 @@ static __attribute__((noinline)) size_t scr_yin_collect(void) { scr_cyc_set_gen(h, SCR_CYC_YIN_PROMOTED); scr_cyc_clear_buffered(h); /* An outside root becomes a candidate (see "candidates" above). */ - if (SCR_RC(obj) > scr_yin_counts[k]) scr_cyc_on_release(obj); + if (SCR_RC_REFS(obj) > scr_yin_counts[k]) scr_cyc_on_release(obj); scr_yin_met(h->id, false); } else { scr_yin_met(h->id, true); @@ -2147,7 +2149,7 @@ static size_t scr_cyc_pass(unsigned gen_limit) { for (size_t j = start; j < end; j++) { void *obj = scr_white.v[j]; if (scr_cyc_color(scr_cyc_hdr(obj)) != SCR_CYC_GRAY) continue; - if (SCR_RC(obj) > 0) scr_restore_from(obj); + if (SCR_RC_REFS(obj) > 0) scr_restore_from(obj); else scr_white.v[kept++] = obj; } from = end; @@ -2176,7 +2178,7 @@ static size_t scr_cyc_pass(unsigned gen_limit) { * whitened only to be restored again. */ for (size_t i = 0; i < scr_cands.n; i++) { void *obj = scr_cands.v[i]; - if (scr_cyc_color(scr_cyc_hdr(obj)) == SCR_CYC_GRAY && SCR_RC(obj) > 0) scr_restore_from(obj); + if (scr_cyc_color(scr_cyc_hdr(obj)) == SCR_CYC_GRAY && SCR_RC_REFS(obj) > 0) scr_restore_from(obj); } for (size_t i = 0; i < scr_cands.n; i++) { void *obj = scr_cands.v[i]; diff --git a/packages/runtime/src/scr_dyn.h b/packages/runtime/src/scr_dyn.h index fdffd73cb..8468f8c0f 100644 --- a/packages/runtime/src/scr_dyn.h +++ b/packages/runtime/src/scr_dyn.h @@ -1019,6 +1019,13 @@ void scr_dyn_typed_ref_bind_record(ScrDyn *d, const ScrDynTypedRecordOps *ops); void scr_record_source_note(void *rec, const ScrDyn *src, const char *const *names, const size_t *lens, size_t count); bool scr_record_source_has(const void *rec); +/* Object.freeze of a record a program may freeze: sets SCR_RC_FROZEN in its + * rc word (immortal records are left as they are). */ +void scr_record_freeze(void *record); +/* Node's TypeError for a write to a frozen record: operation 0 assigns the + * existing property `key`, 1 adds it, 2 deletes it. `owner` is the object's + * text in the message ("#"). */ +void scr_record_throw_frozen(ScrStr *key, ScrStr *owner, double operation); /* The commit of a live record view converts the view into a new record * `from`, whose conversion keeps the view's undeclared properties, then * moves its fields into `to`: this moves the kept properties to `to` too, diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index 798d53ae9..a542b5c24 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -2293,6 +2293,55 @@ ScrDyn *scr_dyn_set_prototype(ScrDyn *object, ScrDyn *prototype) { return scr_dyn_retain(object); } +/* The native record a record capsule refers to, or NULL for any other + * value. */ +static void *scr_dyn_typed_record(const ScrDyn *d) { + if (d->kind != SCR_DYN_TYPED_REF || d->v.typed_ref.type_key_len < 7 || + memcmp(d->v.typed_ref.type_key, "record:", 7) != 0) + return NULL; + return d->v.typed_ref.ptr; +} + +/* Freezes the checked view of a frozen record: its fields, and every + * property added through the view. */ +static void scr_dyn_record_view_freeze(ScrDyn *view) { + view->non_extensible = true; + if (view->kind == SCR_DYN_OBJ) { + for (size_t i = 0; i < view->v.obj.len; i++) { + ScrDynEntry *entry = &view->v.obj.entries[i]; + entry->configurable = false; + if (!entry->accessor) entry->writable = false; + } + } else if (view->kind == SCR_DYN_ARR) { + view->v.arr.sealed = true; + view->v.arr.frozen = true; + if (view->v.arr.properties) scr_dyn_release(scr_dyn_freeze_data(view->v.arr.properties)); + } + if (view->symbol_properties) scr_dyn_release(scr_dyn_freeze_data(view->symbol_properties)); +} + +void scr_record_freeze(void *record) { + size_t *rc = record; + if (*rc != SIZE_MAX) *rc |= SCR_RC_FROZEN; +} + +void scr_record_throw_frozen(ScrStr *key, ScrStr *owner, double operation) { + ScrJsonBuf buffer; + scr_jb_init(&buffer); + if (operation == 1) { + scr_jb_puts(&buffer, "Cannot add property "); + scr_jb_write(&buffer, key->data, key->len); + scr_jb_puts(&buffer, ", object is not extensible"); + } else { + scr_jb_puts(&buffer, operation == 2 ? "Cannot delete property '" : "Cannot assign to read only property '"); + scr_jb_write(&buffer, key->data, key->len); + scr_jb_puts(&buffer, operation == 2 ? "' of " : "' of object '"); + scr_jb_write(&buffer, owner->data, owner->len); + if (operation != 2) scr_jb_puts(&buffer, "'"); + } + scr_throw_error(SCR_ERR_TYPE, scr_jb_finish(&buffer)); +} + /* A class instance's integrity state lives in its native layout, which only * statically typed calls can reach (the compiler adds the state to the * hierarchies they name). Through an untyped value the operation throws. */ @@ -2301,7 +2350,9 @@ static bool scr_dyn_class_integrity_fence(const ScrDyn *object, const char *oper ScrJsonBuf buffer; scr_jb_init(&buffer); scr_jb_puts(&buffer, operation); - scr_jb_puts(&buffer, " of a class instance held in an untyped value is not supported yet"); + scr_jb_puts(&buffer, scr_dyn_typed_record(object) + ? " of an object held in an untyped value is not supported yet" + : " of a class instance held in an untyped value is not supported yet"); ScrStr *message = scr_jb_finish(&buffer); scr_throw_error_msg_code(SCR_ERR_ERROR, message->data, message->len, "SC2020"); scr_str_release(message); @@ -3628,12 +3679,16 @@ ScrDyn *scr_dyn_typed_ref_materialize(const ScrDyn *d) { fresh->rc = fresh_rc; scr_dyn_release(fresh); } + void *record = scr_dyn_typed_record(capsule); + if (record && scr_rc_frozen(record)) scr_dyn_record_view_freeze(capsule->v.typed_ref.materialized); return scr_dyn_retain(capsule->v.typed_ref.materialized); } void scr_dyn_typed_ref_commit(ScrDyn *d) { + /* A frozen record's view cannot have changed its fields. */ if (d && d->kind == SCR_DYN_TYPED_REF && d->v.typed_ref.commit && - d->v.typed_ref.materialized) { + d->v.typed_ref.materialized && + !(scr_dyn_typed_record(d) && scr_rc_frozen(d->v.typed_ref.ptr))) { d->v.typed_ref.commit(d->v.typed_ref.ptr, d->v.typed_ref.materialized); } @@ -15627,8 +15682,18 @@ ScrStr *scr_dyn_to_string_method(const ScrDyn *d, const ScrStr *enc, const ScrSt return scr_dyn_to_string_method_args(d, enc, what, NULL, 0); } +/* Freezing a record capsule freezes the native record, whose emitted writes + * then refuse to change it, and its checked view. */ +static ScrDyn *scr_dyn_freeze_record(ScrDyn *capsule, void *record) { + scr_record_freeze(record); + if (capsule->v.typed_ref.materialized) scr_dyn_record_view_freeze(capsule->v.typed_ref.materialized); + return scr_dyn_retain(capsule); +} + ScrDyn *scr_dyn_freeze(ScrDyn *value) { if (scr_dyn_class_reflection_fence(value) || scr_dyn_error_prototype_fence(value)) return NULL; + void *record = scr_dyn_typed_record(value); + if (record) return scr_dyn_freeze_record(value, record); if (scr_dyn_class_integrity_fence(value, "Object.freeze")) return NULL; ScrDyn *table = scr_dyn_fn_integrity_table(value); if (!table) return scr_dyn_freeze_data(value); @@ -15640,6 +15705,8 @@ ScrDyn *scr_dyn_freeze(ScrDyn *value) { ScrDyn *scr_dyn_freeze_data(ScrDyn *value) { if (scr_dyn_class_reflection_fence(value) || scr_dyn_error_prototype_fence(value)) return NULL; + void *record = scr_dyn_typed_record(value); + if (record) return scr_dyn_freeze_record(value, record); if (scr_dyn_class_integrity_fence(value, "Object.freeze")) return NULL; if (value->symbol_properties) scr_dyn_release(scr_dyn_freeze_data(value->symbol_properties)); if (value->kind == SCR_DYN_ARR) { diff --git a/packages/runtime/src/scr_json.test.c b/packages/runtime/src/scr_json.test.c index 736eca540..a28759876 100644 --- a/packages/runtime/src/scr_json.test.c +++ b/packages/runtime/src/scr_json.test.c @@ -802,8 +802,21 @@ static void json_schema_union_members(void) { assert(test_opts_live == 0); } +/* Freezing a record sets the frozen bit beside its count; an immortal + * record keeps its word, which does not read as frozen. */ +static void record_frozen_bit(void) { + size_t record[2] = {3, 0}; + assert(!scr_rc_frozen(record)); + scr_record_freeze(record); + assert(scr_rc_frozen(record) && SCR_RC_REFS(record) == 3); + size_t immortal[2] = {SIZE_MAX, 0}; + scr_record_freeze(immortal); + assert(immortal[0] == SIZE_MAX && !scr_rc_frozen(immortal)); +} + int main(void) { json_string_boundaries(); + record_frozen_bit(); json_indent_ownership(); scr_init(); checked_storage(); diff --git a/packages/runtime/src/scr_rc.h b/packages/runtime/src/scr_rc.h index 18df3a237..87eb2536a 100644 --- a/packages/runtime/src/scr_rc.h +++ b/packages/runtime/src/scr_rc.h @@ -45,14 +45,32 @@ typedef size_t ScrRcCount; /* Whether `obj`, an object of any kind, is published (immortal). */ #define SCR_RC_PUBLISHED(obj) (SCR_RC_COUNT(obj) >= SCR_RC_PUBLISHED_MIN) +/* Frozen records (Object.freeze). The rc word of a record a program may + * freeze (llvm/shapes.ts) carries SCR_RC_FROZEN beside its count, and that + * record's emitted release ignores the bit. On 64-bit targets the bit is in + * the high half, outside the count; on 32-bit targets it is the count's bit + * 30, which SCR_RC_REFS masks. An immortal word has the bit set too; it + * does not read as frozen. */ +#if SIZE_MAX > UINT32_MAX +#define SCR_RC_FROZEN ((size_t)1 << 62) +#define SCR_RC_REFS(obj) SCR_RC_COUNT(obj) +#else +#define SCR_RC_FROZEN ((size_t)1 << 30) +#define SCR_RC_REFS(obj) (SCR_RC_COUNT(obj) == SCR_RC_IMMORTAL ? SCR_RC_IMMORTAL : SCR_RC_COUNT(obj) & ~SCR_RC_FROZEN) +#endif +static inline bool scr_rc_frozen(const void *obj) { + size_t word = *(const size_t *)obj; + return word != SIZE_MAX && (word & SCR_RC_FROZEN) != 0; +} + /* The class id in the high half of `obj`'s rc word, or 0 for an object - * without one: the high half of any other object's word is 0, or 0xffffffff - * (immortal) or 0x80000000 (program-lifetime), and a class id is never one of - * these. */ + * without one: the high half of any other object's word is 0, 0xffffffff + * (immortal), 0x80000000 (program-lifetime) or 0x40000000 (a frozen record), + * and a class id is never one of these. */ static inline uint32_t scr_rc_class_id(const void *obj) { #if SIZE_MAX > UINT32_MAX uint32_t high = (uint32_t)(*(const size_t *)obj >> 32); - return high == UINT32_MAX || high == UINT32_C(0x80000000) ? 0 : high; + return high == UINT32_MAX || high == UINT32_C(0x80000000) || high == UINT32_C(0x40000000) ? 0 : high; #else (void)obj; return 0; diff --git a/tests/corpus/7404-frozen-records.ts b/tests/corpus/7404-frozen-records.ts new file mode 100644 index 000000000..acf9b393b --- /dev/null +++ b/tests/corpus/7404-frozen-records.ts @@ -0,0 +1,124 @@ +// Object.freeze of records that other references still hold: writes +// through any reference then throw Node's TypeError (this module is strict +// mode code; 7405 covers sloppy mode). Covers static freezes, freezes +// through untyped values, Object.isFrozen, keyed writes and deletes, +// optional-field deletes, Object.assign, tuples, and frozen records that are +// released or collected as cycles. +export {}; + +type Counter = { count: number; label: string }; +type Link = { name: string; next?: Link }; + +function report(e: unknown): void { + console.log((e as Error).name + ": " + (e as Error).message); +} + +function freezeUnknown(value: unknown): unknown { + return Object.freeze(value); +} + +function isFrozenUnknown(value: unknown): boolean { + return Object.isFrozen(value); +} + +function deepFreeze(value: unknown): unknown { + if (typeof value === "object" && value !== null) { + Object.freeze(value); + for (const key of Object.keys(value)) deepFreeze((value as Record)[key]); + } + return value; +} + +type Config = { name: string; limits: { max: number; min: number } }; + +function bump(c: Counter): void { + c.count += 1; +} + +function writes(): void { + const a: Counter = { count: 1, label: "a" }; + const alias = a; + console.log(Object.isFrozen(a), isFrozenUnknown(a)); + const frozen = Object.freeze(alias); + console.log(frozen === a, Object.isFrozen(a), isFrozenUnknown(alias)); + try { bump(a); } catch (e) { report(e); } + try { a.label = "b"; } catch (e) { report(e); } + console.log(a.count, a.label, JSON.stringify(a)); + + const viaUnknown: Counter = { count: 10, label: "u" }; + const back = freezeUnknown(viaUnknown); + console.log(back === viaUnknown, Object.isFrozen(viaUnknown)); + try { viaUnknown.count = 11; } catch (e) { report(e); } + console.log(viaUnknown.count); + + const dict: Record = { one: 1 }; + const dictAlias = dict; + Object.freeze(dictAlias); + try { dict["one"] = 2; } catch (e) { report(e); } + try { dict["two"] = 2; } catch (e) { report(e); } + try { delete dict["one"]; } catch (e) { report(e); } + delete dict["three"]; + console.log(JSON.stringify(dict), Object.isFrozen(dict)); + + const optional: { id: number; note?: string } = { id: 1, note: "n" }; + const optionalAlias = optional; + Object.freeze(optionalAlias); + try { delete optional.note; } catch (e) { report(e); } + console.log(JSON.stringify(optional)); + const bare: { id: number; note?: string } = { id: 2 }; + Object.freeze(bare); + delete bare.note; + console.log(JSON.stringify(bare)); + + const target: Counter = { count: 0, label: "t" }; + Object.freeze(target); + try { Object.assign(target, { count: 5, label: "x" }); } catch (e) { report(e); } + console.log(JSON.stringify(target)); + + const pair: [string, number] = ["p", 1]; + Object.freeze(pair); + try { pair[1] = 2; } catch (e) { report(e); } + console.log(pair[0], pair[1]); +} + +function deepFreezes(): void { + const config: Config = JSON.parse('{"name":"c","limits":{"max":2,"min":1}}'); + const limits = config.limits; + console.log(Object.isFrozen(config), Object.isFrozen(limits)); + deepFreeze(config); + console.log(Object.isFrozen(config), Object.isFrozen(limits), isFrozenUnknown(config.limits)); + try { config.limits.max = 3; } catch (e) { report(e); } + try { limits.min = 0; } catch (e) { report(e); } + try { config.name = "d"; } catch (e) { report(e); } + console.log(JSON.stringify(config)); +} + +function lifetimes(): void { + const list: Counter[] = []; + for (let i = 0; i < 4; i++) { + const item: Counter = { count: i, label: "x" + i }; + list.push(item); + if (i % 2 === 1) Object.freeze(item); + } + for (const item of list) { + try { item.count = item.count * 10; } catch (e) { console.log(item.label, (e as Error).message); } + } + console.log(JSON.stringify(list)); + const kept = list[1]!; + list.length = 0; + console.log(kept.count, Object.isFrozen(kept)); + + for (let round = 0; round < 3; round++) { + const first: Link = { name: "first" + round }; + const second: Link = { name: "second" + round, next: first }; + first.next = second; + Object.freeze(first); + Object.freeze(second); + try { first.name = "changed"; } catch (e) { report(e); } + console.log(first.next?.name, second.next?.name); + } +} + +writes(); +deepFreezes(); +lifetimes(); diff --git a/tests/corpus/7405-frozen-records-sloppy.cjs b/tests/corpus/7405-frozen-records-sloppy.cjs new file mode 100644 index 000000000..c0656b7b2 --- /dev/null +++ b/tests/corpus/7405-frozen-records-sloppy.cjs @@ -0,0 +1,34 @@ +// Writes to frozen records in sloppy-mode code leave them unchanged, where +// strict mode code throws (7404). The records are typed through JSDoc, so +// the writes are native record writes. + +/** @typedef {{ count: number, label: string }} Counter */ +/** @typedef {{ id: number, note?: string }} Note */ + +/** @param {Counter} c */ +function bump(c) { + c.count += 1; + c.label = c.label + "!"; +} + +/** @param {Note} n */ +function dropNote(n) { + delete n.note; +} + +/** @param {number} count @returns {Counter} */ +function counter(count) { + return { count: count, label: "c" + count }; +} + +const open = counter(1); +const frozen = Object.freeze(counter(2)); +bump(open); +bump(frozen); +console.log(JSON.stringify(open), JSON.stringify(frozen), Object.isFrozen(frozen)); + +/** @type {Note} */ +const note = { id: 1, note: "n" }; +Object.freeze(note); +dropNote(note); +console.log(JSON.stringify(note)); diff --git a/tests/harness/corpus-costs/7404-frozen-records.ts.json b/tests/harness/corpus-costs/7404-frozen-records.ts.json new file mode 100644 index 000000000..765134581 --- /dev/null +++ b/tests/harness/corpus-costs/7404-frozen-records.ts.json @@ -0,0 +1 @@ +{ "plain": 4725, "san": 55833 } diff --git a/tests/harness/corpus-costs/7405-frozen-records-sloppy.cjs.json b/tests/harness/corpus-costs/7405-frozen-records-sloppy.cjs.json new file mode 100644 index 000000000..c7cd5ec8f --- /dev/null +++ b/tests/harness/corpus-costs/7405-frozen-records-sloppy.cjs.json @@ -0,0 +1 @@ +{ "plain": 3175, "san": 37248 } From a00d56e5181bffee04a66ab99246807f680b66f5 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 02:26:57 -0500 Subject: [PATCH 09/53] Fix the int32 array CI failures --- packages/compiler/src/ir/int32-arrays.ts | 57 ++++++++++++++++-------- 1 file changed, 39 insertions(+), 18 deletions(-) diff --git a/packages/compiler/src/ir/int32-arrays.ts b/packages/compiler/src/ir/int32-arrays.ts index 0872fcddb..e67f51956 100644 --- a/packages/compiler/src/ir/int32-arrays.ts +++ b/packages/compiler/src/ir/int32-arrays.ts @@ -7,6 +7,8 @@ import { everyExprChild, everyStmtChild, everyStmtList, + mapExprChildren, + mapStmtChildren, transformExpr, transformStmt, transformStmtList, @@ -629,16 +631,34 @@ export function int32ArrayStores( return stores; } +const REPEATED_INDEX_OPS = new Set(["+", "-", "*", "&", "|", "^", "<<", ">>", ">>>"]); + +/** An index the inlined read may evaluate at each of its uses: a plain read, + * a literal, or integer arithmetic over them. Repeating it keeps the forms + * the backend derives from its operands at every use, such as the exact i64 + * of an int32 product, where a bound local holds only the double. */ +function repeatableIndex(e: IrExpr): boolean { + if (e.kind === "varRef" || e.kind === "numLit") return true; + if (e.kind === "unary") return e.op === "-" && repeatableIndex(e.operand); + return ( + e.kind === "bin" && + REPEATED_INDEX_OPS.has(e.op) && + repeatableIndex(e.left) && + repeatableIndex(e.right) + ); +} + /** Rewrite a function's candidates for the backend. A discarded pop uses * the candidate in place of its hidden local. An element-reader helper call * whose array is a candidate is inlined: the helper's returned ternary, with * the candidate for its array and the index for its index, so the read means - * the same. An index that is not a plain read or literal is bound to a fresh - * local first: in its own declaration just before the statement when the - * call is the whole value of a declaration or assignment (the form lowering - * gives `a[i]!`, and the one the backend reads without a box), otherwise in a - * sequence around the ternary. Selection keeps out indexes that write the - * array, so evaluating the index before the array read is unobservable. */ + * the same. An index that is not repeatable (repeatableIndex) is bound to a + * fresh local first: in its own declaration just before the statement when + * the call is the whole value of a declaration or assignment (the form + * lowering gives `a[i]!`, and the one the backend reads without a box), + * otherwise in a sequence around the ternary. Selection keeps out indexes + * that write the array, so evaluating the index before the array read is + * unobservable. */ function rewriteCandidates( fn: IrFunction, candidates: ReadonlySet, @@ -662,14 +682,13 @@ function rewriteCandidates( candidates.has(e.args[0]!.localId) ? e : null; - const simple = (e: IrExpr): boolean => e.kind === "varRef" || e.kind === "numLit"; // The statements of a list, with complex read indexes declared first. const hoist = (list: IrStmt[]): IrStmt[] => list.flatMap((s): IrStmt[] => { if (s.kind !== "varDecl" && s.kind !== "assign") return [s]; const value = s.kind === "varDecl" ? s.init : s.value; const call = readerCall(value); - if (!call || simple(call.args[1]!)) return [s]; + if (!call || repeatableIndex(call.args[1]!)) return [s]; const { decl, ref } = bind(call.args[1]!); const read: IrExpr = { ...call, args: [call.args[0]!, ref] }; if (s.kind === "varDecl") return [decl, { ...s, init: read }]; @@ -732,21 +751,23 @@ function rewriteCandidates( if (ret.kind !== "return" || !ret.value) return e; let index = call.args[1]!; const stmts: IrStmt[] = []; - if (!simple(index)) { + if (!repeatableIndex(index)) { const { decl, ref } = bind(index); stmts.push(decl); index = ref; } const operand = index; - const body = transformExpr(reparseExpr(ret.value), { - expr: (x) => { - if (x.kind !== "varRef") return x; - if (x.localId === arrayParam!.localId) return reparseExpr(array); - if (x.localId === indexParam!.localId) return reparseExpr(operand); - return x; - }, - stmt: (x) => x, - }); + // The arguments' copies are not searched again: an index may read a + // local that has the id of one of the helper's parameters. + const substituteExpr = (x: IrExpr): IrExpr => { + if (x.kind !== "varRef") return mapExprChildren(x, substituteExpr, substituteStmt); + if (x.localId === arrayParam!.localId) return reparseExpr(array); + if (x.localId === indexParam!.localId) return reparseExpr(operand); + return x; + }; + const substituteStmt = (x: IrStmt): IrStmt => + mapStmtChildren(x, substituteExpr, substituteStmt); + const body = substituteExpr(reparseExpr(ret.value)); // The ternary is a fresh copy (reparseExpr): it takes the call's location. const typed = body; typed.loc = e.loc; From 342cf3a063301e46a90e428a848dfd825aff80d8 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Fri, 23 Oct 2026 00:29:22 -0700 Subject: [PATCH 10/53] Copy class-view property bags entry by entry Materializing the checked view of a JavaScript class instance copies its property bag through scr_dyn_copy_property_descriptors, which built a key string and a descriptor object per property and defined each through scr_dyn_define_property (about 2k instructions per property). Plain objects on both sides now copy each entry directly with the attributes, value or accessor pair and own-key order the definition gives; properties the target already has, non-extensible targets and other receivers keep the general path. Co-Authored-By: Claude Opus 5.5 --- .../7620-js-class-view-descriptors.js.json | 6 ++ packages/runtime/src/scr_json.c | 73 +++++++++++++-- packages/runtime/test/test_json.c | 86 ++++++++++++++++++ .../corpus/7620-js-class-view-descriptors.js | 91 +++++++++++++++++++ .../7620-js-class-view-descriptors.js.json | 1 + 5 files changed, 248 insertions(+), 9 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7620-js-class-view-descriptors.js.json create mode 100644 tests/corpus/7620-js-class-view-descriptors.js create mode 100644 tests/harness/corpus-costs/7620-js-class-view-descriptors.js.json diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7620-js-class-view-descriptors.js.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7620-js-class-view-descriptors.js.json new file mode 100644 index 000000000..2cb1ece22 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7620-js-class-view-descriptors.js.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7620-js-class-view-descriptors.js" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index 907486fea..3676dd059 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -13699,19 +13699,74 @@ ScrDyn *scr_dyn_object_rest(const ScrDyn *source, const ScrDyn *excluded) { /* Compiler-owned native class views retain every property and its flags, * including nonenumerable data, without invoking accessors. */ -ScrDyn *scr_dyn_copy_property_descriptors(ScrDyn *target, const ScrDyn *src) { - if (!src || src->kind == SCR_DYN_UNDEF) return scr_dyn_retain(target); - ScrDyn *keys = scr_dyn_obj_own_keys(src); - if (!keys) return NULL; - for (size_t i = 0; i < keys->v.arr.len; i++) { - ScrDyn *key = keys->v.arr.items[i]; - ScrDyn *descriptor = scr_dyn_own_descriptor(src, key->v.str); +static void scr_dyn_copy_property_descriptor(ScrDyn *target, const ScrDyn *src, + const ScrDynEntry *from) { + /* Exactly scr_dyn_define_property(target, key, own descriptor of key) + * when `target` has no such property and accepts new ones: the entry + * takes the source's attributes, value or accessor pair. */ + ScrDynEntry *prior = scr_dyn_find_entry(target, from->key, from->key_len); + if (prior || target->non_extensible) { + ScrStr *name = scr_str_new(from->key, from->key_len); + ScrDyn *key = scr_dyn_new_str(name); + ScrDyn *descriptor = scr_dyn_own_descriptor(src, name); ScrDyn *result = descriptor ? scr_dyn_define_property(target, key, descriptor) : NULL; scr_dyn_release(descriptor); scr_dyn_release(result); - if (scr_exc_pending()) break; + scr_dyn_release(key); + scr_str_release(name); + return; + } + /* The names whose definition may move into a native Error's slots + * (scr_error_sync_slots). */ + static const char *const slots[] = {"cause", "stack", "name", "message", "code"}; + bool slot = false; + for (size_t i = 0; i < sizeof slots / sizeof *slots && !slot; i++) + slot = from->key_len == strlen(slots[i]) && memcmp(from->key, slots[i], from->key_len) == 0; + bool accessor = from->accessor, writable = !accessor && from->writable; + bool enumerable = from->enumerable, configurable = from->configurable; + ScrDyn *value = scr_dyn_retain(accessor ? scr_dyn_undefined() : from->value); + ScrDyn *getter = accessor && from->getter ? scr_dyn_retain(from->getter) : NULL; + ScrDyn *setter = accessor && from->setter ? scr_dyn_retain(from->setter) : NULL; + ScrStr *name = slot ? scr_str_new(from->key, from->key_len) : NULL; + scr_dyn_obj_set(target, from->key, from->key_len, value); + ScrDynEntry *entry = &target->v.obj.entries[target->v.obj.len - 1]; + entry->accessor = accessor; + entry->getter = getter; + entry->setter = setter; + entry->writable = writable; + entry->enumerable = enumerable; + entry->configurable = configurable; + if (name) { + scr_error_sync_cause(target, name); + scr_str_release(name); + } +} + +ScrDyn *scr_dyn_copy_property_descriptors(ScrDyn *target, const ScrDyn *src) { + if (!src || src->kind == SCR_DYN_UNDEF) return scr_dyn_retain(target); + if (src->kind == SCR_DYN_OBJ && target->kind == SCR_DYN_OBJ && src != target && + src != scr_builtin_object_prototype && target != scr_builtin_error_prototype) { + /* Plain objects on both sides: no fence, proxy trap or accessor runs, + * so each property is copied from its entry without building the key + * and descriptor objects the general path goes through. */ + if (src->v.obj.len) scr_dyn_region_store(target, NULL); + size_t *order = scr_dyn_obj_key_order(src); + for (size_t i = 0; i < src->v.obj.len && !scr_exc_pending(); i++) + scr_dyn_copy_property_descriptor(target, src, &src->v.obj.entries[order ? order[i] : i]); + free(order); + } else { + ScrDyn *keys = scr_dyn_obj_own_keys(src); + if (!keys) return NULL; + for (size_t i = 0; i < keys->v.arr.len; i++) { + ScrDyn *key = keys->v.arr.items[i]; + ScrDyn *descriptor = scr_dyn_own_descriptor(src, key->v.str); + ScrDyn *result = descriptor ? scr_dyn_define_property(target, key, descriptor) : NULL; + scr_dyn_release(descriptor); + scr_dyn_release(result); + if (scr_exc_pending()) break; + } + scr_dyn_release(keys); } - scr_dyn_release(keys); if (!scr_exc_pending()) { ScrDyn *symbols = scr_dyn_get_own_property_symbols(src); if (!symbols) return NULL; diff --git a/packages/runtime/test/test_json.c b/packages/runtime/test/test_json.c index 2c072006f..865f1c757 100644 --- a/packages/runtime/test/test_json.c +++ b/packages/runtime/test/test_json.c @@ -551,6 +551,91 @@ static void test_class_values(void) { scr_str_release(name); } +/* The class-view copy (scr_dyn_copy_property_descriptors) gives each + * property of the bag the attributes, value or accessors, and own-key + * position that defining it from its own descriptor would. */ +static ScrDyn *define_from(ScrDyn *target, const char *key, ScrDyn *value, bool writable, + bool enumerable, bool configurable) { + ScrDyn *descriptor = scr_dyn_new_obj(); + scr_dyn_obj_set(descriptor, "value", 5, value); + scr_dyn_obj_set(descriptor, "writable", 8, scr_dyn_new_bool(writable)); + scr_dyn_obj_set(descriptor, "enumerable", 10, scr_dyn_new_bool(enumerable)); + scr_dyn_obj_set(descriptor, "configurable", 12, scr_dyn_new_bool(configurable)); + ScrStr *name = S(key); + ScrDyn *k = scr_dyn_new_str(name); + ScrDyn *result = scr_dyn_define_property(target, k, descriptor); + scr_dyn_release(k); + scr_str_release(name); + scr_dyn_release(descriptor); + return result; +} + +static const ScrDynEntry *entry_of(const ScrDyn *obj, const char *key) { + for (size_t i = 0; i < obj->v.obj.len; i++) + if (obj->v.obj.entries[i].key_len == strlen(key) && + memcmp(obj->v.obj.entries[i].key, key, strlen(key)) == 0) + return &obj->v.obj.entries[i]; + return NULL; +} + +static void test_copy_property_descriptors(void) { + ScrDyn *bag = scr_dyn_new_obj(); + ScrDyn *getter = scr_dyn_new_obj(); + scr_dyn_release(define_from(bag, "label", scr_dyn_new_num(1), true, true, true)); + scr_dyn_release(define_from(bag, "hidden", scr_dyn_new_num(2), false, false, true)); + scr_dyn_obj_set(bag, "7", 1, scr_dyn_new_num(7)); + scr_dyn_obj_set(bag, "3", 1, scr_dyn_new_num(3)); + /* An accessor entry (the getter object stands in for its function). */ + scr_dyn_obj_set(bag, "computed", 8, scr_dyn_retain(scr_dyn_undefined())); + ScrDynEntry *accessor = (ScrDynEntry *)entry_of(bag, "computed"); + accessor->accessor = true; + accessor->getter = scr_dyn_retain(getter); + accessor->writable = false; + accessor->enumerable = false; + accessor->configurable = false; + + ScrDyn *view = scr_dyn_new_obj(); + scr_dyn_release(scr_dyn_copy_property_descriptors(view, bag)); + check(!scr_exc_pending() && view->v.obj.len == 5, "copy keeps every bag property"); + ScrDyn *keys = scr_dyn_obj_own_keys(view); + const char *order[] = {"3", "7", "label", "hidden", "computed"}; + bool ordered = keys->v.arr.len == 5; + for (size_t i = 0; ordered && i < 5; i++) ordered = str_is(keys->v.arr.items[i]->v.str, order[i]); + check(ordered, "copy keeps the bag's own-key order"); + scr_dyn_release(keys); + const ScrDynEntry *hidden = entry_of(view, "hidden"); + check(hidden && !hidden->accessor && hidden->value->v.num == 2 && !hidden->writable && + !hidden->enumerable && hidden->configurable, + "copy keeps data attributes"); + const ScrDynEntry *copied = entry_of(view, "computed"); + check(copied && copied->accessor && copied->getter == getter && !copied->setter && + !copied->writable && !copied->enumerable && !copied->configurable && + copied->value->kind == SCR_DYN_UNDEF, + "copy keeps an accessor without calling it"); + + /* A property the target already has is redefined like defineProperty. */ + ScrDyn *existing = scr_dyn_new_obj(); + scr_dyn_obj_set(existing, "label", 5, scr_dyn_new_num(9)); + scr_dyn_release(scr_dyn_copy_property_descriptors(existing, bag)); + const ScrDynEntry *label = entry_of(existing, "label"); + check(!scr_exc_pending() && existing->v.obj.len == 5 && existing->v.obj.entries[0].key_len == 5 && + label->value->v.num == 1, + "copy redefines an existing property in place"); + + /* A target that takes no new properties throws as defineProperty does. */ + ScrDyn *closed = scr_dyn_new_obj(); + scr_dyn_release(scr_dyn_prevent_extensions(closed)); + ScrDyn *refused = scr_dyn_copy_property_descriptors(closed, bag); + check(!refused && scr_exc_pending() && closed->v.obj.len == 0, "copy onto a non-extensible object throws"); + scr_exc_clear(); + + scr_dyn_release(closed); + scr_dyn_release(existing); + scr_dyn_release(view); + scr_dyn_release(bag); + scr_dyn_release(getter); +} + static void test_primitive_owners(void) { ScrDyn *values[] = {scr_dyn_new_null(), scr_dyn_new_bool(false), scr_dyn_new_bool(true)}; const char *texts[] = {"null", "false", "true"}; @@ -579,6 +664,7 @@ int main(void) { test_class_values(); scr_init(); test_primitive_owners(); + test_copy_property_descriptors(); /* ── primitives ─────────────────────────────────────────────────── */ ScrDyn *d = parse_ok("null", "parse null"); diff --git a/tests/corpus/7620-js-class-view-descriptors.js b/tests/corpus/7620-js-class-view-descriptors.js new file mode 100644 index 000000000..a0dc42813 --- /dev/null +++ b/tests/corpus/7620-js-class-view-descriptors.js @@ -0,0 +1,91 @@ +// An instance of a JS class with public fields keeps its own properties in +// a property bag, and untyped code that calls a method on it sees a view +// built from that bag: data properties with their attributes, accessors, +// index-like keys (which come first, in ascending order), symbols, and +// properties added or removed after construction. Every refresh of the view +// must copy the bag exactly. +'use strict'; + +class Box { + label = 'box'; + count = 0; + constructor(n) { + this.count = n; + } + describe() { + return `${this.label}:${this.count}`; + } + bump() { + this.count++; + return this; + } +} + +function call(target, method, ...args) { + return target[method](...args); +} + +function put(target, key, value) { + target[key] = value; +} + +function drop(target, key) { + delete target[key]; +} + +function show(title, target) { + const lines = []; + for (const key of Reflect.ownKeys(target)) { + const d = Object.getOwnPropertyDescriptor(target, key); + const kind = 'value' in d ? `value=${String(d.value)} writable=${d.writable}` : `get=${typeof d.get} set=${typeof d.set}`; + lines.push(` ${String(key)}: ${kind} enumerable=${d.enumerable} configurable=${d.configurable}`); + } + console.log(`${title}\n${lines.join('\n')}`); +} + +const box = new Box(2); +Object.defineProperty(box, 'hidden', { value: 42, enumerable: false, writable: false, configurable: true }); +Object.defineProperty(box, 'computed', { + get() { + return this.count * 10; + }, + set(v) { + this.count = v / 10; + }, + enumerable: true, + configurable: true, +}); +Object.defineProperty(box, 'getterOnly', { get: () => 'g', enumerable: false, configurable: false }); +put(box, 2, 'two'); +put(box, 1, 'one'); +put(box, 'extra', 'x'); +const tag = Symbol('tag'); +put(box, tag, 'symbol'); + +console.log(call(box, 'describe')); +show('after construction', box); +console.log('keys', Object.keys(box).join(',')); + +call(box, 'bump'); +put(box, 'computed', 70); +drop(box, 'extra'); +put(box, 'later', true); +console.log(call(box, 'describe')); +show('after changes', box); +console.log('computed', box.computed, 'hidden', box.hidden, 'getterOnly', box.getterOnly); + +// Many views in a row see the same, current properties. +let total = 0; +for (let i = 0; i < 5; i++) { + call(box, 'bump'); + total += Object.keys(box).length; +} +console.log('total keys', total, call(box, 'describe')); + +const other = new Box(7); +console.log(call(other, 'describe'), Object.keys(other).join(',')); +try { + call(box, 'missing'); +} catch (error) { + console.log(error instanceof TypeError, error.message); +} diff --git a/tests/harness/corpus-costs/7620-js-class-view-descriptors.js.json b/tests/harness/corpus-costs/7620-js-class-view-descriptors.js.json new file mode 100644 index 000000000..ef439d096 --- /dev/null +++ b/tests/harness/corpus-costs/7620-js-class-view-descriptors.js.json @@ -0,0 +1 @@ +{ "plain": 3000, "san": 4500 } From bcd506fc8b14c50ae7eb8ba96764d7d42091552e Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Fri, 23 Oct 2026 00:29:28 -0700 Subject: [PATCH 11/53] Return ASCII header values as their own ByteString Converting a header value to a ByteString read it one UTF-16 code unit at a time through scr_str_char_code_at into a fresh buffer. An all-ASCII value, which nearly every header is, is already its own ByteString, so it is now returned as is. Co-Authored-By: Claude Opus 5.5 --- packages/runtime/src/scr_fetch.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/runtime/src/scr_fetch.c b/packages/runtime/src/scr_fetch.c index 96e90e47c..549806f39 100644 --- a/packages/runtime/src/scr_fetch.c +++ b/packages/runtime/src/scr_fetch.c @@ -3287,6 +3287,10 @@ static bool sf_header_value_ok(const ScrStr *value) { } static ScrStr *sf_header_value_bytestring(ScrStr *value) { + /* An ASCII value is its own ByteString: each byte is one UTF-16 unit. */ + size_t ascii = 0; + while (ascii < value->len && (unsigned char)value->data[ascii] < 0x80) ascii++; + if (ascii == value->len) return scr_str_retain(value); size_t len = (size_t)scr_str_utf16_len(value); char *bytes = malloc(len ? len : 1); if (!bytes) sf_oom(); From 945a519036530065664e07e85754ff3a7dd5cc6f Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 03:49:51 -0500 Subject: [PATCH 12/53] Match Node for typed pipeline callbacks and rl.cursor on non-terminal interfaces --- .../builtin-apis/streams-and-readline.mdx | 4 +- .../frontend/lowering/builtins/readline.ts | 51 +++++++- .../lowering/expressions/narrowing.ts | 3 + .../src/frontend/lowering/lower-stream.ts | 24 +++- .../compiler/src/frontend/lowering/lowerer.ts | 3 +- .../7500-pipeline-callback-undefined.ts.json | 6 + .../7501-pipeline-callback-undefined.mjs.json | 6 + .../7502-readline-cursor-undefined.ts.json | 6 + .../7500-pipeline-callback-undefined.ts | 120 ++++++++++++++++++ .../7501-pipeline-callback-undefined.mjs | 22 ++++ .../corpus/7502-readline-cursor-undefined.ts | 28 ++++ .../7500-pipeline-callback-undefined.ts.json | 1 + .../7501-pipeline-callback-undefined.mjs.json | 1 + .../7502-readline-cursor-undefined.ts.json | 1 + 14 files changed, 268 insertions(+), 8 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7500-pipeline-callback-undefined.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7501-pipeline-callback-undefined.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7502-readline-cursor-undefined.ts.json create mode 100644 tests/corpus/7500-pipeline-callback-undefined.ts create mode 100644 tests/corpus/7501-pipeline-callback-undefined.mjs create mode 100644 tests/corpus/7502-readline-cursor-undefined.ts create mode 100644 tests/harness/corpus-costs/7500-pipeline-callback-undefined.ts.json create mode 100644 tests/harness/corpus-costs/7501-pipeline-callback-undefined.mjs.json create mode 100644 tests/harness/corpus-costs/7502-readline-cursor-undefined.ts.json diff --git a/docs/content/limitations/builtin-apis/streams-and-readline.mdx b/docs/content/limitations/builtin-apis/streams-and-readline.mdx index 166fc3402..45a784742 100644 --- a/docs/content/limitations/builtin-apis/streams-and-readline.mdx +++ b/docs/content/limitations/builtin-apis/streams-and-readline.mdx @@ -18,7 +18,7 @@ Stages may take the `{ signal }` argument. With a function destination, `pipeline()` returns an object-mode PassThrough that receives an async generator destination's values, or an async function destination's value unless it is `null` or `undefined`. `@types/node` types that result as `NodeJS.WritableStream`, so `for await` over it needs the `PassThrough` type (`pipeline(...) as PassThrough`, or a JSDoc `@type`). - In typed code, the callback of a pipeline with an async function destination receives `null` instead of `undefined` when there is no error. + In typed code, a pipeline callback with an async function or async generator destination receives `null` for no error when its error parameter cannot hold `undefined` (@types/node declares it `NodeJS.ErrnoException | null`), and `undefined` when it can, as in Node. `Readable.from` accepts `objectMode` and `highWaterMark` options. The `Readable` operators (`map`, `filter`, `forEach`, `toArray`, `some`, `find`, `every`, `flatMap`, `drop`, `take`, and `reduce`) accept `concurrency`, `highWaterMark`, and `signal` options. Their callbacks may take the `{ signal }` argument, `map` may return any value, and `flatMap` may return any iterable, async iterable, or stream. @@ -43,7 +43,7 @@ The output's `'resize'` event does not redraw the line. The cursor never stops between the two UTF-16 units of a supplementary character: where Node's Up or Down movement through a multiline line would, the cursor moves to the character's start or end. In JavaScript, a `'history'` listener receives a copy of the history; its changes are written back when the listener returns, but the copy is not `rl.history` itself. - `rl.history`, `historySize`, `historyIndex`, `crlfDelay`, `tabSize`, `escapeCodeTimeout`, `line`, `cursor`, and `terminal` read as in Node, except that `rl.cursor` of an interface that is not a terminal throws instead of answering `undefined`. + `rl.history`, `historySize`, `historyIndex`, `crlfDelay`, `tabSize`, `escapeCodeTimeout`, `line`, `cursor`, and `terminal` read as in Node; `rl.cursor` is `undefined` on an interface that is not a terminal. `rl[Symbol.dispose]()` closes the interface, and `instanceof` with `readline.Interface` or `readlinePromises.Interface` answers whether the interface came from that module, as in Node. In JavaScript, the input and output can be any expression that evaluates to a stream, such as a function's result or an object's member; a value that is not a stream is unsupported. In JavaScript, `question()` and `setPrompt()` accept a value that is not a string: when such a prompt is written, an interface that is not a terminal throws `ERR_INVALID_ARG_TYPE` as Node does (the `readline/promises` question rejects with it) and the question stays pending, and a terminal interface writes `String(value)`. diff --git a/packages/compiler/src/frontend/lowering/builtins/readline.ts b/packages/compiler/src/frontend/lowering/builtins/readline.ts index 846b9a926..aca209017 100644 --- a/packages/compiler/src/frontend/lowering/builtins/readline.ts +++ b/packages/compiler/src/frontend/lowering/builtins/readline.ts @@ -6,9 +6,11 @@ import { DYN, F64, type IrExpr, + type IrType, type IrStmt, type SrcLoc, STRING, + UNDEFINED_T, VOID, arrayOf, BYTES_U8, @@ -54,6 +56,21 @@ export function isReadlineTyped(lowerer: Lowerer, node: ts.Expression): boolean }); } +/** `rl.cursor`: @types/node declares a number, but an interface that is + * not a terminal answers undefined, so the read keeps its + * `number | undefined` union. */ +export function isOptionalReadlineCursorRead( + lowerer: Lowerer, + expr: ts.PropertyAccessExpression, +): boolean { + return ( + expr.name.text === "cursor" && + !expr.questionDotToken && + isReadlineTyped(lowerer, expr.expression) && + lowerer.isStdlibMember(expr) + ); +} + /** The runtime's listener event numbers (scr_readline.c). */ const RL_EVENTS: Record = { line: 0, @@ -1341,7 +1358,7 @@ export function lowerReadlineCreateCall( /** Property reads on an Interface: `history` (the history option itself, * or the interface's own empty array), `historySize`, `historyIndex`, * `crlfDelay`, `tabSize`, `escapeCodeTimeout`, `line`, `terminal` and - * `cursor` (a non-terminal interface's undefined cursor throws SC2020). + * `cursor` (undefined on a non-terminal interface). * Null for other receivers. */ export function lowerReadlineProperty( lowerer: Lowerer, @@ -1373,10 +1390,38 @@ export function lowerReadlineProperty( loc, }; } - const reads: Record = { + if (name === "cursor") { + // `number | undefined`: a non-terminal interface has no cursor + const recv = lowerer.lowerExprExpecting(access.expression, F64); + const type: IrType = { kind: "union", unionId: lowerer.unions.intern([F64, UNDEFINED_T]) }; + const local = recv.kind === "varRef" ? null : lowerer.declareHiddenLocal("%rlCursor", F64); + const id: IrExpr = local ? { kind: "varRef", localId: local.id, type: F64, loc } : recv; + const result: IrExpr = { + kind: "ternary", + cond: { kind: "libCall", fn: "rl.isTerminal", args: [id], type: BOOL, loc }, + then: lowerer.coerceToExpected( + { kind: "libCall", fn: "rl.cursor", args: [id], type: F64, loc }, + type, + ), + else_: lowerer.coerceToExpected( + { kind: "unitLit", unit: "undefined", type: UNDEFINED_T, loc }, + type, + ), + type, + loc, + }; + if (!local) return result; + return { + kind: "seqExpr", + stmts: [{ kind: "varDecl", localId: local.id, init: recv, loc }], + result, + type, + loc, + }; + } + const reads: Record = { line: ["rl.line", STRING], terminal: ["rl.isTerminal", BOOL], - cursor: ["rl.cursor", F64], }; const read = reads[name]; if (read !== undefined) { diff --git a/packages/compiler/src/frontend/lowering/expressions/narrowing.ts b/packages/compiler/src/frontend/lowering/expressions/narrowing.ts index fa8ad686f..01cab9901 100644 --- a/packages/compiler/src/frontend/lowering/expressions/narrowing.ts +++ b/packages/compiler/src/frontend/lowering/expressions/narrowing.ts @@ -33,6 +33,7 @@ import { isOptionalProcessVersionsRead, } from "../builtins/process.js"; import { requestGeneralStreamSurface } from "../general-streams.js"; +import { isOptionalReadlineCursorRead } from "../builtins/readline.js"; import { literalValues } from "../../literal-values.js"; import { literalUnionArm } from "../../union-discriminants.js"; import { checkedArmExtraction } from "../coercions/unions.js"; @@ -617,6 +618,8 @@ export function maybeNarrow(lowerer: Lowerer, expr: IrExpr, node: ts.Node): IrEx if (ts.isPropertyAccessExpression(node) && isOptionalProcessStreamProperty(lowerer, node)) return expr; if (isOptionalProcessVersionsRead(lowerer, node)) return expr; + if (ts.isPropertyAccessExpression(node) && isOptionalReadlineCursorRead(lowerer, node)) + return expr; // Represented fields and index values can include undefined even when // checker inference or lib.d.ts declares strings (RegExp captures). diff --git a/packages/compiler/src/frontend/lowering/lower-stream.ts b/packages/compiler/src/frontend/lowering/lower-stream.ts index cf9f19b9d..687418350 100644 --- a/packages/compiler/src/frontend/lowering/lower-stream.ts +++ b/packages/compiler/src/frontend/lowering/lower-stream.ts @@ -2906,6 +2906,22 @@ class StreamArgTypeThrow extends Error { } } +/** Whether a checked-dynamic pipeline callback can take undefined as its + * error: a callback that is already checked-dynamic, one without + * parameters, and one whose first parameter (or rest element) type holds + * undefined. */ +function callbackErrorAdmitsUndefined(lowerer: Lowerer, cb: IrExpr): boolean { + if (cb.kind !== "dynFrom" || cb.value.type.kind !== "func") return true; + const fnT = cb.value.type; + const first = fnT.params[0]; + if (first === undefined) return true; + const p0 = + fnT.rest === true && fnT.params.length === 1 && first.kind === "array" ? first.elem : first; + if (p0.kind === "dyn" || p0.kind === "jsval" || p0.kind === "undefinedT") return true; + if (p0.kind !== "union") return false; + return lowerer.unions.get(p0.unionId)?.arms.some((a) => a.kind === "undefinedT") ?? false; +} + /** The finished/pipeline completion callback: an inline function lowers * through the option-callback machinery (leading `this`, an `Error | * null` prefix — Node binds the stream and passes the error); any other @@ -3367,12 +3383,16 @@ export function lowerStreamModuleCall( // an async generator) reaches the callback through the // checked-dynamic path. const cb = lowerer.lowerExprExpecting(args[args.length - 1]!, DYN); - // SCR_PIPE_TYPED_CB: typed code's callback takes null for no error + // SCR_PIPE_TYPED_CB: a typed callback whose error parameter cannot + // hold undefined takes null for no error; others take undefined. const stagedArgs = [...staged.args]; const flagsArg = stagedArgs[3]!; if (flagsArg.kind !== "numLit") throw new InternalCompilerError("pipeline flags not a literal"); - if (!isJsSourceFile(ts.getSourceFileOfNode(call))) + if ( + !isJsSourceFile(ts.getSourceFileOfNode(call)) && + !callbackErrorAdmitsUndefined(lowerer, cb) + ) stagedArgs[3] = numLit(flagsArg.value | 128, loc); return staged.wrap({ kind: "libCall", diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index a765c91ee..a17522333 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -496,7 +496,7 @@ import { lowerBuiltinExtraProperty } from "./builtins/properties.js"; import { registerPromisifiedBuiltinDecl } from "./builtins/callable-aliases.js"; import { lowerExecFileAsyncCall } from "./builtins/child-process-async.js"; import { lowerStringDecoderMethodCall, lowerTextCodecCall } from "./builtins/text-codecs.js"; -import { lowerReadlineMethodCall } from "./builtins/readline.js"; +import { isOptionalReadlineCursorRead, lowerReadlineMethodCall } from "./builtins/readline.js"; import { lowerDiagnosticsChannelMethodCall, lowerDiagnosticsChannelProperty, @@ -5547,6 +5547,7 @@ export class Lowerer { if (isIndexedRead(e)) return true; if (ts.isPropertyAccessExpression(e) && isRecordIndexPropertyRead(e)) return true; if (ts.isPropertyAccessExpression(e) && isOptionalProcessStreamProperty(this, e)) return true; + if (ts.isPropertyAccessExpression(e) && isOptionalReadlineCursorRead(this, e)) return true; if (isOptionalProcessVersionsRead(this, e)) return true; if (ts.isPropertyAccessExpression(e) && isSettledIteratorValueRead(this, e)) return true; if (ts.isPropertyAccessExpression(e) && isOsConstantsSymbol(symbolOf(e.expression), this)) diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7500-pipeline-callback-undefined.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7500-pipeline-callback-undefined.ts.json new file mode 100644 index 000000000..fe65b63f0 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7500-pipeline-callback-undefined.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7500-pipeline-callback-undefined.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7501-pipeline-callback-undefined.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7501-pipeline-callback-undefined.mjs.json new file mode 100644 index 000000000..8d52980fe --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7501-pipeline-callback-undefined.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7501-pipeline-callback-undefined.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7502-readline-cursor-undefined.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7502-readline-cursor-undefined.ts.json new file mode 100644 index 000000000..6fd23042e --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7502-readline-cursor-undefined.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7502-readline-cursor-undefined.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7500-pipeline-callback-undefined.ts b/tests/corpus/7500-pipeline-callback-undefined.ts new file mode 100644 index 000000000..e1503ff6b --- /dev/null +++ b/tests/corpus/7500-pipeline-callback-undefined.ts @@ -0,0 +1,120 @@ +// pipeline() with an async function or async generator destination: a +// typed callback whose error parameter admits undefined receives +// undefined when the pipeline succeeds, as in Node, and the error when it +// fails. +import { pipeline, Readable } from "node:stream"; + +function source(): Readable { + return Readable.from(["a", "b", "c"]); +} + +async function join(src: AsyncIterable): Promise { + let s = ""; + for await (const c of src) s += c; + return s; +} + +function optional(): Promise { + return new Promise((resolve) => { + pipeline(source(), join, (err?: Error | null, value?: string) => { + console.log("optional", err, err === undefined, value); + resolve(); + }); + }); +} + +function declared(): Promise { + return new Promise((resolve) => { + pipeline( + source(), + async function (src: AsyncIterable) { + let n = 0; + for await (const c of src) n += c.length; + return n; + }, + (err: Error | null | undefined, value?: number) => { + console.log("declared", err, typeof err, value); + resolve(); + }, + ); + }); +} + +function unknownParam(): Promise { + return new Promise((resolve) => { + pipeline(source(), join, (err: unknown, value: unknown) => { + console.log("unknown", err, value); + resolve(); + }); + }); +} + +function rest(): Promise { + return new Promise((resolve) => { + pipeline( + source(), + async function* (src: AsyncIterable) { + for await (const c of src) yield c.toUpperCase(); + }, + async function (src: AsyncIterable) { + for await (const c of src) void c; + }, + (...args: unknown[]) => { + console.log("rest", args.length, args); + resolve(); + }, + ); + }); +} + +function generatorDestination(): Promise { + return new Promise((resolve) => { + pipeline( + source(), + async function* (src: AsyncIterable) { + for await (const c of src) yield c; + }, + (err: Error | null | undefined, value: unknown) => { + console.log("generator", err, value); + resolve(); + }, + ); + }); +} + +function named(): Promise { + return new Promise((resolve) => { + function done(err: Error | null | undefined, value: string | undefined): void { + console.log("named", err, value); + resolve(); + } + pipeline(source(), join, done); + }); +} + +function failing(): Promise { + return new Promise((resolve) => { + pipeline( + source(), + async function (src: AsyncIterable) { + for await (const c of src) if (c === "b") throw new Error("boom at " + c); + return "never"; + }, + (err?: Error | null, value?: string) => { + console.log("failing", err?.message, value); + resolve(); + }, + ); + }); +} + +async function main(): Promise { + await optional(); + await declared(); + await unknownParam(); + await rest(); + await generatorDestination(); + await named(); + await failing(); +} +void main(); diff --git a/tests/corpus/7501-pipeline-callback-undefined.mjs b/tests/corpus/7501-pipeline-callback-undefined.mjs new file mode 100644 index 000000000..c51738e77 --- /dev/null +++ b/tests/corpus/7501-pipeline-callback-undefined.mjs @@ -0,0 +1,22 @@ +// pipeline() with an async function destination in JavaScript: the +// callback receives undefined and the destination's value on success. +import { pipeline, Readable } from "node:stream"; + +pipeline( + Readable.from(["x", "y"]), + async function (src) { + const out = []; + for await (const c of src) out.push(c); + return out.join("-"); + }, + (err, value) => { + console.log("callback", err, typeof err, value); + pipeline( + Readable.from(["z"]), + async function (src) { + for await (const c of src) void c; + }, + (...args) => console.log("no value", args), + ); + }, +); diff --git a/tests/corpus/7502-readline-cursor-undefined.ts b/tests/corpus/7502-readline-cursor-undefined.ts new file mode 100644 index 000000000..abac985cb --- /dev/null +++ b/tests/corpus/7502-readline-cursor-undefined.ts @@ -0,0 +1,28 @@ +// rl.cursor answers undefined on an interface that is not a terminal and +// the cursor position on a terminal interface. +import * as readline from "node:readline"; +import { PassThrough, Writable } from "node:stream"; + +const input = new PassThrough(); +const rl = readline.createInterface({ input }); +const c = rl.cursor; +console.log("cursor", c, typeof rl.cursor, rl.cursor === undefined); +console.log("arithmetic", rl.cursor + 1, Number.isNaN(rl.cursor * 2)); +console.log("fallback", rl.cursor ?? -1, `${rl.cursor}`); + +const interfaces = [rl]; +function first(): readline.Interface { + return interfaces[0]!; +} +console.log("call receiver", first().cursor); +rl.close(); + +const sink = new Writable({ + write(_chunk: Buffer, _enc, cb) { + cb(); + }, +}); +const terminal = readline.createInterface({ input: new PassThrough(), output: sink, terminal: true }); +terminal.write("abc"); +console.log("terminal", terminal.cursor, typeof terminal.cursor, terminal.cursor + 1); +terminal.close(); diff --git a/tests/harness/corpus-costs/7500-pipeline-callback-undefined.ts.json b/tests/harness/corpus-costs/7500-pipeline-callback-undefined.ts.json new file mode 100644 index 000000000..122c66b27 --- /dev/null +++ b/tests/harness/corpus-costs/7500-pipeline-callback-undefined.ts.json @@ -0,0 +1 @@ +{ "plain": 3783, "san": 4994 } diff --git a/tests/harness/corpus-costs/7501-pipeline-callback-undefined.mjs.json b/tests/harness/corpus-costs/7501-pipeline-callback-undefined.mjs.json new file mode 100644 index 000000000..d9c00a7de --- /dev/null +++ b/tests/harness/corpus-costs/7501-pipeline-callback-undefined.mjs.json @@ -0,0 +1 @@ +{ "plain": 2457, "san": 3243 } diff --git a/tests/harness/corpus-costs/7502-readline-cursor-undefined.ts.json b/tests/harness/corpus-costs/7502-readline-cursor-undefined.ts.json new file mode 100644 index 000000000..7a36db8f3 --- /dev/null +++ b/tests/harness/corpus-costs/7502-readline-cursor-undefined.ts.json @@ -0,0 +1 @@ +{ "plain": 2592, "san": 3421 } From d1f3eda7326b81ca84b9fcdb6a29f574aa14f54a Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 04:50:06 -0500 Subject: [PATCH 13/53] Freeze arrays, Maps and Sets that other references still hold Object.freeze of an array that another reference may still hold stopped at compile time with SC2020, and freezing an array, Map or Set held in an untyped value threw SC2020 at run time. Node freezes the object itself: a frozen array refuses every later write through any reference with the TypeError of the operation's first write, and a frozen Map or Set only stops being extensible, since its entries are not properties. - An array a program may freeze keeps a frozen bit in its storage flags. An array type is freezable when a typed Object.freeze names it, or, in a program that freezes untyped values, when a value of it can be held as a live untyped reference. The writes of those types test the bit with the guard published arrays already use; other array types are unchanged. - The guard's errors follow Node for every operation: splice, fill, copyWithin, sort and reverse throw at their first write, and do not throw when they would write nothing. Published arrays get the same errors, and a statement splice of a published array is now guarded. - A Map or Set records that it is not extensible, for Object.freeze, seal and preventExtensions, typed or untyped. Object.isFrozen, isSealed and isExtensible report it, and its entries stay writable. - Writes to a frozen array through an untyped value are refused with the same errors, as are writes to frozen checked arrays and objects, which now name the property and the object as Node does. - Programs that neither freeze nor publish arrays compile to the same IR and LLVM. Corpus 7406 covers typed and untyped freezes of arrays, Maps and Sets, every array write on frozen arrays of zero, one and three elements, comparator sorts, nested arrays, and frozen arrays that are released, and matches Node. Refs #1374 Co-authored-by: blurname <50226652+blurname@users.noreply.github.com> --- packages/compiler/src/backend/llvm/common.ts | 3 + packages/compiler/src/backend/llvm/emitter.ts | 63 +++++++- .../src/backend/llvm/expr-containers.ts | 39 ++++- .../compiler/src/backend/llvm/lib-dispatch.ts | 1 + .../compiler/src/backend/llvm/lib-shared.ts | 4 + .../compiler/src/backend/publication.test.ts | 31 ++++ packages/compiler/src/backend/publication.ts | 27 +++- .../src/frontend/lowering/frozen-records.ts | 62 +++++--- .../src/frontend/lowering/lower-calls.ts | 57 ++++++- .../compiler/src/frontend/lowering/lowerer.ts | 9 +- .../compiler/src/ir/builtin-signatures.ts | 10 ++ .../src/ir/function-references.test.ts | 1 + packages/compiler/src/ir/ir.ts | 4 + .../corpus/7406-frozen-containers.ts.json | 6 + packages/runtime/src/scr_array.c | 122 +++++++++++++++ packages/runtime/src/scr_array.h | 22 ++- packages/runtime/src/scr_dyn_invoke.c | 35 +++-- packages/runtime/src/scr_json.c | 142 +++++++++++++----- packages/runtime/src/scr_map.c | 6 + packages/runtime/src/scr_map.h | 8 + packages/runtime/src/scr_publish.c | 58 +------ packages/runtime/src/scr_rc.h | 8 +- tests/corpus/7406-frozen-containers.ts | 132 ++++++++++++++++ .../7406-frozen-containers.ts.json | 1 + 24 files changed, 690 insertions(+), 161 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7406-frozen-containers.ts.json create mode 100644 tests/corpus/7406-frozen-containers.ts create mode 100644 tests/harness/corpus-costs/7406-frozen-containers.ts.json diff --git a/packages/compiler/src/backend/llvm/common.ts b/packages/compiler/src/backend/llvm/common.ts index ac4e28708..cca1b603b 100644 --- a/packages/compiler/src/backend/llvm/common.ts +++ b/packages/compiler/src/backend/llvm/common.ts @@ -29,6 +29,9 @@ export const SCR_ARR_FIELD = { * (SCR_ARR_STATES in scr_array.h). */ export const SCR_ARR_STATES = 16; +/** The `spare` flag of a frozen %ScrArr (SCR_ARR_FROZEN in scr_array.h). */ +export const SCR_ARR_FROZEN = 32; + /** Loads the i32 `len` or `cap` of a %ScrArr at `ptr` into `dest` as a * size-type count. */ export function loadArrCount( diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index c0cc9fef2..a66e00736 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -184,7 +184,7 @@ import { } from "./string-slices.js"; import { LazyCaptures } from "./lazy-captures.js"; import { StackCaptures } from "./stack-captures.js"; -import { emitFieldAbsentTest, f64Lit, SCR_ARR_FIELD } from "./common.js"; +import { emitFieldAbsentTest, f64Lit, SCR_ARR_FIELD, SCR_ARR_FROZEN } from "./common.js"; import { ffiExtendsNarrowIntegers } from "../targets.js"; import { emitLiteralExpr, @@ -6264,6 +6264,41 @@ export class LlEmitter { return ok; } + /** Branches on "the array `arr` is frozen" (SCR_ARR_FROZEN in its storage + * flags, scr_array.h), or published too when `published`; like + * emitPublishedTest, leaves the builder in the cold block and answers the + * continuation label. */ + private emitFrozenArrayTest(arr: string, published: boolean): string { + const B = this.B; + const sparep = B.tmp(), + spare = B.tmp(), + bit = B.tmp(); + let frozen = B.tmp(); + B.line( + `${sparep} = getelementptr inbounds %ScrArr, ptr ${arr}, i32 0, i32 ${SCR_ARR_FIELD.spare}`, + ); + B.line(`${spare} = load i16, ptr ${sparep}`); + B.line(`${bit} = and i16 ${spare}, ${SCR_ARR_FROZEN}`); + B.line(`${frozen} = icmp ne i16 ${bit}, 0`); + if (published) { + const rc = B.tmp(), + immortal = B.tmp(), + either = B.tmp(); + B.line(`${rc} = load ${this.sizeType}, ptr ${arr}`); + B.line(`${immortal} = icmp eq ${this.sizeType} ${rc}, -1`); + B.line(`${either} = or i1 ${frozen}, ${immortal}`); + frozen = either; + } + this.declare(`declare i1 @llvm.expect.i1(i1, i1)`); + const expected = B.tmp(); + B.line(`${expected} = call i1 @llvm.expect.i1(i1 ${frozen}, i1 false)`); + const cold = B.newLabel("arr.frozen"), + ok = B.newLabel("arr.ok"); + B.condBr(expected, cold, ok); + B.startBlock(cold); + return ok; + } + private publishedNamesSym: string | null = null; /** The class-name table the hierarchy field guards index by preorder. */ @@ -6328,15 +6363,29 @@ export class LlEmitter { op: number, index = f64Lit(0), count = f64Lit(0), + third: string | null = null, ): void { - if (this.publishedTypes?.containerGuarded(arrType) !== true) return; + const published = this.publishedTypes; + if (published?.containerGuarded(arrType) !== true) return; const B = this.B; - const ok = this.emitPublishedTest(arr); - this.declare(`declare zeroext i1 @scr_throw_published_array(i32, ptr, double, double)`); + const guards = published.arrayGuards(arrType); + const ok = guards.frozen + ? this.emitFrozenArrayTest(arr, guards.published) + : this.emitPublishedTest(arr); const threw = B.tmp(); - B.line( - `${threw} = call zeroext i1 @scr_throw_published_array(i32 ${op}, ptr ${arr}, double ${index}, double ${count})`, - ); + if (third === null) { + this.declare(`declare zeroext i1 @scr_throw_published_array(i32, ptr, double, double)`); + B.line( + `${threw} = call zeroext i1 @scr_throw_published_array(i32 ${op}, ptr ${arr}, double ${index}, double ${count})`, + ); + } else { + this.declare( + `declare zeroext i1 @scr_throw_published_array_range(i32, ptr, double, double, double)`, + ); + B.line( + `${threw} = call zeroext i1 @scr_throw_published_array_range(i32 ${op}, ptr ${arr}, double ${index}, double ${count}, double ${third})`, + ); + } const unwind = B.newLabel("pub.throw"); B.condBr(threw, unwind, ok); B.startBlock(unwind); diff --git a/packages/compiler/src/backend/llvm/expr-containers.ts b/packages/compiler/src/backend/llvm/expr-containers.ts index 1ba5da818..93cc615a4 100644 --- a/packages/compiler/src/backend/llvm/expr-containers.ts +++ b/packages/compiler/src/backend/llvm/expr-containers.ts @@ -257,6 +257,16 @@ function emitDiscarded(host: LlvmEmitterContext, e: IrExpr): void { const start = host.emitExpr(e.args[0]!); const count = e.args[1] ? host.emitExpr(e.args[1]).name : F64_INF; const values = items.map((item) => host.emitExpr(item)); + // Before the values move into the buffer: a throw releases them with the + // frame. + host.emitPublishedArrayGuard( + r.name, + e.receiver.type, + 10, + start.name, + count, + f64Lit(values.length), + ); if (acc === "ref") values.forEach((value) => host.toHeap(value)); let buffer = "null"; if (values.length > 0) { @@ -978,7 +988,25 @@ export function emitArrIntrinsic( case "fill": case "fillUndefined": { const args = e.args.map((arg) => host.emitExpr(arg)); - host.emitPublishedArrayGuard(r.name, e.receiver.type, 7); + // fill's range is its last two arguments; copyWithin's target, + // start and end are its three. + if (e.method === "copyWithin") + host.emitPublishedArrayGuard( + r.name, + e.receiver.type, + 9, + args[0]!.name, + args[1]!.name, + args[2]!.name, + ); + else + host.emitPublishedArrayGuard( + r.name, + e.receiver.type, + 8, + args[args.length - 2]!.name, + args[args.length - 1]!.name, + ); if (host.checkedFills.has(e)) { // A new int32 array with checked stores, filled with a value that // may not be an int32: widen it first then. @@ -1042,7 +1070,7 @@ export function emitArrIntrinsic( // removes to the end (+Infinity, the slice convention). const start = host.emitExpr(e.args[0]!); const cnt = e.args[1] ? host.emitExpr(e.args[1]).name : F64_INF; - host.emitPublishedArrayGuard(r.name, e.receiver.type, 7); + host.emitPublishedArrayGuard(r.name, e.receiver.type, 10, start.name, cnt, f64Lit(0)); host.declare(`declare ptr @scr_arr_splice(ptr, double, double)`); const t = B.tmp(); B.line(`${t} = call ptr @scr_arr_splice(ptr ${r.name}, double ${start.name}, double ${cnt})`); @@ -1052,7 +1080,12 @@ export function emitArrIntrinsic( const start = host.emitExpr(e.args[0]!); const count = host.emitExpr(e.args[1]!); const items = emitBorrowedInput(host, e.args[2]!); - host.emitPublishedArrayGuard(r.name, e.receiver.type, 7); + if (host.publishedTypes?.containerGuarded(e.receiver.type) === true) { + host.declare(`declare double @scr_arr_len(ptr)`); + const n = B.tmp(); + B.line(`${n} = call double @scr_arr_len(ptr ${items.name})`); + host.emitPublishedArrayGuard(r.name, e.receiver.type, 10, start.name, count.name, n); + } host.declare(`declare ptr @scr_arr_splice_insert(ptr, double, double, ptr)`); const t = B.tmp(); B.line( diff --git a/packages/compiler/src/backend/llvm/lib-dispatch.ts b/packages/compiler/src/backend/llvm/lib-dispatch.ts index 4f1a4fef8..cc8953fc4 100644 --- a/packages/compiler/src/backend/llvm/lib-dispatch.ts +++ b/packages/compiler/src/backend/llvm/lib-dispatch.ts @@ -891,6 +891,7 @@ export function emitLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlValue { case "blob": case "wasm": case "array": + case "collection": case "record": case "disposable": case "punycode": diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index 6fbad7e11..f8e59e7b7 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -1016,6 +1016,10 @@ export const LIB_FN_SYMS: Record = { "class.throwNotExtensible": "scr_class_throw_not_extensible", "record.freeze": "scr_record_freeze", "record.throwFrozen": "scr_record_throw_frozen", + "array.freeze": "scr_arr_freeze", + "array.isFrozen": "scr_arr_is_frozen", + "collection.preventExtensions": "scr_map_prevent_extensions", + "collection.isExtensible": "scr_map_is_extensible", "dyn.nativeSetNew": "scr_dyn_native_set_new", "dyn.nativeSetIs": "scr_dyn_native_set_is", "dyn.nativeMapIs": "scr_dyn_native_map_is", diff --git a/packages/compiler/src/backend/publication.test.ts b/packages/compiler/src/backend/publication.test.ts index 149ea4bb9..7b3e5f5cc 100644 --- a/packages/compiler/src/backend/publication.test.ts +++ b/packages/compiler/src/backend/publication.test.ts @@ -85,3 +85,34 @@ test("published types close over fields, containers and whole hierarchies", () = computePublishedTypes({ ...withStores, functions: withStores.functions.slice(1) }), ).toBeNull(); }); + +test("freezable array types carry the array write guard without a publish", () => { + const numbers = arrayOf(F64); + const push: IrStmt = { + kind: "exprStmt", + expr: { + kind: "arrIntrinsic", + method: "push", + receiver: ref("a", numbers), + args: [{ kind: "numLit", value: 1, type: F64, loc }], + type: F64, + loc, + }, + loc, + }; + const mod: IrModule = { + irVersion: 17, + sourceFile: loc.file, + entry: "main", + functions: [fn("main", []), fn("pushNumber", [push])], + freezableArrays: ["array"], + }; + const guarded = computePublishedTypes(mod)!; + expect(guarded.containerGuarded(numbers)).toBe(true); + expect(guarded.arrayGuards(numbers)).toEqual({ published: false, frozen: true }); + expect(guarded.containerGuarded(arrayOf(STRING))).toBe(false); + // A frozen array's push throws, so the function may throw. + expect(computeMayThrow(mod).fns.has("pushNumber")).toBe(true); + const { freezableArrays: _frozen, ...plain } = mod; + expect(computePublishedTypes(plain)).toBeNull(); +}); diff --git a/packages/compiler/src/backend/publication.ts b/packages/compiler/src/backend/publication.ts index af65fa39a..7f7918fb1 100644 --- a/packages/compiler/src/backend/publication.ts +++ b/packages/compiler/src/backend/publication.ts @@ -46,6 +46,10 @@ export class PublishedTypes { readonly records = new Set(); /** typeKeys of array, map and set types. */ readonly containers = new Set(); + /** typeKeys of the array types the program may freeze + * (IrModule.freezableArrays): a frozen array refuses writes like a + * published one, so their stores carry the same guard. */ + readonly frozenArrays = new Set(); /** Constructor stores into a `this` nothing else can reach yet * (immutability.ts): never a published target, so never guarded. */ privateStores: ReadonlySet = new Set(); @@ -64,10 +68,16 @@ export class PublishedTypes { } containerGuarded(t: IrType): boolean { - return ( - (t.kind === "array" || t.kind === "map" || t.kind === "set") && - this.containers.has(typeKey(t)) - ); + if (t.kind !== "array" && t.kind !== "map" && t.kind !== "set") return false; + const key = typeKey(t); + return this.containers.has(key) || this.frozenArrays.has(key); + } + + /** Whether an array of type `t` may be published, and whether it may be + * frozen: the two states its write guard tests. */ + arrayGuards(t: IrType): { published: boolean; frozen: boolean } { + const key = typeKey(t); + return { published: this.containers.has(key), frozen: this.frozenArrays.has(key) }; } } @@ -82,7 +92,13 @@ export function computePublishedTypes(mod: IrModule): PublishedTypes | null { }; for (const fn of mod.functions) everyStmtList(fn.body, { expr: visit, stmt: (_s: IrStmt) => true }); - if (roots.length === 0) return null; + const frozenArrays = mod.freezableArrays ?? []; + if (roots.length === 0 && frozenArrays.length === 0) return null; + if (roots.length === 0) { + const frozen = new PublishedTypes(); + for (const key of frozenArrays) frozen.frozenArrays.add(key); + return frozen; + } const classes = new Map((mod.classes ?? []).map((c) => [c.name, c])); const children = new Map(); @@ -95,6 +111,7 @@ export function computePublishedTypes(mod: IrModule): PublishedTypes | null { const records = new Map((mod.records ?? []).map((r) => [r.id, r])); const unions = new Map((mod.unions ?? []).map((u) => [u.id, u])); const out = new PublishedTypes(); + for (const key of frozenArrays) out.frozenArrays.add(key); out.privateStores = computeConstructionFacts(mod).privateStores; const seen = new Set(); const stack = [...roots]; diff --git a/packages/compiler/src/frontend/lowering/frozen-records.ts b/packages/compiler/src/frontend/lowering/frozen-records.ts index f61de39ec..30eb9e455 100644 --- a/packages/compiler/src/frontend/lowering/frozen-records.ts +++ b/packages/compiler/src/frontend/lowering/frozen-records.ts @@ -3,6 +3,7 @@ import { BOOL, VOID, shapeHasAccessorSlots, + typeKey, type IrExpr, type IrFunction, type IrStmt, @@ -15,24 +16,31 @@ import type { Lowerer } from "./lowerer.js"; import { recordHasOwnHelper } from "./lower-calls.js"; import { isStrictCode } from "./strict-mode.js"; -/* Object.freeze of records that other references can still write. Only the - * shapes a program may freeze pay for it: their values keep a frozen bit in - * the rc word (scr_rc.h SCR_RC_FROZEN), their release ignores the bit, and - * every write to them checks it first (guardFrozenRecordWrites). A shape is +/* Object.freeze of records and arrays that other references can still + * write. Only the types a program may freeze pay for it. A record keeps its + * frozen bit in the rc word (scr_rc.h SCR_RC_FROZEN), which its release + * ignores, and every write to it checks the bit first + * (guardFrozenWrites). An array keeps it in its storage flags + * (scr_array.h SCR_ARR_FROZEN), which the backend's array write guards test + * for the array types the module lists (IrModule.freezableArrays). A type is * freezable when a static Object.freeze names it, or, in a program that * freezes untyped values, when a value of it or of a type reaching it can be * held as a live untyped reference. */ type RecordType = Extract; type RecordWrite = Extract; +type FreezeFn = "record.freeze" | "array.freeze" | "collection.preventExtensions"; -/** Object.freeze of a record: sets its frozen bit and answers the record. */ -export function lowerRecordFreeze( +/** Object.freeze of a record or an array, or Object.freeze, seal or + * preventExtensions of a Map or Set: `fn` sets the value's bit, and the + * call answers the value. */ +export function lowerNativeFreeze( lowerer: Lowerer, + fn: FreezeFn, target: IrExpr, - type: RecordType, loc: SrcLoc, ): IrExpr { + const type = target.type; const slot = lowerer.declareHiddenLocal("%freezeTarget", type); const receiver = (): IrExpr => varRef(slot.id, type, loc); return { @@ -41,7 +49,7 @@ export function lowerRecordFreeze( { kind: "varDecl", localId: slot.id, init: target, loc }, { kind: "exprStmt", - expr: { kind: "libCall", fn: "record.freeze", args: [receiver()], type: VOID, loc }, + expr: { kind: "libCall", fn, args: [receiver()], type: VOID, loc }, loc, }, ], @@ -92,9 +100,14 @@ function holdsAbsentState(e: IrExpr): boolean { return e.kind === "ternary" && (holdsAbsentState(e.then) || holdsAbsentState(e.else_)); } -/** The shapes whose values the program may freeze. */ -function freezableShapes(lowerer: Lowerer, functions: IrFunction[]): Set { +/** The record shapes and array types (typeKeys) whose values the program + * may freeze. */ +function freezableTypes( + lowerer: Lowerer, + functions: IrFunction[], +): { shapes: Set; arrays: Set } { const shapes = new Set(); + const arrays = new Set(); const live: IrType[] = []; let dynFreeze = false; for (const fn of functions) @@ -103,13 +116,16 @@ function freezableShapes(lowerer: Lowerer, functions: IrFunction[]): Set if (e.kind === "libCall" && e.fn === "record.freeze") { const target = e.args[0]; if (target && target.type.kind === "record") shapes.add(target.type.shapeId); + } else if (e.kind === "libCall" && e.fn === "array.freeze") { + const target = e.args[0]; + if (target && target.type.kind === "array") arrays.add(typeKey(target.type)); } else if (e.kind === "libCall" && e.fn === "dyn.freeze") dynFreeze = true; else if (e.kind === "dynFrom") live.push(e.value.type); return true; }, stmt: () => true, }); - if (!dynFreeze) return shapes; + if (!dynFreeze) return { shapes, arrays }; // An untyped reference to a class instance, or to a record or array // boxed live, reaches the values its fields and elements hold as further // live references. Class instances also cross into untyped code through @@ -135,7 +151,14 @@ function freezableShapes(lowerer: Lowerer, functions: IrFunction[]): Set seen.add(`union:${type.unionId}`); for (const arm of lowerer.unions.get(type.unionId)?.arms ?? []) stack.push(arm); break; - case "array": + case "array": { + const key = typeKey(type); + if (seen.has(key)) break; + seen.add(key); + arrays.add(key); + stack.push(type.elem); + break; + } case "set": stack.push(type.elem); break; @@ -156,7 +179,7 @@ function freezableShapes(lowerer: Lowerer, functions: IrFunction[]): Set break; } } - return shapes; + return { shapes, arrays }; } /** The guarded form of a write to a freezable record: the receiver and the @@ -186,15 +209,18 @@ function guardedWrite( } /** Checks the frozen bit before every write to a record of a freezable - * shape. A write to a frozen record throws Node's TypeError in strict mode - * code; in sloppy-mode code it leaves the record unchanged. Deleting a key - * the record does not have succeeds either way. */ -export function guardFrozenRecordWrites(lowerer: Lowerer, functions: IrFunction[]): void { - const freezable = freezableShapes(lowerer, functions); + * shape, and lists the freezable array types, whose writes the backend + * guards (IrModule.freezableArrays). A write to a frozen record throws + * Node's TypeError in strict mode code; in sloppy-mode code it leaves the + * record unchanged. Deleting a key the record does not have succeeds either + * way. */ +export function guardFrozenWrites(lowerer: Lowerer, functions: IrFunction[]): void { + const { shapes: freezable, arrays } = freezableTypes(lowerer, functions); for (const id of freezable) { const shape = lowerer.shapes.get(id); if (shape) shape.freezable = true; } + for (const key of arrays) lowerer.freezableArrays.add(key); if (freezable.size === 0) return; const lifted = lowerer.liftedFns.length; for (const fn of functions) { diff --git a/packages/compiler/src/frontend/lowering/lower-calls.ts b/packages/compiler/src/frontend/lowering/lower-calls.ts index e2ed5f9fe..819f90534 100644 --- a/packages/compiler/src/frontend/lowering/lower-calls.ts +++ b/packages/compiler/src/frontend/lowering/lower-calls.ts @@ -245,7 +245,7 @@ import { import { fenceSymbolFieldCopy } from "./symbol-fields.js"; import { lowerClassDataDescriptor, lowerClassDescriptorRead } from "./class-descriptors.js"; import { lowerClassFieldDefine, lowerClassIntegrity } from "./class-attributes.js"; -import { freezableRecordType, lowerRecordFreeze } from "./frozen-records.js"; +import { freezableRecordType, lowerNativeFreeze } from "./frozen-records.js"; import { classStaticDataFor } from "./class-static-data.js"; import { lowerUrlFactory } from "./lower-url.js"; import { nullishValueUnitOf } from "./binding-analysis.js"; @@ -8422,6 +8422,22 @@ function lowerObjectStaticCall( locOf(call), ); if (fnIntegrity) return fnIntegrity; + // A Map or Set has no own properties: freeze, seal and preventExtensions + // all only make it not extensible. + if (value.type.kind === "map" || value.type.kind === "set") { + if (member === "preventExtensions" || member === "seal") + return lowerNativeFreeze(lowerer, "collection.preventExtensions", value, locOf(call)); + const extensible: IrExpr = { + kind: "libCall", + fn: "collection.isExtensible", + args: [value], + type: BOOL, + loc: locOf(call), + }; + return member === "isExtensible" + ? extensible + : { kind: "unary", op: "!", operand: extensible, type: BOOL, loc: locOf(call) }; + } if (member === "preventExtensions" || member === "seal") { const integrity = lowerClassIntegrity(lowerer, call, member, value); if (integrity) return integrity; @@ -8461,15 +8477,36 @@ function lowerObjectStaticCall( // reference would need a runtime frozen bit (strict mode throws). if (member === "isFrozen" && call.arguments.nodes.length === 1) { const argNode = call.arguments.nodes[0]!; - if (tryLowerExpression(lowerer, argNode)?.type.kind === "record") { - const record = lowerer.lowerExpr(argNode); - if (record.type.kind === "record") + const probedKind = tryLowerExpression(lowerer, argNode)?.type.kind; + if ( + probedKind === "record" || + probedKind === "array" || + probedKind === "map" || + probedKind === "set" + ) { + const native = lowerer.lowerExpr(argNode); + const loc = locOf(call); + if (native.type.kind === "record" || native.type.kind === "array") return { kind: "libCall", - fn: "record.isFrozen", - args: [record], + fn: native.type.kind === "record" ? "record.isFrozen" : "array.isFrozen", + args: [native], type: BOOL, - loc: locOf(call), + loc, + }; + if (native.type.kind === "map" || native.type.kind === "set") + return { + kind: "unary", + op: "!", + operand: { + kind: "libCall", + fn: "collection.isExtensible", + args: [native], + type: BOOL, + loc, + }, + type: BOOL, + loc, }; } const value = lowerer.lowerExprExpecting(argNode, DYN); @@ -8523,7 +8560,11 @@ function lowerObjectStaticCall( const integrity = lowerClassIntegrity(lowerer, call, "freeze", value); if (integrity) return integrity; if (freezableRecordType(lowerer, value.type)) - return lowerRecordFreeze(lowerer, value, value.type, locOf(call)); + return lowerNativeFreeze(lowerer, "record.freeze", value, locOf(call)); + if (value.type.kind === "array") + return lowerNativeFreeze(lowerer, "array.freeze", value, locOf(call)); + if (value.type.kind === "map" || value.type.kind === "set") + return lowerNativeFreeze(lowerer, "collection.preventExtensions", value, locOf(call)); lowerer.noLowering( "Object.freeze of a possibly-aliased value", call, diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index ba6d1214f..95deb648b 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -150,7 +150,7 @@ import { defaultAfterUndefined, lowerStaticallyUndefinedArgument } from "./optio import { type ProtocolDispatch } from "./class-protocol-dispatch.js"; import { ClassDynamicDispatch, trackClassFieldCreation } from "./class-dynamic-dispatch.js"; import { guardClassAttributeWrites } from "./class-attributes.js"; -import { guardFrozenRecordWrites, markSloppyRecordWrites } from "./frozen-records.js"; +import { guardFrozenWrites, markSloppyRecordWrites } from "./frozen-records.js"; import { applyHandleProperties } from "./handle-properties.js"; import { ClassProtocols } from "./class-protocols.js"; import { earlyReadVarSymbols } from "./var-hoisting.js"; @@ -3246,6 +3246,8 @@ export class Lowerer { * absent-state writes of `delete r.f` (markSloppyRecordWrites). */ readonly sloppyRecordWrites = new Set(); readonly recordFieldDeletes = new Set(); + /** typeKeys of the array types the program may freeze (IrModule.freezableArrays). */ + readonly freezableArrays = new Set(); readonly liftedFns: IrFunction[] = []; /** The first lifted function of each name (liftedFnNamed), for the * liftedFns indexed so far. liftedFns only grows. */ @@ -7694,7 +7696,7 @@ export class Lowerer { finalizePrivateBrands(this, functions); trackClassFieldCreation(this, functions); guardClassAttributeWrites(this, functions); - guardFrozenRecordWrites(this, functions); + guardFrozenWrites(this, functions); applyHandleProperties(this, functions); // Globals typed by a class that never REGISTERED (a JS class whose @@ -7803,6 +7805,9 @@ export class Lowerer { ...(this.threadSlotKeys.size > 0 ? { threadSlotKeys: [...this.threadSlotKeys].sort() } : {}), + ...(this.freezableArrays.size > 0 + ? { freezableArrays: [...this.freezableArrays].sort() } + : {}), ...(isNodeEsmFile(this.entry, this.program) ? {} : { commonJsEntry: true }), ...(this.ffiImports.length > 0 ? { ffiImports: [...this.ffiImports] } : {}), }; diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index 1b2c1082d..717f7ed8a 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -283,6 +283,16 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ * object's text in the message ("#") and the operation (0 assign, * 1 add, 2 delete). */ "record.throwFrozen": { argTypes: [STRING, STRING, F64], result: VOID }, + /** Object.freeze of an array (scr_arr_freeze): sets its frozen bit, which + * the write guards of IrModule.freezableArrays test, and isFrozen reads. + * The receiver is any array type. */ + "array.freeze": { argTypes: [null], result: VOID }, + "array.isFrozen": { argTypes: [null], result: BOOL }, + /** Object.freeze, seal or preventExtensions of a Map or Set + * (scr_map_prevent_extensions), which only makes it not extensible: its + * entries stay writable. The receiver is any map or set type. */ + "collection.preventExtensions": { argTypes: [null], result: VOID }, + "collection.isExtensible": { argTypes: [null], result: BOOL }, "dyn.nativeSetNew": { argTypes: [DYN], result: DYN }, "dyn.nativeSetIs": { argTypes: [DYN], result: BOOL }, "dyn.nativeMapIs": { argTypes: [DYN], result: BOOL }, diff --git a/packages/compiler/src/ir/function-references.test.ts b/packages/compiler/src/ir/function-references.test.ts index 26539b17a..91e1d5f05 100644 --- a/packages/compiler/src/ir/function-references.test.ts +++ b/packages/compiler/src/ir/function-references.test.ts @@ -219,6 +219,7 @@ const MODULE_FIELDS: Record>, Role> = { "wasmUntranslated[].sha256": "other", "wasmUntranslated[].reason": "other", "threadSlotKeys[]": "other", + "freezableArrays[]": "other", }; const loc = { file: "references.ts", start: 0, end: 0 }; diff --git a/packages/compiler/src/ir/ir.ts b/packages/compiler/src/ir/ir.ts index 3211e48f3..df23f0360 100644 --- a/packages/compiler/src/ir/ir.ts +++ b/packages/compiler/src/ir/ir.ts @@ -1056,6 +1056,10 @@ export interface IrModule { * each one's hierarchy carries a slot word (the slots.* intrinsics; * backend/llvm/thread-slots.ts). */ threadSlotKeys?: string[]; + /** typeKeys of the array types whose values the program may freeze + * (frontend frozen-records.ts): their writes test the array's frozen bit + * (SCR_ARR_FROZEN) and throw Node's TypeError when it is set. */ + freezableArrays?: string[]; /** The entry module is CommonJS under Node's classification. Node runs a * CommonJS main synchronously and drains process.nextTick callbacks * before promise jobs at the first checkpoint; an ES-module main's diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7406-frozen-containers.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7406-frozen-containers.ts.json new file mode 100644 index 000000000..34fb3ba0e --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7406-frozen-containers.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7406-frozen-containers.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_array.c b/packages/runtime/src/scr_array.c index 91328a1e8..af810483f 100644 --- a/packages/runtime/src/scr_array.c +++ b/packages/runtime/src/scr_array.c @@ -1689,6 +1689,126 @@ void scr_arr_release(ScrArr *a) { double scr_arr_len(ScrArr *a) { return SCR_ARR_LEN_F64(a->len); } +/* ── frozen arrays ────────────────────────────────────────────────────── + * Node's TypeErrors for writes to a frozen array, shared by the emitted + * guards of published and frozen arrays and the checked-dynamic tier. */ +static void scr_frozen_throw(const char *before, double index, const char *after) { + char text[64]; + size_t n = scr_f64_to_str(index, text); + ScrJsonBuf buffer; + scr_jb_init(&buffer); + scr_jb_puts(&buffer, before); + scr_jb_write(&buffer, text, n); + scr_jb_puts(&buffer, after); + scr_throw_error(SCR_ERR_TYPE, scr_jb_finish(&buffer)); +} + +/* A relative array index argument (fill, copyWithin, splice): ToIntegerOrInfinity + * of `value`, counted from the end when negative, clamped to [0, len]. */ +static size_t scr_frozen_relative_index(double value, size_t len) { + if (value != value) return 0; + double n = trunc(value); + if (n < 0) return n + (double)len <= 0 ? 0 : (size_t)(n + (double)len); + return n >= (double)len ? len : (size_t)n; +} + +static void scr_frozen_read_only(double index) { + scr_frozen_throw("Cannot assign to read only property '", index, "' of object '[object Array]'"); +} + +static void scr_frozen_not_extensible(double index) { + scr_frozen_throw("Cannot add property ", index, ", object is not extensible"); +} + +static void scr_frozen_undeletable(double index) { + scr_frozen_throw("Cannot delete property '", index, "' of [object Array]"); +} + +/* Each case finds the operation's first write in the order the spec makes + * its writes. */ +bool scr_array_frozen_error(int op, size_t len, double x, double y, double z) { + switch (op) { + case 0: + if (x >= 0 && x < (double)len && x == (double)(size_t)x) scr_frozen_read_only(x); + else scr_frozen_not_extensible(x); + return true; + case 1: + if (y == 0) break; + scr_frozen_not_extensible((double)len); + return true; + case 2: + case 6: + if (len == 0) break; + if (op == 6 && len > 1) scr_frozen_read_only(0); + else scr_frozen_undeletable((double)(len - 1)); + return true; + case 3: break; + case 4: + if (y == 0) break; + scr_frozen_not_extensible(len ? (double)len + y - 1 : 0); + return true; + case 5: + if (!(x >= 0 && x < (double)len && x == (double)(size_t)x)) return false; + scr_frozen_undeletable(x); + return true; + case 7: + if (len < 2) return false; + scr_frozen_read_only(0); + return true; + case 8: { + size_t k = scr_frozen_relative_index(x, len), final = scr_frozen_relative_index(y, len); + if (k >= final) return false; + scr_frozen_read_only((double)k); + return true; + } + case 9: { + size_t to = scr_frozen_relative_index(x, len), from = scr_frozen_relative_index(y, len); + size_t final = scr_frozen_relative_index(z, len); + if (final <= from || len <= to) return false; + size_t count = final - from < len - to ? final - from : len - to; + /* An overlapping copy toward the end writes its last index first. */ + if (from < to && to < from + count) to += count - 1; + scr_frozen_read_only((double)to); + return true; + } + case 10: { + size_t start = scr_frozen_relative_index(x, len); + double count = y != y ? 0 : trunc(y); + size_t removed = count <= 0 ? 0 : count >= (double)(len - start) ? len - start : (size_t)count; + size_t items = (size_t)z; + if (items < removed) { + /* Elements after the removed ones move down, then the tail goes. */ + if (start < len - removed) scr_frozen_read_only((double)(start + items)); + else scr_frozen_undeletable((double)(len - 1)); + return true; + } + if (items > removed) { + /* Elements after the removed ones move up, past the end first. */ + if (len - removed > start) scr_frozen_not_extensible((double)(len - removed + items - 1)); + else if (start < len) scr_frozen_read_only((double)start); + else scr_frozen_not_extensible((double)start); + return true; + } + if (items > 0) { + scr_frozen_read_only((double)start); + return true; + } + break; + } + default: + return false; + } + static const char length[] = "Cannot assign to read only property 'length' of object '[object Array]'"; + scr_throw_error_msg(SCR_ERR_TYPE, length, sizeof length - 1); + return true; +} + +void scr_arr_freeze(ScrArr *a) { + if (a->rc != SIZE_MAX) a->spare |= SCR_ARR_FROZEN; +} + +bool scr_arr_is_frozen(const ScrArr *a) { return scr_arr_frozen(a); } + bool scr_arr_has(const ScrArr *a, double i) { size_t idx; if (!scr_arr_valid_index(i, &idx)) return scr_arr_prop_get(a, i, NULL); @@ -4990,6 +5110,8 @@ bool scr_arr_sort_dense(ScrArr *a, ScrClosure *f, uint32_t arity, bool owned, in if (a->rc == SIZE_MAX || elem == SCR_ELEM_CREF || !scr_arr_dense_values(a)) return false; size_t n = a->len; if (n < 2) return true; + /* A frozen array takes the snapshot path, whose write-back throws. */ + if (scr_arr_frozen(a)) return false; if (n > SIZE_MAX / 2 / sizeof(uint64_t)) scr_arr_oom(); ScrSortCompare c = scr_sort_compare_for(a, f, arity, order, key); /* Nothing else can reach a private copy: its own storage is sorted, and diff --git a/packages/runtime/src/scr_array.h b/packages/runtime/src/scr_array.h index 4666750ea..d0c0ea1a8 100644 --- a/packages/runtime/src/scr_array.h +++ b/packages/runtime/src/scr_array.h @@ -128,7 +128,8 @@ typedef struct ScrArr { * header. Kept beside `elem` so the RC fast paths test it with one load. */ uint8_t cyc; /* The low SCR_ARR_OPS_SHIFT bits: storage flags (SCR_ARR_INLINE_SLOTS, - * SCR_ARR_HEAD_ROOM, SCR_ARR_UNSHIFTED, SCR_ARR_HAS_EXT, SCR_ARR_STATES). The rest: the + * SCR_ARR_HEAD_ROOM, SCR_ARR_UNSHIFTED, SCR_ARR_HAS_EXT, SCR_ARR_STATES) + * and SCR_ARR_FROZEN. The rest: the * registry id of the element type's ops (scr_arr_ops), 0 for every kind * but SCR_ELEM_REF and SCR_ELEM_CREF. */ uint16_t spare; @@ -204,6 +205,10 @@ _Static_assert(sizeof(ScrArr) == sizeof(size_t) + 16 + sizeof(void *), #define SCR_ARR_UNSHIFTED 4u #define SCR_ARR_HAS_EXT 8u #define SCR_ARR_STATES 16u +/* Object.freeze of the array (scr_arr_freeze). Emitted guards test it + * before the writes of array types a program may freeze; the runtime's own + * writes assume it is clear. */ +#define SCR_ARR_FROZEN 32u /* An array starts without state bytes. Without them, every index below * `len` is a present value in dense storage: len <= cap, there are no @@ -214,6 +219,19 @@ _Static_assert(sizeof(ScrArr) == sizeof(size_t) + 16 + sizeof(void *), * states (scr_arr_add_states), which it keeps. */ static inline bool scr_arr_has_states(const ScrArr *a) { return (a->spare & SCR_ARR_STATES) != 0; } +static inline bool scr_arr_frozen(const ScrArr *a) { return (a->spare & SCR_ARR_FROZEN) != 0; } +/* Sets SCR_ARR_FROZEN; immortal arrays are left as they are. */ +void scr_arr_freeze(ScrArr *a); +bool scr_arr_is_frozen(const ScrArr *a); +/* Throws Node's TypeError for the first write that the array operation + * `op` makes to a frozen array of length `len`, or answers false when the + * operation writes nothing. op: 0 index write (x = index), 1 push (y = + * argument count), 2 pop, 3 length write, 4 unshift (y = argument count), + * 5 delete (x = index), 6 shift, 7 reverse or sort, 8 fill (x = start, y = + * end), 9 copyWithin (x = target, y = start, z = end), 10 splice (x = + * start, y = delete count, z = item count). */ +bool scr_array_frozen_error(int op, size_t len, double x, double y, double z); + /* The start of the dense slots' allocation: `data` less any head room. */ static inline uint64_t *scr_arr_block(const ScrArr *a) { return (a->spare & SCR_ARR_HEAD_ROOM) ? a->data - (size_t)a->data[-1] : a->data; @@ -236,7 +254,7 @@ static inline struct ScrArrExt *scr_arr_ext_of(const ScrArr *a) { * Id 0 is no ops; ids are assigned on each descriptor's first array, in * any thread. A program with more element types than the field holds keeps * the ops of the rest in the array's ext (SCR_ARR_OPS_OVERFLOW). */ -#define SCR_ARR_OPS_SHIFT 5 +#define SCR_ARR_OPS_SHIFT 6 #define SCR_ARR_OPS_OVERFLOW (UINT16_MAX >> SCR_ARR_OPS_SHIFT) extern const ScrArrOps *scr_arr_ops_table[SCR_ARR_OPS_OVERFLOW]; /* By registry id: 1 when the descriptor has SCR_ARR_OPS_RELEASE_NOTHING. diff --git a/packages/runtime/src/scr_dyn_invoke.c b/packages/runtime/src/scr_dyn_invoke.c index 4039fda94..727e3e835 100644 --- a/packages/runtime/src/scr_dyn_invoke.c +++ b/packages/runtime/src/scr_dyn_invoke.c @@ -740,6 +740,8 @@ static ScrDyn *dyn_live_array_method(ScrDyn *recv, int mid, ScrDyn *const *args, if (!ops) return NULL; ScrArr *arr = recv->v.typed_ref.ptr; size_t len = arr->len; + /* A frozen array's view refuses the write. */ + if (scr_arr_frozen(arr) && ((mid == DYN_M_push) || (mid == DYN_M_pop))) return NULL; if ((mid == DYN_M_push) && argc == 1 && ops->set) { if (!ops->set(arr, (double)len, args[0])) return NULL; *handled = true; @@ -1098,16 +1100,20 @@ static ScrDyn *scr_dyn_invoke_impl( return NULL; } } - if (recv->v.arr.frozen && ((mid == DYN_M_push) || - (mid == DYN_M_pop) || (mid == DYN_M_shift) || - (mid == DYN_M_unshift) || (mid == DYN_M_splice))) { - static const char message[] = "Cannot assign to read only property 'length'"; - scr_throw_error_msg(SCR_ERR_TYPE, message, sizeof message - 1); + /* A frozen array throws at the operation's first write, as in Node. */ + int frozen_op = !recv->v.arr.frozen ? -1 + : (mid == DYN_M_push) ? 1 + : (mid == DYN_M_pop) ? 2 + : (mid == DYN_M_shift) ? 6 + : (mid == DYN_M_unshift) ? 4 + : (mid == DYN_M_splice) ? 10 + : ((mid == DYN_M_sort) || (mid == DYN_M_reverse)) ? 7 + : -1; + double items = (mid == DYN_M_splice) ? (argc > 2 ? (double)(argc - 2) : 0) : (double)argc; + if (frozen_op >= 0 && + scr_array_frozen_error(frozen_op, len, (double)splice_start, frozen_op == 10 ? (double)splice_removed : items, items)) return NULL; - } - if ((grows && recv->non_extensible) || (shrinks && recv->v.arr.sealed && !(mid == DYN_M_shift)) || - (recv->v.arr.frozen && (((mid == DYN_M_sort) && len > 1) || - ((mid == DYN_M_reverse) && len > 1)))) { + if ((grows && recv->non_extensible) || (shrinks && recv->v.arr.sealed && !(mid == DYN_M_shift))) { static const char message[] = "Cannot modify a sealed or frozen array"; scr_throw_error_msg(SCR_ERR_TYPE, message, sizeof message - 1); return NULL; @@ -1376,11 +1382,7 @@ static ScrDyn *scr_dyn_invoke_impl( if (!dyn_arr_length_unchanged(recv, len)) return NULL; size_t start = dyn_rel_index(startD, len); size_t end = dyn_rel_index(endD, len); - if (recv->v.arr.frozen && start < end) { - static const char message[] = "Cannot assign to read only array index"; - scr_throw_error_msg(SCR_ERR_TYPE, message, sizeof message - 1); - return NULL; - } + if (recv->v.arr.frozen && scr_array_frozen_error(8, len, (double)start, (double)end, 0)) return NULL; ScrDyn *fill = argc ? args[0] : scr_dyn_undefined(); if (start < end) scr_dyn_region_store(recv, fill); for (size_t i = start; i < end; i++) { @@ -1409,11 +1411,8 @@ static ScrDyn *scr_dyn_invoke_impl( size_t end = dyn_rel_index(endD, len); size_t count = end > start ? end - start : 0; if (count > len - target) count = len - target; - if (recv->v.arr.frozen && count > 0) { - static const char message[] = "Cannot assign to read only array index"; - scr_throw_error_msg(SCR_ERR_TYPE, message, sizeof message - 1); + if (recv->v.arr.frozen && scr_array_frozen_error(9, len, (double)target, (double)start, (double)end)) return NULL; - } if (start < target && target < start + count) { for (size_t i = count; i > 0; i--) { ScrDyn *old = recv->v.arr.items[target + i - 1]; diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index a542b5c24..13bd4d066 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -2302,8 +2302,32 @@ static void *scr_dyn_typed_record(const ScrDyn *d) { return d->v.typed_ref.ptr; } -/* Freezes the checked view of a frozen record: its fields, and every - * property added through the view. */ +/* The native array an array capsule refers to, or NULL for any other + * value. */ +static ScrArr *scr_dyn_typed_array(const ScrDyn *d) { + if (d->kind != SCR_DYN_TYPED_REF || d->v.typed_ref.type_key_len < 6 || + memcmp(d->v.typed_ref.type_key, "array<", 6) != 0) + return NULL; + return d->v.typed_ref.ptr; +} + +/* The native Map or Set a handle holds, or NULL for any other value. */ +static ScrMap *scr_dyn_native_collection(const ScrDyn *d) { + if (d->kind != SCR_DYN_HANDLE || (d->v.handle.tag != SCR_DYNH_MAP && d->v.handle.tag != SCR_DYNH_SET)) + return NULL; + return d->v.handle.ptr; +} + +/* Whether a capsule's native record or array is frozen. */ +static bool scr_dyn_typed_ref_frozen(const ScrDyn *d) { + void *record = scr_dyn_typed_record(d); + if (record) return scr_rc_frozen(record); + ScrArr *array = scr_dyn_typed_array(d); + return array && scr_arr_frozen(array); +} + +/* Freezes the checked view of a frozen record or array: its fields or + * elements, and every property added through the view. */ static void scr_dyn_record_view_freeze(ScrDyn *view) { view->non_extensible = true; if (view->kind == SCR_DYN_OBJ) { @@ -2350,9 +2374,9 @@ static bool scr_dyn_class_integrity_fence(const ScrDyn *object, const char *oper ScrJsonBuf buffer; scr_jb_init(&buffer); scr_jb_puts(&buffer, operation); - scr_jb_puts(&buffer, scr_dyn_typed_record(object) - ? " of an object held in an untyped value is not supported yet" - : " of a class instance held in an untyped value is not supported yet"); + scr_jb_puts(&buffer, scr_dyn_typed_record(object) ? " of an object held in an untyped value is not supported yet" + : scr_dyn_typed_array(object) ? " of an array held in an untyped value is not supported yet" + : " of a class instance held in an untyped value is not supported yet"); ScrStr *message = scr_jb_finish(&buffer); scr_throw_error_msg_code(SCR_ERR_ERROR, message->data, message->len, "SC2020"); scr_str_release(message); @@ -2400,6 +2424,11 @@ static ScrDyn *scr_dyn_fn_existing_table(const ScrDyn *function) { ScrDyn *scr_dyn_prevent_extensions(ScrDyn *object) { if (scr_dyn_class_reflection_fence(object) || scr_dyn_error_prototype_fence(object)) return NULL; if (object->kind == SCR_DYN_PROXY) { scr_dyn_proxy_unsupported("Object.preventExtensions"); return NULL; } + ScrMap *collection = scr_dyn_native_collection(object); + if (collection) { + scr_map_prevent_extensions(collection); + return scr_dyn_retain(object); + } if (scr_dyn_class_integrity_fence(object, "Object.preventExtensions")) return NULL; ScrDyn *table = scr_dyn_fn_integrity_table(object); if (table) { @@ -2419,6 +2448,8 @@ bool scr_dyn_is_extensible(ScrDyn *object) { if (object->kind == SCR_DYN_PROXY) { scr_dyn_proxy_unsupported("Object.isExtensible"); return false; } bool answer; if (scr_dyn_typed_ref_view_answer(object, scr_dyn_is_extensible_view, &answer)) return answer; + ScrMap *collection = scr_dyn_native_collection(object); + if (collection) return scr_map_is_extensible(collection); if (scr_dyn_plain_fn(object)) { ScrDyn *table = scr_dyn_fn_existing_table(object); answer = !table || !table->non_extensible; @@ -2431,6 +2462,11 @@ bool scr_dyn_is_extensible(ScrDyn *object) { ScrDyn *scr_dyn_seal(ScrDyn *object) { if (scr_dyn_class_reflection_fence(object) || scr_dyn_error_prototype_fence(object)) return NULL; if (object->kind == SCR_DYN_PROXY) { scr_dyn_proxy_unsupported("Object.seal"); return NULL; } + ScrMap *collection = scr_dyn_native_collection(object); + if (collection) { + scr_map_prevent_extensions(collection); + return scr_dyn_retain(object); + } if (scr_dyn_class_integrity_fence(object, "Object.seal")) return NULL; if (object->symbol_properties) scr_dyn_release(scr_dyn_seal(object->symbol_properties)); ScrDyn *table = scr_dyn_fn_integrity_table(object); @@ -2455,6 +2491,8 @@ bool scr_dyn_is_sealed(const ScrDyn *object) { if (object->kind == SCR_DYN_PROXY) { scr_dyn_proxy_unsupported("Object.isSealed"); return false; } bool answer; if (scr_dyn_typed_ref_view_answer((ScrDyn *)object, scr_dyn_is_sealed, &answer)) return answer; + ScrMap *collection = scr_dyn_native_collection(object); + if (collection) return !scr_map_is_extensible(collection); if (object->symbol_properties && !scr_dyn_is_sealed(object->symbol_properties)) return false; if (scr_dyn_plain_fn(object)) { ScrDyn *table = scr_dyn_fn_existing_table(object); @@ -3679,16 +3717,14 @@ ScrDyn *scr_dyn_typed_ref_materialize(const ScrDyn *d) { fresh->rc = fresh_rc; scr_dyn_release(fresh); } - void *record = scr_dyn_typed_record(capsule); - if (record && scr_rc_frozen(record)) scr_dyn_record_view_freeze(capsule->v.typed_ref.materialized); + if (scr_dyn_typed_ref_frozen(capsule)) scr_dyn_record_view_freeze(capsule->v.typed_ref.materialized); return scr_dyn_retain(capsule->v.typed_ref.materialized); } void scr_dyn_typed_ref_commit(ScrDyn *d) { - /* A frozen record's view cannot have changed its fields. */ + /* A frozen record's or array's view cannot have changed its contents. */ if (d && d->kind == SCR_DYN_TYPED_REF && d->v.typed_ref.commit && - d->v.typed_ref.materialized && - !(scr_dyn_typed_record(d) && scr_rc_frozen(d->v.typed_ref.ptr))) { + d->v.typed_ref.materialized && !scr_dyn_typed_ref_frozen(d)) { d->v.typed_ref.commit(d->v.typed_ref.ptr, d->v.typed_ref.materialized); } @@ -3927,6 +3963,8 @@ static bool scr_typed_ref_array_has_own(const ScrDyn *d, const ScrStr *key) { static bool scr_typed_ref_array_set(ScrDyn *d, const ScrStr *key, const ScrDyn *value) { const ScrDynTypedArrayOps *ops = scr_dyn_typed_ref_array_ops(d); if (!ops || !ops->set) return false; + /* A frozen array's view refuses the write. */ + if (scr_arr_frozen(d->v.typed_ref.ptr)) return false; /* Numeric properties live in the native array's property table, which * holds values of the element type. */ double number; @@ -9230,6 +9268,26 @@ bool scr_dyn_in(const ScrDyn *v, const ScrStr *key) { return false; } +/* Node's TypeErrors for assigning a read-only property and deleting a + * non-configurable one of a checked object: the object reads as + * #, or [object Object] without a prototype. */ +static void scr_dyn_throw_property(const ScrDyn *object, const char *key, size_t key_len, bool deleting) { + const char *name = scr_dyn_class_name(object); + ScrJsonBuf buffer; + scr_jb_init(&buffer); + scr_jb_puts(&buffer, deleting ? "Cannot delete property '" : "Cannot assign to read only property '"); + scr_jb_write(&buffer, key, key_len); + scr_jb_puts(&buffer, deleting ? "' of " : "' of object '"); + if (object->null_proto) scr_jb_puts(&buffer, "[object Object]"); + else { + scr_jb_puts(&buffer, "#<"); + scr_jb_puts(&buffer, name ? name : "Object"); + scr_jb_puts(&buffer, ">"); + } + if (!deleting) scr_jb_puts(&buffer, "'"); + scr_throw_error(SCR_ERR_TYPE, scr_jb_finish(&buffer)); +} + static void scr_dyn_object_key_set(ScrDyn *recv, ScrStr *key, ScrDyn *value, const ScrDyn *receiver) { for (size_t i = 0; i < recv->v.obj.len; i++) { ScrDynEntry *entry = &recv->v.obj.entries[i]; @@ -9245,8 +9303,7 @@ static void scr_dyn_object_key_set(ScrDyn *recv, ScrStr *key, ScrDyn *value, con return; } if (entry->accessor || !entry->writable) { - static const char msg[] = "Cannot assign to read only property"; - scr_throw_error_msg(SCR_ERR_TYPE, msg, sizeof msg - 1); + scr_dyn_throw_property(receiver, key->data, key->len, false); return; } } @@ -9273,8 +9330,7 @@ static void scr_dyn_object_key_set(ScrDyn *recv, ScrStr *key, ScrDyn *value, con return; } if (inherited->accessor || !inherited->writable) { - static const char message[] = "Cannot assign to read only property"; - scr_throw_error_msg(SCR_ERR_TYPE, message, sizeof message - 1); + scr_dyn_throw_property(receiver, key->data, key->len, false); scr_dyn_release(table); return; } @@ -9354,8 +9410,7 @@ void scr_dyn_key_set(ScrDyn *recv, ScrStr *key, ScrDyn *value) { if (recv->kind == SCR_DYN_ARR) { if (key->len == 6 && memcmp(key->data, "length", 6) == 0) { if (recv->v.arr.frozen) { - static const char message[] = "Cannot assign to read only property"; - scr_throw_error_msg(SCR_ERR_TYPE, message, sizeof message - 1); + scr_array_frozen_error(3, recv->v.arr.len, 0, 0, 0); return; } double number; @@ -9374,8 +9429,7 @@ void scr_dyn_key_set(ScrDyn *recv, ScrStr *key, ScrDyn *value) { while (recv->v.arr.len < length) scr_dyn_arr_push_hole(recv); while (recv->v.arr.len > length) { if (recv->v.arr.sealed && scr_dyn_arr_has_index(recv, recv->v.arr.len - 1)) { - static const char message[] = "Cannot delete non-configurable property"; - scr_throw_error_msg(SCR_ERR_TYPE, message, sizeof message - 1); + scr_array_frozen_error(5, recv->v.arr.len, (double)(recv->v.arr.len - 1), 0, 0); return; } scr_dyn_release(recv->v.arr.items[--recv->v.arr.len]); @@ -9393,14 +9447,13 @@ void scr_dyn_key_set(ScrDyn *recv, ScrStr *key, ScrDyn *value) { } if (is_index && idx < 4294967295ULL) { if (scr_dyn_array_prototype_element_fence(recv)) return; - if (recv->v.arr.frozen && idx < recv->v.arr.len) { - static const char message[] = "Cannot assign to read only property"; - scr_throw_error_msg(SCR_ERR_TYPE, message, sizeof message - 1); + if (recv->v.arr.frozen && idx < recv->v.arr.len && scr_dyn_arr_has_index(recv, idx)) { + scr_array_frozen_error(0, recv->v.arr.len, (double)idx, 0, 0); return; } if (recv->non_extensible && !scr_dyn_arr_has_index(recv, idx)) { - static const char message[] = "Cannot add property, object is not extensible"; - scr_throw_error_msg(SCR_ERR_TYPE, message, sizeof message - 1); + /* An absent index, a hole included, is a property to add. */ + scr_array_frozen_error(0, 0, (double)idx, 0, 0); return; } if (idx > recv->v.arr.len + 1000000) { @@ -11229,10 +11282,7 @@ void scr_dyn_key_delete(ScrDyn *recv, const ScrStr *key, bool strict) { if (recv->kind == SCR_DYN_OBJ) { ScrDynEntry *entry = scr_dyn_entry(recv, key); if (entry && !entry->configurable) { - if (strict) { - static const char message[] = "Cannot delete non-configurable property"; - scr_throw_error_msg(SCR_ERR_TYPE, message, sizeof message - 1); - } + if (strict) scr_dyn_throw_property(recv, key->data, key->len, true); return; } scr_json_delete_member(recv, key); @@ -11244,10 +11294,7 @@ void scr_dyn_key_delete(ScrDyn *recv, const ScrStr *key, bool strict) { for (size_t i = 0; i < key->len; i++) index = index * 10 + (size_t)(key->data[i] - '0'); if (!scr_dyn_arr_has_index(recv, index)) return; if (recv->v.arr.sealed) { - if (strict) { - static const char message[] = "Cannot delete non-configurable property"; - scr_throw_error_msg(SCR_ERR_TYPE, message, sizeof message - 1); - } + if (strict) scr_array_frozen_error(5, recv->v.arr.len, (double)index, 0, 0); return; } if (!recv->v.arr.presence) { @@ -15682,18 +15729,29 @@ ScrStr *scr_dyn_to_string_method(const ScrDyn *d, const ScrStr *enc, const ScrSt return scr_dyn_to_string_method_args(d, enc, what, NULL, 0); } -/* Freezing a record capsule freezes the native record, whose emitted writes - * then refuse to change it, and its checked view. */ -static ScrDyn *scr_dyn_freeze_record(ScrDyn *capsule, void *record) { - scr_record_freeze(record); - if (capsule->v.typed_ref.materialized) scr_dyn_record_view_freeze(capsule->v.typed_ref.materialized); - return scr_dyn_retain(capsule); +/* Freezing a record or array capsule freezes the native value, whose + * emitted writes then refuse to change it, and its checked view. A Map or + * Set handle only stops being extensible: its entries stay writable. Answers + * NULL for every other value. */ +static ScrDyn *scr_dyn_freeze_native(ScrDyn *value) { + void *record = scr_dyn_typed_record(value); + ScrArr *array = record ? NULL : scr_dyn_typed_array(value); + ScrMap *collection = scr_dyn_native_collection(value); + if (collection) { + scr_map_prevent_extensions(collection); + return scr_dyn_retain(value); + } + if (!record && !array) return NULL; + if (record) scr_record_freeze(record); + else scr_arr_freeze(array); + if (value->v.typed_ref.materialized) scr_dyn_record_view_freeze(value->v.typed_ref.materialized); + return scr_dyn_retain(value); } ScrDyn *scr_dyn_freeze(ScrDyn *value) { if (scr_dyn_class_reflection_fence(value) || scr_dyn_error_prototype_fence(value)) return NULL; - void *record = scr_dyn_typed_record(value); - if (record) return scr_dyn_freeze_record(value, record); + ScrDyn *native = scr_dyn_freeze_native(value); + if (native) return native; if (scr_dyn_class_integrity_fence(value, "Object.freeze")) return NULL; ScrDyn *table = scr_dyn_fn_integrity_table(value); if (!table) return scr_dyn_freeze_data(value); @@ -15705,8 +15763,8 @@ ScrDyn *scr_dyn_freeze(ScrDyn *value) { ScrDyn *scr_dyn_freeze_data(ScrDyn *value) { if (scr_dyn_class_reflection_fence(value) || scr_dyn_error_prototype_fence(value)) return NULL; - void *record = scr_dyn_typed_record(value); - if (record) return scr_dyn_freeze_record(value, record); + ScrDyn *native = scr_dyn_freeze_native(value); + if (native) return native; if (scr_dyn_class_integrity_fence(value, "Object.freeze")) return NULL; if (value->symbol_properties) scr_dyn_release(scr_dyn_freeze_data(value->symbol_properties)); if (value->kind == SCR_DYN_ARR) { @@ -15737,6 +15795,8 @@ bool scr_dyn_is_frozen(const ScrDyn *value) { if (scr_dyn_class_reflection_fence(value)) return false; bool answer; if (scr_dyn_typed_ref_view_answer((ScrDyn *)value, scr_dyn_is_frozen, &answer)) return answer; + ScrMap *collection = scr_dyn_native_collection(value); + if (collection) return !scr_map_is_extensible(collection); if (value->symbol_properties && !scr_dyn_is_frozen(value->symbol_properties)) return false; if (scr_dyn_plain_fn(value)) { ScrDyn *table = scr_dyn_fn_existing_table(value); diff --git a/packages/runtime/src/scr_map.c b/packages/runtime/src/scr_map.c index 1a6a3fc00..1cb77b0f6 100644 --- a/packages/runtime/src/scr_map.c +++ b/packages/runtime/src/scr_map.c @@ -988,6 +988,12 @@ double scr_map_size(const ScrMap *m) { return (double)m->nlive; } * table. Regrowth is amortized against the insertions that need it. * A map without entries has empty tables already: clearing it does no * work and keeps its storage for the next fill. */ +void scr_map_prevent_extensions(ScrMap *m) { + if (m->rc != SIZE_MAX) m->flags |= SCR_MAP_NOT_EXTENSIBLE; +} + +bool scr_map_is_extensible(const ScrMap *m) { return (m->flags & SCR_MAP_NOT_EXTENSIBLE) == 0; } + void scr_map_clear(ScrMap *m) { size_t walked = m->nentries; if (walked == 0) return; diff --git a/packages/runtime/src/scr_map.h b/packages/runtime/src/scr_map.h index d1ca387bf..a581cc1c7 100644 --- a/packages/runtime/src/scr_map.h +++ b/packages/runtime/src/scr_map.h @@ -116,6 +116,10 @@ typedef struct ScrMapOps { * whose object has a null prototype (Object.groupBy results). Compiled * code sets and tests the byte; the runtime never copies it. */ #define SCR_MAP_NULL_PROTO 1u +/* Object.freeze, Object.seal or Object.preventExtensions of a Map or Set: + * it has no own properties, so it is then frozen, sealed and not + * extensible at once. Its entries stay writable, as in Node. */ +#define SCR_MAP_NOT_EXTENSIBLE 2u typedef struct ScrMap { size_t rc; /* SIZE_MAX = immortal (unused for maps; kept per convention) */ @@ -272,6 +276,10 @@ void scr_map_release_v(void *m); void scr_map_trace_v(void *m, ScrTraceVisit visit, void *ctx); double scr_map_size(const ScrMap *m); /* live entries (Map.size) */ +/* Object.freeze, seal and preventExtensions of a Map or Set + * (SCR_MAP_NOT_EXTENSIBLE); immortal maps are left as they are. */ +void scr_map_prevent_extensions(ScrMap *m); +bool scr_map_is_extensible(const ScrMap *m); void scr_map_clear(ScrMap *m); bool scr_map_has_f64(const ScrMap *m, double key); diff --git a/packages/runtime/src/scr_publish.c b/packages/runtime/src/scr_publish.c index 4767122dd..afbe4d6af 100644 --- a/packages/runtime/src/scr_publish.c +++ b/packages/runtime/src/scr_publish.c @@ -527,62 +527,12 @@ void scr_throw_published_field(const char *prop, const char *owner) { scr_pub_throw("Cannot assign to read only property '%s' of object '#<%s>'", prop, owner); } -static void scr_pub_index_text(double index, char *out, size_t size) { - char buf[64]; - size_t n = scr_f64_to_str(index, buf); - if (n > size - 1) n = size - 1; - memcpy(out, buf, n); - out[n] = 0; +bool scr_throw_published_array(int op, const ScrArr *a, double index, double n) { + return scr_array_frozen_error(op, a->len, index, n, 0); } -/* op: 0 index write, 1 push (n = argument count), 2 pop, 3 length write, - * 4 unshift (n = argument count), 5 delete, 6 shift, 7 other in-place - * rewrites (splice, reverse, fill, copyWithin, sort). Returns false when - * Node's operation would not write the frozen array at all (it then is not - * an error: e.g. reversing an empty array). */ -bool scr_throw_published_array(int op, const ScrArr *a, double index, double n) { - static const char length[] = "Cannot assign to read only property 'length' of object '[object Array]'"; - char text[64]; - size_t len = a->len; - switch (op) { - case 0: - scr_pub_index_text(index, text, sizeof text); - if (index >= 0 && index < (double)len && index == (double)(size_t)index) - scr_pub_throw("Cannot assign to read only property '%s' of object '[object Array]'%s", text, ""); - else scr_pub_throw("Cannot add property %s, object is not extensible%s", text, ""); - return true; - case 1: - if (n == 0) break; - scr_pub_index_text((double)len, text, sizeof text); - scr_pub_throw("Cannot add property %s, object is not extensible%s", text, ""); - return true; - case 2: - case 6: - if (len == 0) break; - if (op == 6 && len > 1) { - scr_pub_throw("Cannot assign to read only property '0' of object '[object Array]'%s%s", "", ""); - return true; - } - scr_pub_index_text((double)(len - 1), text, sizeof text); - scr_pub_throw("Cannot delete property '%s' of [object Array]%s", text, ""); - return true; - case 3: break; - case 4: - if (n == 0) break; - scr_pub_index_text(len ? (double)len + n - 1 : 0, text, sizeof text); - scr_pub_throw("Cannot add property %s, object is not extensible%s", text, ""); - return true; - case 5: - scr_pub_index_text(index, text, sizeof text); - scr_pub_throw("Cannot delete property '%s' of [object Array]%s", text, ""); - return true; - default: - if (len == 0) return false; - scr_pub_throw("Cannot assign to read only property '0' of object '[object Array]'%s%s", "", ""); - return true; - } - scr_throw_error_msg(SCR_ERR_TYPE, length, sizeof length - 1); - return true; +bool scr_throw_published_array_range(int op, const ScrArr *a, double x, double y, double z) { + return scr_array_frozen_error(op, a->len, x, y, z); } void scr_throw_published_collection(bool set) { diff --git a/packages/runtime/src/scr_rc.h b/packages/runtime/src/scr_rc.h index 87eb2536a..ccb74514d 100644 --- a/packages/runtime/src/scr_rc.h +++ b/packages/runtime/src/scr_rc.h @@ -743,12 +743,14 @@ void scr_pub_dyn_fn(void *v, ScrPub *ctx); void scr_pub_arr(void *v, ScrPub *ctx, ScrPubFn elem); void scr_pub_map(void *v, ScrPub *ctx, ScrPubFn key, ScrPubFn val); void scr_publish_refuse(const char *message); -/* Writes into published objects (emitted guards): Node's frozen-object - * TypeErrors. scr_throw_published_array answers false (and throws nothing) - * when Node's operation would not write the array. */ +/* Writes into published or frozen objects (emitted guards): Node's + * frozen-object TypeErrors. The array guards answer false (and throw + * nothing) when Node's operation would not write the array; their ops are + * scr_array_frozen_error's (scr_array.h). */ struct ScrArr; SCR_COLD void scr_throw_published_field(const char *prop, const char *owner); bool scr_throw_published_array(int op, const struct ScrArr *a, double index, double n); +bool scr_throw_published_array_range(int op, const struct ScrArr *a, double x, double y, double z); SCR_COLD void scr_throw_published_collection(bool set); /* Young-in nursery (scr_cycle.c built with SCR_YIN, linked by programs diff --git a/tests/corpus/7406-frozen-containers.ts b/tests/corpus/7406-frozen-containers.ts new file mode 100644 index 000000000..450cbf8cb --- /dev/null +++ b/tests/corpus/7406-frozen-containers.ts @@ -0,0 +1,132 @@ +// Object.freeze of arrays, Maps and Sets that other references still hold. +// A frozen array refuses every write through any reference with Node's +// TypeError for the operation's first write (this module is strict mode +// code). A frozen Map or Set only stops being extensible: its entries stay +// writable, and Object.isFrozen, isSealed and isExtensible report it. +export {}; + +function report(e: unknown): void { + console.log((e as Error).name + ": " + (e as Error).message); +} + +function freezeUnknown(value: unknown): unknown { + return Object.freeze(value); +} + +function writeAny(value: any): void { + value[0] = 7; +} + +function pushAny(value: any): void { + value.push(8); +} + +function integrity(value: unknown): string { + return [Object.isFrozen(value), Object.isSealed(value), Object.isExtensible(value)].join(" "); +} + +function arrays(): void { + const a: number[] = [1, 2, 3]; + const alias = a; + console.log(Object.isFrozen(a), integrity(a)); + Object.freeze(alias); + console.log(Object.isFrozen(a), integrity(a)); + try { a[0] = 9; } catch (e) { report(e); } + try { a[5] = 9; } catch (e) { report(e); } + try { a.length = 0; } catch (e) { report(e); } + try { a.push(4); } catch (e) { report(e); } + try { a.pop(); } catch (e) { report(e); } + try { a.shift(); } catch (e) { report(e); } + try { a.unshift(0); } catch (e) { report(e); } + try { a.splice(0, 1); } catch (e) { report(e); } + try { a.splice(1, 0, 7); } catch (e) { report(e); } + try { a.reverse(); } catch (e) { report(e); } + try { a.sort(); } catch (e) { report(e); } + try { a.fill(0, 1); } catch (e) { report(e); } + try { a.copyWithin(1, 0); } catch (e) { report(e); } + console.log(a.join(), a.length); + + const one: string[] = ["x"]; + Object.freeze(one); + console.log(one.reverse().join(), one.sort().join(), one.copyWithin(0, 1).join()); + try { one.splice(0, 1); } catch (e) { report(e); } + try { one.shift(); } catch (e) { report(e); } + const none: string[] = []; + Object.freeze(none); + console.log(none.reverse().length, none.fill("y").length); + try { none.pop(); } catch (e) { report(e); } + try { none.splice(0, 0, "z"); } catch (e) { report(e); } + + let calls = 0; + const byValue = (x: number, y: number): number => { + calls++; + return x - y; + }; + const unsorted: number[] = [3, 1, 2]; + Object.freeze(unsorted); + try { unsorted.sort(byValue); } catch (e) { report(e); } + const single: number[] = [5]; + Object.freeze(single); + console.log(unsorted.join(), calls > 0, single.sort(byValue).join()); + + const open: number[] = [4, 5]; + open.push(6); + open[0] = 1; + console.log(open.join(), Object.isFrozen(open)); +} + +function throughUnknown(): void { + const a: number[] = [1, 2]; + console.log(integrity(a)); + const back = freezeUnknown(a); + console.log(back === a, Object.isFrozen(a), integrity(a)); + try { a.push(3); } catch (e) { report(e); } + try { a[1] = 5; } catch (e) { report(e); } + try { writeAny(a); } catch (e) { report(e); } + try { pushAny(a); } catch (e) { report(e); } + console.log(a.join()); + + const rows: number[][] = [[1], [2]]; + freezeUnknown(rows[1]); + rows[0]!.push(10); + try { rows[1]!.push(20); } catch (e) { report(e); } + console.log(JSON.stringify(rows), Object.isFrozen(rows), Object.isFrozen(rows[1])); +} + +function collections(): void { + const m = new Map([["x", 1]]); + console.log(integrity(m)); + const frozen = Object.freeze(m); + m.set("y", 2); + m.delete("x"); + console.log(frozen === m, Object.isFrozen(m), Object.isSealed(m), Object.isExtensible(m), m.size); + const s = new Set([1]); + freezeUnknown(s); + s.add(2); + console.log(integrity(s), s.size); + const sealed = new Set(); + Object.seal(sealed); + const fresh = new Map(); + Object.preventExtensions(fresh); + console.log(Object.isFrozen(sealed), Object.isExtensible(fresh), integrity(new Map())); +} + +function lifetimes(): void { + const kept: string[][] = []; + for (let round = 0; round < 3; round++) { + const row = ["r" + round, "s" + round]; + if (round !== 1) Object.freeze(row); + kept.push(row); + } + for (const row of kept) { + try { row[0] = "changed"; } catch (e) { console.log(row[1], (e as Error).message); } + } + console.log(JSON.stringify(kept)); + kept.length = 0; + console.log(kept.length); +} + +arrays(); +throughUnknown(); +collections(); +lifetimes(); diff --git a/tests/harness/corpus-costs/7406-frozen-containers.ts.json b/tests/harness/corpus-costs/7406-frozen-containers.ts.json new file mode 100644 index 000000000..30bfa4b73 --- /dev/null +++ b/tests/harness/corpus-costs/7406-frozen-containers.ts.json @@ -0,0 +1 @@ +{ "plain": 3146, "san": 3439 } From 967f2ec85dcd408bea5d24e2e425aafd3ea248ac Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 05:55:39 -0500 Subject: [PATCH 14/53] Compile minified npm entries and untyped regex arguments Refs #1777 npm packages are no longer refused up front for minified JavaScript. A bundle whose interop helpers lost their names is still refused by the bundle recognizers, and a package whose attempt fails is still refused on its own. A `var` read in a later declarator of the statement that stores it (`var a = {...}, b = {...a}`) runs after the store. A read in a function body is now located at the body's own statement instead of the body, so JavaScript `var` bindings read after their store in a function keep their type instead of gaining undefined. `replace` and `replaceAll` with an untyped search value and a callback call the regex callback replacement for a RegExp and the string-search helper otherwise. That helper now accepts untyped callback parameters, passes undefined to parameters after the subject, and converts number, boolean and untyped results with ToString. In JavaScript, `||` and `&&` over primitives of different types whose result tsc types `any` evaluate in the checked domain. --- docs/content/docs/dependencies.mdx | 2 +- packages/compiler/src/cli/usage.ts | 6 +- .../lowering/containers/string-replacement.ts | 64 +++++++++---- .../frontend/lowering/expressions/binary.ts | 18 ++++ .../src/frontend/lowering/var-hoisting.ts | 43 ++++++--- packages/compiler/src/frontend/npm-static.ts | 26 ++---- .../7543-var-list-declarator-order.js.json | 6 ++ ...545-string-regex-checked-arguments.js.json | 6 ++ .../7546-minified-esm-rules/main.ts.json | 6 ++ .../corpus/7543-var-list-declarator-order.js | 66 ++++++++++++++ .../7545-string-regex-checked-arguments.js | 89 +++++++++++++++++++ .../npm/cases/7546-minified-esm-rules/main.ts | 22 +++++ .../node_modules/gtminimark/dist/index.d.ts | 1 + .../npm/node_modules/gtminimark/dist/index.js | 1 + .../npm/node_modules/gtminimark/package.json | 8 ++ .../7543-var-list-declarator-order.js.json | 1 + ...545-string-regex-checked-arguments.js.json | 1 + tests/harness/library-mode.test.ts | 26 +++--- tests/harness/npm-static.test.ts | 9 ++ tests/library-mode/npm-refuse/lib-minified.ts | 4 +- tests/library-mode/npm-static/lib.ts | 2 +- .../npm-static/node_modules/mathkit/index.js | 2 +- 22 files changed, 340 insertions(+), 69 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7543-var-list-declarator-order.js.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7545-string-regex-checked-arguments.js.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/7546-minified-esm-rules/main.ts.json create mode 100644 tests/corpus/7543-var-list-declarator-order.js create mode 100644 tests/corpus/7545-string-regex-checked-arguments.js create mode 100644 tests/fixtures/npm/cases/7546-minified-esm-rules/main.ts create mode 100644 tests/fixtures/npm/node_modules/gtminimark/dist/index.d.ts create mode 100644 tests/fixtures/npm/node_modules/gtminimark/dist/index.js create mode 100644 tests/fixtures/npm/node_modules/gtminimark/package.json create mode 100644 tests/harness/corpus-costs/7543-var-list-declarator-order.js.json create mode 100644 tests/harness/corpus-costs/7545-string-regex-checked-arguments.js.json diff --git a/docs/content/docs/dependencies.mdx b/docs/content/docs/dependencies.mdx index 14130b234..f02dcfbf0 100644 --- a/docs/content/docs/dependencies.mdx +++ b/docs/content/docs/dependencies.mdx @@ -53,7 +53,7 @@ Type-only imports follow Node.js's type stripping. `import type` and `export typ scriptc compiles a package automatically when it: - ships its own type declarations, or is typed by its DefinitelyTyped `@types/` package, or is a dependency of another package that has no declarations at all, and -- ships readable JavaScript: unminified, or minified by a bundler whose helpers keep their names. +- ships JavaScript that isn't a bundle whose bundler-interop helpers, such as `__commonJS`, have lost their names. Minified JavaScript compiles like unminified JavaScript. The declarations check your program; the package's JavaScript compiles with the types scriptc infers from it. A program that doesn't typecheck against the inferred types, for example because an `@types/` package describes another release, fails the build at that use. diff --git a/packages/compiler/src/cli/usage.ts b/packages/compiler/src/cli/usage.ts index 23e57c66f..c4d07656a 100644 --- a/packages/compiler/src/cli/usage.ts +++ b/packages/compiler/src/cli/usage.ts @@ -77,9 +77,9 @@ Options: --npm-static compile the named npm packages' shipped JS statically as program modules (repeatable; "auto" opts in every eligible - direct import: own .d.ts, unminified JS, no build-transform - markers; "pkg..." also names every package pkg depends - on). Packages that fail preflight are refused with a + direct import: own .d.ts, no build-transform markers; + "pkg..." also names every package pkg depends on). + Packages that fail preflight are refused with a diagnostic. Eligible dependencies compile automatically. --provenance-sources EXPERIMENTAL: compile npm dependencies from their diff --git a/packages/compiler/src/frontend/lowering/containers/string-replacement.ts b/packages/compiler/src/frontend/lowering/containers/string-replacement.ts index 780c77d0f..156f74869 100644 --- a/packages/compiler/src/frontend/lowering/containers/string-replacement.ts +++ b/packages/compiler/src/frontend/lowering/containers/string-replacement.ts @@ -2,6 +2,7 @@ import { keyedHelperName } from "../stable-names.js"; import * as ts from "../../ts7/adapter.js"; import { BOOL, + DYN, F64, REGEX, STRING, @@ -14,7 +15,7 @@ import { type IrType, type SrcLoc, } from "../../../ir/ir.js"; -import { numLit, strLit, varRef } from "../../../ir/build.js"; +import { dynUndefinedExpr, numLit, strLit, varRef } from "../../../ir/build.js"; import { InternalCompilerError } from "../../../errors.js"; import { locOf } from "../../program.js"; import type { Lowerer } from "../lowerer.js"; @@ -65,14 +66,14 @@ export function lowerStringReplacement( const search = argument(args[0], false); const replacement = argument(args[1], true); if (search.type.kind === "dyn" || replacement.type.kind === "dyn") { - if (replacement.type.kind === "func") { - lowerer.noLowering(`string .${method} with checked search values and callbacks`, call); - } // Evaluate all inputs once before choosing the search protocol. Regexes // retain their native pattern; plain values use the string-search helper. + // A typed callback receives the regex match's arguments or the plain + // search's (match, offset, subject). const values = [receiver, search, replacement]; const locals = values.map((value) => lowerer.declareHiddenLocal("%replaceArg", value.type)); const refs = locals.map((local) => varRef(local.id, local.type, loc)); + const callback = replacement.type.kind === "func"; const checkedReplacement = replacement.type.kind === "dyn" ? lowerer.coerceInto(args[1] ?? call, refs[2]!, STRING) @@ -91,14 +92,27 @@ export function lowerStringReplacement( ? { kind: "ternary", cond: { kind: "libCall", fn: "dyn.nativeRegexIs", args: [refs[1]!], type: BOOL, loc }, - then: { - kind: "regexIntrinsic", - method, - receiver: refs[0]!, - args: [lowerer.coerceInto(args[0] ?? call, refs[1]!, REGEX), checkedReplacement], - type: STRING, - loc, - }, + then: callback + ? { + kind: "libCall", + fn: "regex.replaceCallback", + args: [ + refs[0]!, + lowerer.coerceInto(args[0] ?? call, refs[1]!, REGEX), + lowerer.coerceToExpected(refs[2]!, DYN), + { kind: "boolLit", value: method === "replaceAll", type: BOOL, loc }, + ], + type: STRING, + loc, + } + : { + kind: "regexIntrinsic", + method, + receiver: refs[0]!, + args: [lowerer.coerceInto(args[0] ?? call, refs[1]!, REGEX), checkedReplacement], + type: STRING, + loc, + }, else_: plain, type: STRING, loc, @@ -148,11 +162,17 @@ function lowerStringReplacementValues( if (callback.kind !== "func") { throw new InternalCompilerError(`string .${method} replacement of type ${callback.kind}`); } + // A checked parameter receives the boxed argument (undefined after the + // subject), and a checked, number or boolean result converts with + // ToString. const full = [STRING, F64, STRING]; + const ret = callback.ret.kind; if ( - callback.ret.kind !== "string" || - callback.params.length > full.length || - !callback.params.every((type, index) => typeEquals(type, full[index]!)) + (ret !== "string" && ret !== "dyn" && ret !== "f64" && ret !== "bool") || + !callback.params.every( + (type, index) => + type.kind === "dyn" || (index < full.length && typeEquals(type, full[index]!)), + ) ) { lowerer.noLowering( `string .${method} callback signature`, @@ -175,13 +195,21 @@ function lowerStringReplacementValues( const chunksType = arrayOf(STRING); const chunks = varRef("chunks.0", chunksType, loc); const builder = stringBuilder(loc, chunks); - const value: IrExpr = { + const result: IrExpr = { kind: "callValue", callee: replace, - args: [needle, match, subject].slice(0, callback.params.length), - type: STRING, + args: callback.params.map((type, index) => { + const arg = [needle, match, subject][index]; + if (arg === undefined) return dynUndefinedExpr(loc); + return type.kind === "dyn" ? lowerer.coerceToExpected(arg, DYN) : arg; + }), + type: callback.ret, loc, }; + const value: IrExpr = + ret === "dyn" + ? { kind: "libCall", fn: "dyn.toStringCoerce", args: [result], type: STRING, loc } + : lowerer.ensureString(result, call); const stop: IrStmt = { kind: "break", loc }; const body: IrStmt[] = [ builder.declare("chunks.0", { kind: "arrayLit", elems: [], type: chunksType, loc }), diff --git a/packages/compiler/src/frontend/lowering/expressions/binary.ts b/packages/compiler/src/frontend/lowering/expressions/binary.ts index 58f6e16b5..17aef7b85 100644 --- a/packages/compiler/src/frontend/lowering/expressions/binary.ts +++ b/packages/compiler/src/frontend/lowering/expressions/binary.ts @@ -2841,6 +2841,24 @@ function lowerLogicalPair(lowerer: Lowerer, expr: ts.BinaryExpression, left: IrE loc, }; } + // A JavaScript default that tsc types `any` (`e.title || null` in an + // instance of a function with untyped parameters, where `e.title` is a + // string): the checked domain, as for union operands above. + if ( + target === null && + isJsSourceFile(ts.getSourceFileOfNode(expr)) && + lowerer.dynConvertible(left.type) && + lowerer.dynConvertible(right.type) + ) { + return { + kind: "logical", + op: op === ts.SyntaxKind.AmpersandAmpersandToken ? "&&" : "||", + left: lowerer.coerceInto(expr.left, left, DYN), + right: lowerer.coerceInto(expr.right, right, DYN), + type: DYN, + loc, + }; + } lowerer.unsupported("SC1042", expr); } return { diff --git a/packages/compiler/src/frontend/lowering/var-hoisting.ts b/packages/compiler/src/frontend/lowering/var-hoisting.ts index 3f950f3ca..1a60e7fda 100644 --- a/packages/compiler/src/frontend/lowering/var-hoisting.ts +++ b/packages/compiler/src/frontend/lowering/var-hoisting.ts @@ -194,6 +194,15 @@ function referencesBySymbol( return out; } +/** Where a binding is first stored unconditionally: the scope statement's + * index, and for a declarator of a `var` statement or a `for` initializer, + * the list and the declarator's end. */ +interface Settle { + index: number; + list: ts.VariableDeclarationList | null; + end: number; +} + class ScopeAnalysis { private readonly js: boolean; /** Whether a function declaration's references all settle, by scope @@ -214,8 +223,10 @@ class ScopeAnalysis { } /** The index of the scope statement that first stores `symbol` - * unconditionally, or Infinity. */ - settleIndex(scope: Scope, symbol: ts.Symbol): number { + * unconditionally (Infinity when none does), and, when that statement + * declares it in a `var` statement or a `for` initializer, the end of the + * storing declarator: later declarators of the list run after it. */ + settleIndex(scope: Scope, symbol: ts.Symbol): Settle { const statements = statementsOf(scope); const checker = this.lowerer.checker; for (let i = 0; i < statements.length; i++) { @@ -234,7 +245,7 @@ class ScopeAnalysis { (d.initializer !== undefined || loop) && boundIdentifiersOf(d.name).some((n) => checker.getSymbolAtLocation(n) === symbol) ) - return i; + return { index: i, list: loop ? null : list, end: d.end }; } } else if ( ts.isExpressionStatement(s) && @@ -243,20 +254,21 @@ class ScopeAnalysis { ts.isIdentifier(s.expression.left) && checker.getSymbolAtLocation(s.expression.left) === symbol ) { - return i; + return { index: i, list: null, end: -1 }; } } - return Infinity; + return { index: Infinity, list: null, end: -1 }; } /** Whether `ref` (inside `scope`, or in another module) runs only after - * the statement at index `settle`. `read`: `ref` reads the binding - * itself, so tsc vouches for one in the scope's own code. */ - settles(scope: Scope, settle: number, ref: ts.Identifier, read = true): boolean { + * the store `settle` describes. `read`: `ref` reads the binding itself, + * so tsc vouches for one in the scope's own code. */ + settles(scope: Scope, settle: Settle, ref: ts.Identifier, read = true): boolean { if (ts.getSourceFileOfNode(ref) !== this.sf) return true; const statements = statementsOf(scope); let deferred: ts.Node | null = null; let top: ts.Node | null = null; + const container: ts.Node | undefined = ts.isSourceFile(scope) ? scope : scope.body; for (let n: ts.Node = ref; n.parent !== undefined && n !== scope; n = n.parent) { if ( ts.isFunctionLike(n) || @@ -265,18 +277,25 @@ class ScopeAnalysis { ts.isClassStaticBlockDeclaration(n) ) deferred = n; - if (n.parent === scope || (!ts.isSourceFile(scope) && n.parent === scope.body)) top = n; + // The scope's own statement, not the function body that holds it. + if (n.parent === container) top = n; } if (top === null) return false; if (deferred === null && read && !this.js) return true; if (deferred !== null && ts.isFunctionDeclaration(deferred) && deferred.name !== undefined) return this.functionSettles(scope, settle, deferred); const index = statements.indexOf(top as ts.Statement); - return index > settle || (index === settle && settlesWithin(top, ref)); + return ( + index > settle.index || + (index === settle.index && + (settlesWithin(top, ref) || + // A later declarator of the storing list (`var a = {}, b = { ...a }`). + (settle.list !== null && isInside(ref, settle.list) && ref.pos >= settle.end))) + ); } - private functionSettles(scope: Scope, settle: number, fn: ts.FunctionDeclaration): boolean { - const key = `${fn.pos}:${settle}`; + private functionSettles(scope: Scope, settle: Settle, fn: ts.FunctionDeclaration): boolean { + const key = `${fn.pos}:${settle.index}:${settle.end}`; const known = this.functionMemo.get(key); if (known !== undefined) return known; this.functionMemo.set(key, true); diff --git a/packages/compiler/src/frontend/npm-static.ts b/packages/compiler/src/frontend/npm-static.ts index 800d02eff..0d6c8ec8e 100644 --- a/packages/compiler/src/frontend/npm-static.ts +++ b/packages/compiler/src/frontend/npm-static.ts @@ -614,27 +614,13 @@ export function clearProgramBuiltinAliasCache(): void { * but the bundler-emitted CJS export rewrite (npm-static-rewrite.ts) * types the getter-table shapes and the offender attribution degrades a * package whose surface still breaks its consumers — so those bundles are - * worth ATTEMPTING: eligible when the .d.ts and unminified criteria hold, - * gracefully per-package-degraded when the attempt fails. */ + * worth ATTEMPTING: eligible when the .d.ts criteria hold, gracefully + * per-package-degraded when the attempt fails. Minified JS is attempted + * too: a whitespace-minified dist is ordinary JavaScript, and an + * identifier-minified bundle whose interop helpers lost their names is + * refused by the bundle recognizers, which name the reason. */ const TRANSFORM_MARKERS = ["__webpack_require__"]; -/** Unminified-JS heuristic over an entry source: at least two lines and - * an average line length under 200 characters (minified dists are one - * enormous line; readable code averages well under 100). */ -function looksUnminified(source: string): boolean { - const lines = source.split("\n"); - if (lines.length < 2) return false; - return source.length / lines.length <= 200; -} - -/** A whitespace/syntax-minified bundler dist whose interop helpers keep - * their names (`var __toESM=…,__toCommonJS=mod=>…`): the recognizers - * verify those shapes and refuse the package precisely when they cannot, - * so the minification heuristic need not refuse it up front. */ -function namedBundlerHelpers(source: string): boolean { - return /\b__(?:toCommonJS|commonJS|toESM)\d*\s*=\s*(?:\(|[A-Za-z_$][\w$]*\s*=>)/.test(source); -} - function hasTransformMarkers(source: string): boolean { return TRANSFORM_MARKERS.some((m) => source.includes(m)); } @@ -676,8 +662,6 @@ export function npmStaticIneligibleReason( if (jsEntry === null) return "no runtime JS entry resolves"; const source = trackedReadFile(jsEntry); if (source === null) return `its runtime entry ${jsEntry} cannot be read`; - if (!looksUnminified(source) && !namedBundlerHelpers(source)) - return "its shipped JS looks minified"; if (hasTransformMarkers(source)) { return "its shipped JS carries build-transform markers (bundled/transpiled dist)"; } diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7543-var-list-declarator-order.js.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7543-var-list-declarator-order.js.json new file mode 100644 index 000000000..d193e84ed --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7543-var-list-declarator-order.js.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7543-var-list-declarator-order.js" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7545-string-regex-checked-arguments.js.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7545-string-regex-checked-arguments.js.json new file mode 100644 index 000000000..03b3197c4 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7545-string-regex-checked-arguments.js.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7545-string-regex-checked-arguments.js" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/7546-minified-esm-rules/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/7546-minified-esm-rules/main.ts.json new file mode 100644 index 000000000..d026fb402 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/7546-minified-esm-rules/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/7546-minified-esm-rules/main.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7543-var-list-declarator-order.js b/tests/corpus/7543-var-list-declarator-order.js new file mode 100644 index 000000000..8fae06ff9 --- /dev/null +++ b/tests/corpus/7543-var-list-declarator-order.js @@ -0,0 +1,66 @@ +// A `var` read in a later declarator of the statement that stores it runs +// after the store; a read in the same or an earlier declarator sees undefined. +var base = { a: 1, b: "x" }, + more = { ...base, c: 2 }, + keys = Object.keys({ ...more }).join(","), + first = base.a + 1; +console.log(JSON.stringify(more), keys, first); + +function describeBase() { + return base.b + more.c; +} +var rules = { space: /\s+/, word: /\w+/ }, + merged = { ...rules, digit: /\d/ }, + described = describeBase(); +console.log(Object.keys(merged).join(","), merged.word.test("ab"), described); + +var table = { one: 1, two: 2 }, + lookup = function (key) { + return table[key]; + }, + arrow = (key) => ({ ...table })[key]; +console.log(lookup("one"), arrow("two")); + +var readLater = function () { + return later; + }, + early = readLater(), + later = { k: 1 }, + after = readLater().k; +console.log(early, after); + +var self = self ? 1 : 2; +console.log(self); + +for (var i = 0, j = i + 10; i < 2; i++) console.log(i, j); + +function nested() { + var inner = [1, 2, 3], + copy = [...inner, 4], + total = copy.reduce((sum, n) => sum + n, 0); + return { copy, total }; +} +console.log(JSON.stringify(nested())); + +function earlyInFunction() { + show("before"); + var point = { n: 1 }, + next = { m: point.n + 1 }; + show("after"); + function show(when) { + console.log(when, typeof point, point === undefined ? "none" : point.n, next === undefined ? "none" : next.m); + } + var check = () => console.log("check", items === undefined); + check(); + var items = [1]; + check(); +} +earlyInFunction(); + +function loopLists() { + for (var k = 0, sum = k; k < 3; k++) sum += k; + var doubled = sum * 2, + again = doubled + 1; + return [sum, doubled, again]; +} +console.log(JSON.stringify(loopLists())); diff --git a/tests/corpus/7545-string-regex-checked-arguments.js b/tests/corpus/7545-string-regex-checked-arguments.js new file mode 100644 index 000000000..a7ace9cdb --- /dev/null +++ b/tests/corpus/7545-string-regex-checked-arguments.js @@ -0,0 +1,89 @@ +// String search, match and replace with untyped pattern values: a RegExp +// is used as is, and any other value is converted as Node converts it. +const rules = JSON.parse('{"other":{}}'); +rules.other.space = /\s+/g; +rules.other.word = /o+/; +rules.other.text = "l"; +rules.other.dot = "."; +rules.other.none = undefined; +rules.other.nul = null; +rules.other.one = /(l)(o)/; +rules.other.many = /l+/g; +const t = " hello world"; + +console.log( + t.search(rules.other.word), + t.search(rules.other.text), + t.search(rules.other.none), + t.search(rules.other.nul), + t.search(rules.other.space), + "a.b".search(rules.other.dot), +); +console.log(rules.other.space.lastIndex); + +let conversions = 0; +rules.other.obj = { + toString() { + conversions++; + return "w"; + }, +}; +console.log(t.search(rules.other.obj), t.match(rules.other.obj)[0], conversions); + +const all = t.match(rules.other.many); +console.log(JSON.stringify(all), rules.other.many.lastIndex); +const pair = t.match(rules.other.one); +console.log(pair[0], pair[1], pair[2], pair.index, pair.length, pair.input === t); +const literal = t.match(rules.other.dot); +console.log(literal[0], literal.index, literal.length); +console.log(JSON.stringify(t.match(rules.other.none)), t.match(rules.other.zz)); +rules.other.zz = "zz"; +console.log(t.match(rules.other.zz)); +function first(text, re) { + const m = text.match(re); + return m ? m[0] : "none"; +} +console.log(first("abc", rules.other.dot), first("", rules.other.dot)); + +console.log(t.replace(rules.other.space, (m) => "[" + m.length + "]")); +console.log(t.replace(rules.other.word, (m, offset) => m.toUpperCase() + offset)); +console.log(t.replace(rules.other.text, (m, offset, subject) => "<" + m + offset + subject.length + ">")); +console.log(t.replaceAll(rules.other.text, (m) => m.toUpperCase())); +console.log(t.replaceAll(rules.other.space, () => "_")); +console.log(t.replace(rules.other.one, (m, p1, p2, offset) => p1 + "|" + p2 + "|" + offset)); +console.log(t.replace(rules.other.text, (m, offset, subject, extra) => typeof extra)); +console.log(t.replace(rules.other.text, (m) => m.length)); +try { + t.replaceAll(rules.other.word, (m) => m); +} catch (e) { + console.log(e.constructor.name, e.message); +} + +function tabs(text, width) { + let column = 0; + let out = ""; + for (const ch of text) { + if (ch === "\t") { + const pad = 4 - (column % 4); + out += " ".repeat(pad); + column += pad; + } else { + out += ch; + column++; + } + } + return out + width; +} +class Lexer { + constructor(table) { + this.rules = table; + } + indent(line, width) { + return line.replace(this.rules.leading, (w) => tabs(w, width)); + } + firstText(line) { + return line.search(this.rules.nonSpace); + } +} +const lexer = new Lexer({ leading: /^( {1,4}| {0,3}\t)/, nonSpace: /[^ ]/ }); +console.log(JSON.stringify(lexer.indent("\tabc", 2)), lexer.firstText(" x")); diff --git a/tests/fixtures/npm/cases/7546-minified-esm-rules/main.ts b/tests/fixtures/npm/cases/7546-minified-esm-rules/main.ts new file mode 100644 index 000000000..0c94acb0d --- /dev/null +++ b/tests/fixtures/npm/cases/7546-minified-esm-rules/main.ts @@ -0,0 +1,22 @@ +// A one-line minified ES module. Its module-level var statement spreads +// earlier declarators into later ones, its tokenizer reads regexes through +// an untyped rules table (search, match and replace with callbacks), and +// one function receives both typed and untyped link objects. +import { render } from "gtminimark"; + +const doc = [ + "# Title \\*one\\*", + "## Sub", + '[a](/x "T\\*")', + "[b](/y)", + '[r]: /ref "Ref"', + "[n]: /none", + "[R]", + "[n]", + "[z]", + "\tcode *em* line", + " three spaces *x* and *y*", + "plain words here", +].join("\n"); +console.log(render(doc)); +console.log(render(doc, { pedantic: true })); diff --git a/tests/fixtures/npm/node_modules/gtminimark/dist/index.d.ts b/tests/fixtures/npm/node_modules/gtminimark/dist/index.d.ts new file mode 100644 index 000000000..295430b86 --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtminimark/dist/index.d.ts @@ -0,0 +1 @@ +export declare function render(src: string, options?: { pedantic?: boolean }): string; diff --git a/tests/fixtures/npm/node_modules/gtminimark/dist/index.js b/tests/fixtures/npm/node_modules/gtminimark/dist/index.js new file mode 100644 index 000000000..c5e1daa31 --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtminimark/dist/index.js @@ -0,0 +1 @@ +var m = { heading: /^(#{1,6}) +(.*)$/, link: /^\[([^\]]*)\]\(([^ )]*)(?: "([^"]*)")?\)/, ref: /^\[([^\]]*)\]$/, lead: /^( {1,4}| {0,3}\t)/, nonSpace: /[^ ]/, punct: /\\([\\*_\[\]])/g, word: /\w+/g, def: /^\[([^\]]*)\]: (\S+)(?: "([^"]*)")?$/ },B = { ...m, emph: /\*([^*]+)\*/g },P = { ...B, word: /[a-z]+/g },K = { normal: { other: m, inline: B }, pedantic: { other: m, inline: P, strict: !0 } };function X(a, e) { let t = 0, n = ""; for (let i of a) if (i === "\t") { let r = 4 - t % 4; n += " ".repeat(r), t += r } else n += i, t++; return n + e; }function Se(a, e, t, n) { let r = e.href, o = e.title || null; n.links++; return { type: "link", raw: t, href: r, title: o, text: a[1] }; }class T {constructor() { this.state = { links: 0 }; }heading(s) { let c = s.match(this.rules.other.heading); return c ? "h" + c[1].length + ":" + c[2].replace(this.rules.other.punct, "$1") : null; }indent(s, w) { return s.replace(this.rules.other.lead, W => X(W, w)); }first(s) { return s.search(this.rules.other.nonSpace); }emph(s) { return s.replace(this.rules.inline.emph, (W, x) => "" + x + ""); }words(s) { let c = s.match(this.rules.inline.word); return c ? c.length : 0; }link(s) { let c = this.rules.other.link.exec(s); if (!c) return null; let r = c[2], o = ""; o = c[3] ? c[3] : ""; return Se(c, { href: r && r.replace(this.rules.other.punct, "$1"), title: o && o.replace(this.rules.other.punct, "$1") }, c[0], this.state); }ref(s, l) { let c = this.rules.other.ref.exec(s); if (!c) return null; let o = l[c[1].toLowerCase()]; return o ? Se(c, o, c[0], this.state) : { type: "text", raw: c[0] }; }}class L {constructor(e) { this.options = e || {}; this.links = {}; this.tokenizer = new T(); this.tokenizer.rules = this.options.pedantic ? K.pedantic : K.normal; }define(s) { let c = s.match(this.tokenizer.rules.other.def); if (c) this.links[c[1].toLowerCase()] = { href: c[2], title: c[3] || "" }; return !!c; }line(s) {let t = this.tokenizer, h = t.heading(s);if (h) return h;let k = t.link(s);if (k) return JSON.stringify(k);let f = t.ref(s, this.links);if (f) return JSON.stringify(f);return t.first(s) + "|" + JSON.stringify(t.indent(s, t.first(s))) + "|" + t.emph(s) + "|" + t.words(s);}}export function render(src, options) {let l = new L(options), out = [];for (let s of src.split("\n")) if (!l.define(s)) out.push(l.line(s));return out.join("\n") + "\nlinks " + l.tokenizer.state.links;} diff --git a/tests/fixtures/npm/node_modules/gtminimark/package.json b/tests/fixtures/npm/node_modules/gtminimark/package.json new file mode 100644 index 000000000..ff188c682 --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtminimark/package.json @@ -0,0 +1,8 @@ +{ + "name": "gtminimark", + "version": "1.0.0", + "description": "A one-line minified ES module: a module-level var statement whose later declarators spread earlier ones, regexes read through an untyped rules table, and a function called with typed and untyped link objects", + "type": "module", + "main": "./dist/index.js", + "types": "./dist/index.d.ts" +} diff --git a/tests/harness/corpus-costs/7543-var-list-declarator-order.js.json b/tests/harness/corpus-costs/7543-var-list-declarator-order.js.json new file mode 100644 index 000000000..cc47657bd --- /dev/null +++ b/tests/harness/corpus-costs/7543-var-list-declarator-order.js.json @@ -0,0 +1 @@ +{ "plain": 4300, "san": 5900 } diff --git a/tests/harness/corpus-costs/7545-string-regex-checked-arguments.js.json b/tests/harness/corpus-costs/7545-string-regex-checked-arguments.js.json new file mode 100644 index 000000000..83b82ee18 --- /dev/null +++ b/tests/harness/corpus-costs/7545-string-regex-checked-arguments.js.json @@ -0,0 +1 @@ +{ "plain": 4900, "san": 4600 } diff --git a/tests/harness/library-mode.test.ts b/tests/harness/library-mode.test.ts index 0d4052952..db4fabb3e 100644 --- a/tests/harness/library-mode.test.ts +++ b/tests/harness/library-mode.test.ts @@ -50,8 +50,8 @@ * no-teachings default is pinned by K5/K7 over * the teachings-free traps profile * K13 npm-posture bare npm specifiers are static-or-refuse: an - * eligible package (own .d.ts, unminified JS, - * no transform markers) compiles statically + * eligible package (own .d.ts, no transform + * markers) compiles statically * into the library graph, its own npm deps * judged by the same bar; every ineligible one * refuses SC4020 naming the failed bar — never @@ -746,11 +746,11 @@ describe("K9: library-mode refusals", () => { * synthesized per test like K9's. Emission never engages — SC4020 is a * frontend verdict — so one lane suffices. */ let npmRefusalCounter = 0; -async function npmRefusal( +async function compileNpmFixture( entryFile: string, exports: unknown[], profilePatch: Record = {}, -): Promise<{ code: string; message: string; hint?: string; note?: string; file: string }[]> { +): Promise>> { const outDir = join(cacheDir, `npm-refusal-${npmRefusalCounter++}`); mkdirSync(outDir, { recursive: true }); const profile = { @@ -770,7 +770,15 @@ async function npmRefusal( }; const profilePath = join(outDir, "profile.json"); writeFileSync(profilePath, JSON.stringify(profile)); - const result = await compileLibrary({ profilePath, outDir }); + return compileLibrary({ profilePath, outDir }); +} + +async function npmRefusal( + entryFile: string, + exports: unknown[], + profilePatch: Record = {}, +): Promise<{ code: string; message: string; hint?: string; note?: string; file: string }[]> { + const result = await compileNpmFixture(entryFile, exports, profilePatch); expect(result.ok).toBe(false); if (result.ok) throw new Error("unreachable"); return result.diagnostics.map((d) => ({ @@ -795,13 +803,11 @@ describe("K13: library-mode npm refusals (SC4020, static-or-refuse)", () => { expect(diags[0]!.file.endsWith("lib-markers.ts")).toBe(true); }); - test("minified shipped JS refuses", async () => { - const diags = await npmRefusal("lib-minified.ts", [ + test("minified shipped JS compiles", async () => { + const result = await compileNpmFixture("lib-minified.ts", [ { export: "f", symbol: "kx_f", params: ["f64", "f64"], returns: "f64" }, ]); - expect(diags.map((d) => d.code)).toEqual(["SC4020"]); - expect(diags[0]!.message).toContain("'slimmed'"); - expect(diags[0]!.message).toContain("looks minified"); + expect(result.ok ? [] : result.diagnostics.map((d) => d.message)).toEqual([]); }); test("no own .d.ts refuses by name — never the generic import fence", async () => { diff --git a/tests/harness/npm-static.test.ts b/tests/harness/npm-static.test.ts index 7ee76307f..11e729f60 100644 --- a/tests/harness/npm-static.test.ts +++ b/tests/harness/npm-static.test.ts @@ -1490,6 +1490,15 @@ describe(`npm-static pilots${sanitize ? " (sanitized)" : ""}`, () => { ]); }, 180_000); + // 7546: a one-line minified ES module: a var statement whose later + // declarators spread earlier ones, regexes read through an untyped rules + // table, and a function instantiated with typed and untyped arguments. + test("the minified module 7546-minified-esm-rules compiles statically and matches Node", async () => { + await expectNodeParity(join(fixturesRoot, "npm/cases/7546-minified-esm-rules/main.ts"), [ + "gtminimark", + ]); + }, 180_000); + // A whitespace/syntax-minified bundle whose helpers keep their names is // judged by the recognizers, not refused up front as minified. test("--npm-static=auto opts a minified bundle with named helpers in", () => { diff --git a/tests/library-mode/npm-refuse/lib-minified.ts b/tests/library-mode/npm-refuse/lib-minified.ts index 9e5c16e75..492cd5369 100644 --- a/tests/library-mode/npm-refuse/lib-minified.ts +++ b/tests/library-mode/npm-refuse/lib-minified.ts @@ -1,5 +1,5 @@ -// SC4020 fixture: slimmed ships a one-long-line dist — minified JS fails -// the eligibility bar. +// slimmed ships a one-long-line dist: minified JS compiles like any other +// eligible package. import { pick } from "slimmed"; export function f(a: number, b: number): number { diff --git a/tests/library-mode/npm-static/lib.ts b/tests/library-mode/npm-static/lib.ts index 27107f29d..15bad17a3 100644 --- a/tests/library-mode/npm-static/lib.ts +++ b/tests/library-mode/npm-static/lib.ts @@ -1,5 +1,5 @@ // Library-mode npm posture fixture (static-or-refuse): a bare specifier -// naming an ELIGIBLE package (own .d.ts, unminified shipped JS, no +// naming an ELIGIBLE package (own .d.ts, no // build-transform markers) compiles statically as part of the library // graph — no flag, no island — and the package's own top-level require // ("mathdep") joins through the same bar. The node:path import alongside diff --git a/tests/library-mode/npm-static/node_modules/mathkit/index.js b/tests/library-mode/npm-static/node_modules/mathkit/index.js index f9f9242c3..eadfefba9 100644 --- a/tests/library-mode/npm-static/node_modules/mathkit/index.js +++ b/tests/library-mode/npm-static/node_modules/mathkit/index.js @@ -1,6 +1,6 @@ "use strict"; // A vendored fake package for the library-mode npm posture: eligible by -// every bar (own .d.ts, unminified CJS, no build-transform markers), with +// every bar (own .d.ts, no build-transform markers), with // its OWN npm dep edge (mathdep) so the transitive fixpoint is exercised. const steps = require("mathdep"); From ba5d73e90b3289a52a31d75095da4b7d50daad50 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 06:20:02 -0500 Subject: [PATCH 15/53] Recognize esbuild's __commonJS helper under a minified name Refs #1777 An identifier-minified esbuild bundle renames the __commonJS helper (var j=(u,t)=>()=>(t||u((t={exports:{}}).exports,t),t.exports)). The recognizer already compared the helper's shape independently of its parameter names, but only used a renamed match to refuse the package. A stable top-level declaration with the helper's shape now splits the bundle's chunks like the named helper, so lru-cache's CommonJS build is no longer refused. --- docs/content/docs/dependencies.mdx | 2 +- .../src/frontend/lowering/lower-calls.ts | 27 +++++++++++++++++++ .../npm-static-bundled-chunks.test.ts | 11 ++++---- .../src/frontend/npm-static-bundled-chunks.ts | 20 ++++++-------- .../src/frontend/npm-static-rewrite-syntax.ts | 14 ++++++++-- packages/compiler/src/frontend/npm-static.ts | 5 ++-- packages/compiler/src/frontend/program.ts | 16 +++++++++-- .../main.cjs.json | 7 +++++ .../main.cjs.json | 8 ++++++ .../7547-minified-cjs-chunks/main.ts.json | 6 +++++ .../lib.cjs | 10 +++++++ .../main.cjs | 19 +++++++++++++ .../7549-cjs-minified-esmodule-stamp/main.cjs | 7 +++++ .../module-stamped.cjs | 3 +++ .../stamped.cjs | 5 ++++ .../cases/7547-minified-cjs-chunks/main.ts | 12 +++++++++ .../node_modules/gtminicache/dist/index.d.ts | 7 +++++ .../node_modules/gtminicache/dist/index.js | 1 + .../npm/node_modules/gtminicache/package.json | 7 +++++ .../main.cjs.json | 1 + .../main.cjs.json | 1 + tests/harness/npm-static.test.ts | 9 +++++++ 22 files changed, 173 insertions(+), 25 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7548-cjs-receiverless-builtin-calls/main.cjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7549-cjs-minified-esmodule-stamp/main.cjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/7547-minified-cjs-chunks/main.ts.json create mode 100644 tests/corpus/7548-cjs-receiverless-builtin-calls/lib.cjs create mode 100644 tests/corpus/7548-cjs-receiverless-builtin-calls/main.cjs create mode 100644 tests/corpus/7549-cjs-minified-esmodule-stamp/main.cjs create mode 100644 tests/corpus/7549-cjs-minified-esmodule-stamp/module-stamped.cjs create mode 100644 tests/corpus/7549-cjs-minified-esmodule-stamp/stamped.cjs create mode 100644 tests/fixtures/npm/cases/7547-minified-cjs-chunks/main.ts create mode 100644 tests/fixtures/npm/node_modules/gtminicache/dist/index.d.ts create mode 100644 tests/fixtures/npm/node_modules/gtminicache/dist/index.js create mode 100644 tests/fixtures/npm/node_modules/gtminicache/package.json create mode 100644 tests/harness/corpus-costs/7548-cjs-receiverless-builtin-calls/main.cjs.json create mode 100644 tests/harness/corpus-costs/7549-cjs-minified-esmodule-stamp/main.cjs.json diff --git a/docs/content/docs/dependencies.mdx b/docs/content/docs/dependencies.mdx index f02dcfbf0..a9c5ae138 100644 --- a/docs/content/docs/dependencies.mdx +++ b/docs/content/docs/dependencies.mdx @@ -53,7 +53,7 @@ Type-only imports follow Node.js's type stripping. `import type` and `export typ scriptc compiles a package automatically when it: - ships its own type declarations, or is typed by its DefinitelyTyped `@types/` package, or is a dependency of another package that has no declarations at all, and -- ships JavaScript that isn't a bundle whose bundler-interop helpers, such as `__commonJS`, have lost their names. Minified JavaScript compiles like unminified JavaScript. +- ships JavaScript that isn't a webpack bundle. Minified JavaScript compiles like unminified JavaScript. The declarations check your program; the package's JavaScript compiles with the types scriptc infers from it. A program that doesn't typecheck against the inferred types, for example because an `@types/` package describes another release, fails the build at that use. diff --git a/packages/compiler/src/frontend/lowering/lower-calls.ts b/packages/compiler/src/frontend/lowering/lower-calls.ts index 436d4f1d4..672801e9e 100644 --- a/packages/compiler/src/frontend/lowering/lower-calls.ts +++ b/packages/compiler/src/frontend/lowering/lower-calls.ts @@ -306,6 +306,22 @@ function mayHoldError(lowerer: Lowerer, type: ts.Type, errorType: ts.Type | null ); } +/** The `f` of a `(0, f)` callee (a numeric left operand, so dropping it + * changes only the call's receiver), or null. */ +function receiverlessCalleeOf(callee: ts.Expression): ts.Expression | null { + let e = callee; + while (ts.isParenthesizedExpression(e)) e = e.expression; + if ( + !ts.isBinaryExpression(e) || + e.operatorToken.kind !== ts.SyntaxKind.CommaToken || + !ts.isNumericLiteral(e.left) + ) + return null; + let right = e.right; + while (ts.isParenthesizedExpression(right)) right = right.expression; + return right; +} + export function lowerCall(lowerer: Lowerer, expr: ts.CallExpression): IrExpr { const loc = locOf(expr); if (lowerer.errorAccessorReflection === null) noteErrorAccessorReflection(lowerer, expr); @@ -329,6 +345,17 @@ export function lowerCall(lowerer: Lowerer, expr: ts.CallExpression): IrExpr { const json = lowerJsonRequireCall(lowerer, expr); if (json) return json; + // `(0, ns.member)(...)` over a builtin module (tsc's and esbuild's call + // without a receiver): Node's module functions do not read `this`, so it + // is the member call. + { + const member = receiverlessCalleeOf(expr.expression); + if (member !== null && ts.isPropertyAccessExpression(member)) { + const builtin = lowerer.lowerNamespaceBuiltinCall(expr, member); + if (builtin) return builtin; + } + } + // eval(...) and Function(...): Node's EvalError under // --disallow-code-generation-from-strings (code-generation.ts). { diff --git a/packages/compiler/src/frontend/npm-static-bundled-chunks.test.ts b/packages/compiler/src/frontend/npm-static-bundled-chunks.test.ts index 0779db3ce..dc238c622 100644 --- a/packages/compiler/src/frontend/npm-static-bundled-chunks.test.ts +++ b/packages/compiler/src/frontend/npm-static-bundled-chunks.test.ts @@ -276,13 +276,14 @@ export default require_f(); ).toBeNull(); }); -test("an identifier-minified bundle names its lost helper names", () => { - // esbuild --minify output: the helper shapes survive, their names do not. - const result = split( +test("an identifier-minified bundle splits through its helper's shape", () => { + // esbuild --minify output: the helper shape survives, its name does not. + const answer = expectSameBehavior( `var c=(r,e)=>()=>(e||r((e={exports:{}}).exports,e),e.exports);` + - `var x=c((t,m)=>{m.exports={v:1}});console.log(x().v);`, + `var x=c((t,m)=>{m.exports={v:1}});var y=c(t=>{t.w=x().v+1});` + + `console.log(x().v,y().w,x()===x());`, ); - expect(result).toEqual({ degrade: expect.stringContaining("identifier-minified") as string }); + expect(answer.chunks.map((c) => basename(c.path))).toEqual(["index.js.x.cjs", "index.js.y.cjs"]); }); test("an ES-module bundle of CommonJS chunks names the chunk interop", () => { diff --git a/packages/compiler/src/frontend/npm-static-bundled-chunks.ts b/packages/compiler/src/frontend/npm-static-bundled-chunks.ts index 37a74f2be..66c25403b 100644 --- a/packages/compiler/src/frontend/npm-static-bundled-chunks.ts +++ b/packages/compiler/src/frontend/npm-static-bundled-chunks.ts @@ -525,7 +525,7 @@ export function splitBundledChunks( } } } - const helperNames = [...top.keys()].filter((n) => helperBase(n) === "__commonJS"); + let helperNames = [...top.keys()].filter((n) => helperBase(n) === "__commonJS"); // Every name a recognizer relies on is written nowhere and declared // once at the top level; `Object` itself is never rebound. @@ -552,18 +552,14 @@ export function splitBundledChunks( ); }; if (helperNames.length === 0) { - // The helper's shape under another name: an identifier-minified bundle, - // whose other interop helpers lost their names as well. - for (const [name, t] of top) { + // An identifier-minified bundle: the helper's shape under another name. + helperNames = [...top].flatMap(([name, t]) => { const init = ts.isVariableDeclaration(t.node) ? t.node.initializer : undefined; - if (init !== undefined && stable(name) && helperForm(init, sf, ownNames) !== null) - return { - degrade: - "its shipped JS is an identifier-minified bundle whose bundler-interop helpers " + - "(__commonJS and the rest) have lost the names the recognizers verify", - }; - } - return null; + return init !== undefined && stable(name) && helperForm(init, sf, ownNames) !== null + ? [name] + : []; + }); + if (helperNames.length === 0) return null; } const forms = new Map(); for (const name of helperNames) { diff --git a/packages/compiler/src/frontend/npm-static-rewrite-syntax.ts b/packages/compiler/src/frontend/npm-static-rewrite-syntax.ts index c48420625..917041603 100644 --- a/packages/compiler/src/frontend/npm-static-rewrite-syntax.ts +++ b/packages/compiler/src/frontend/npm-static-rewrite-syntax.ts @@ -260,11 +260,21 @@ function definePropertyExportOf( value = getterReturnOf(p); } } - const esModuleStamp = - name === "__esModule" && value !== null && value.kind === ts.SyntaxKind.TrueKeyword; + const esModuleStamp = name === "__esModule" && value !== null && isTrueLiteral(value); return { name, value, esModuleStamp }; } +/** `true`, or `!0`, its minified spelling. */ +function isTrueLiteral(expr: ts.Expression): boolean { + return ( + expr.kind === ts.SyntaxKind.TrueKeyword || + (ts.isPrefixUnaryExpression(expr) && + expr.operator === ts.SyntaxKind.ExclamationToken && + ts.isNumericLiteral(expr.operand) && + expr.operand.text === "0") + ); +} + /** Space-pads [start, end) of `text` preserving newlines — the recognized * statement disappears while every original offset survives. */ function padSpan(text: string, start: number, end: number): string { diff --git a/packages/compiler/src/frontend/npm-static.ts b/packages/compiler/src/frontend/npm-static.ts index 0d6c8ec8e..eed02c95c 100644 --- a/packages/compiler/src/frontend/npm-static.ts +++ b/packages/compiler/src/frontend/npm-static.ts @@ -616,9 +616,8 @@ export function clearProgramBuiltinAliasCache(): void { * package whose surface still breaks its consumers — so those bundles are * worth ATTEMPTING: eligible when the .d.ts criteria hold, gracefully * per-package-degraded when the attempt fails. Minified JS is attempted - * too: a whitespace-minified dist is ordinary JavaScript, and an - * identifier-minified bundle whose interop helpers lost their names is - * refused by the bundle recognizers, which name the reason. */ + * too: a minified dist is ordinary JavaScript, and the bundle recognizers + * know esbuild's __commonJS helper by its shape under any name. */ const TRANSFORM_MARKERS = ["__webpack_require__"]; function hasTransformMarkers(source: string): boolean { diff --git a/packages/compiler/src/frontend/program.ts b/packages/compiler/src/frontend/program.ts index 68b649b07..932ac7054 100644 --- a/packages/compiler/src/frontend/program.ts +++ b/packages/compiler/src/frontend/program.ts @@ -1436,8 +1436,20 @@ function requiresOf7( return out; } +/** `true`, or `!0`, its minified spelling. */ +function isTrueLiteral(expr: ts.Expression): boolean { + return ( + expr.kind === ts.SyntaxKind.TrueKeyword || + (ts.isPrefixUnaryExpression(expr) && + expr.operator === ts.SyntaxKind.ExclamationToken && + ts.isNumericLiteral(expr.operand) && + expr.operand.text === "0") + ); +} + /** The exact `Object.defineProperty(exports|module.exports, "__esModule", - * { value: true })` interop stamp (shared by preflight and the no-op lowering). */ + * { value: true })` interop stamp (shared by preflight and the no-op lowering); + * `{ value: !0 }` is the same stamp. */ export function isEsModuleStamp(expr: ts.Expression): boolean { if (!ts.isCallExpression(expr) || expr.questionDotToken !== undefined) return false; const callee = expr.expression; @@ -1471,7 +1483,7 @@ export function isEsModuleStamp(expr: ts.Expression): boolean { ts.isPropertyAssignment(p) && ts.isIdentifier(p.name) && p.name.text === "value" && - p.initializer.kind === ts.SyntaxKind.TrueKeyword + isTrueLiteral(p.initializer) ); } diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7548-cjs-receiverless-builtin-calls/main.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7548-cjs-receiverless-builtin-calls/main.cjs.json new file mode 100644 index 000000000..f3b8ba419 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7548-cjs-receiverless-builtin-calls/main.cjs.json @@ -0,0 +1,7 @@ +{ + "order": [ + "/tests/corpus/7548-cjs-receiverless-builtin-calls/lib.cjs", + "/tests/corpus/7548-cjs-receiverless-builtin-calls/main.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7549-cjs-minified-esmodule-stamp/main.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7549-cjs-minified-esmodule-stamp/main.cjs.json new file mode 100644 index 000000000..08adae885 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7549-cjs-minified-esmodule-stamp/main.cjs.json @@ -0,0 +1,8 @@ +{ + "order": [ + "/tests/corpus/7549-cjs-minified-esmodule-stamp/stamped.cjs", + "/tests/corpus/7549-cjs-minified-esmodule-stamp/module-stamped.cjs", + "/tests/corpus/7549-cjs-minified-esmodule-stamp/main.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/7547-minified-cjs-chunks/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/7547-minified-cjs-chunks/main.ts.json new file mode 100644 index 000000000..212551ccd --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/7547-minified-cjs-chunks/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/7547-minified-cjs-chunks/main.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7548-cjs-receiverless-builtin-calls/lib.cjs b/tests/corpus/7548-cjs-receiverless-builtin-calls/lib.cjs new file mode 100644 index 000000000..248e66270 --- /dev/null +++ b/tests/corpus/7548-cjs-receiverless-builtin-calls/lib.cjs @@ -0,0 +1,10 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: !0 }); +exports.label = exports.describe = void 0; +const node_util_1 = require("node:util"); +const node_path_1 = require("node:path"); +function describe(name, size) { + return (0, node_util_1.format)("%s holds %d (%s)", name, size, (0, node_path_1.basename)("/a/b/c.txt")); +} +exports.describe = describe; +exports.label = "lib"; diff --git a/tests/corpus/7548-cjs-receiverless-builtin-calls/main.cjs b/tests/corpus/7548-cjs-receiverless-builtin-calls/main.cjs new file mode 100644 index 000000000..6fd0b78a9 --- /dev/null +++ b/tests/corpus/7548-cjs-receiverless-builtin-calls/main.cjs @@ -0,0 +1,19 @@ +"use strict"; +// tsc and esbuild output: the __esModule stamp spelled { value: !0 }, and +// calls of builtin module functions without a receiver, (0, ns.member)(...). +const lib = require("./lib.cjs"); +const dc = require("node:diagnostics_channel"); +const util = require("node:util"); +const os = require("node:os"); +const path = require("node:path"); +console.log(lib.label, lib.describe("cache", 3)); +const metrics = (0, dc.channel)("cache:metrics"); +console.log(metrics.name, metrics.hasSubscribers); +const seen = []; +dc.subscribe("cache:metrics", (message) => seen.push(message.key)); +metrics.publish({ key: "a" }); +console.log(seen.join(","), metrics.hasSubscribers); +const tracing = (0, dc.tracingChannel)("cache"); +console.log(tracing.hasSubscribers); +console.log((0, util.inspect)({ a: [1, 2] }), (0, util.isDeepStrictEqual)([1], [1])); +console.log((0, path.join)("a", "b"), typeof (0, os.platform)()); diff --git a/tests/corpus/7549-cjs-minified-esmodule-stamp/main.cjs b/tests/corpus/7549-cjs-minified-esmodule-stamp/main.cjs new file mode 100644 index 000000000..c619a847a --- /dev/null +++ b/tests/corpus/7549-cjs-minified-esmodule-stamp/main.cjs @@ -0,0 +1,7 @@ +"use strict"; +// Minifiers spell `true` as `!0`: Object.defineProperty(exports, +// "__esModule", { value: !0 }) is the same interop stamp as { value: true }. +const stamped = require("./stamped.cjs"); +const moduleStamped = require("./module-stamped.cjs"); +console.log(stamped.answer, stamped.twice(4), moduleStamped.label); +console.log(typeof stamped.twice, moduleStamped.label.length); diff --git a/tests/corpus/7549-cjs-minified-esmodule-stamp/module-stamped.cjs b/tests/corpus/7549-cjs-minified-esmodule-stamp/module-stamped.cjs new file mode 100644 index 000000000..96a2aae7b --- /dev/null +++ b/tests/corpus/7549-cjs-minified-esmodule-stamp/module-stamped.cjs @@ -0,0 +1,3 @@ +"use strict"; +Object.defineProperty(module.exports, "__esModule", { value: !0 }); +module.exports.label = "module"; diff --git a/tests/corpus/7549-cjs-minified-esmodule-stamp/stamped.cjs b/tests/corpus/7549-cjs-minified-esmodule-stamp/stamped.cjs new file mode 100644 index 000000000..cd3b9c400 --- /dev/null +++ b/tests/corpus/7549-cjs-minified-esmodule-stamp/stamped.cjs @@ -0,0 +1,5 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: !0 }); +exports.answer = void 0; +exports.answer = 42; +exports.twice = (n) => n * 2; diff --git a/tests/fixtures/npm/cases/7547-minified-cjs-chunks/main.ts b/tests/fixtures/npm/cases/7547-minified-cjs-chunks/main.ts new file mode 100644 index 000000000..ad2303f47 --- /dev/null +++ b/tests/fixtures/npm/cases/7547-minified-cjs-chunks/main.ts @@ -0,0 +1,12 @@ +// An identifier-minified esbuild CommonJS bundle (lru-cache's shape): its +// __commonJS helper and chunk parameters lost their names, the chunks +// stamp __esModule with { value: !0 }, and they call node:util and +// node:diagnostics_channel functions in the (0, ns.member)(...) form. +import { Cache } from "gtminicache"; + +const cache = new Cache(2); +cache.set("a", 1).set("b", 2); +cache.get("a"); +cache.set("c", 3); +console.log(cache.keys().join(","), cache.get("b"), cache.get("c")); +console.log(cache.describe("cache")); diff --git a/tests/fixtures/npm/node_modules/gtminicache/dist/index.d.ts b/tests/fixtures/npm/node_modules/gtminicache/dist/index.d.ts new file mode 100644 index 000000000..5d12cad7e --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtminicache/dist/index.d.ts @@ -0,0 +1,7 @@ +export declare class Cache { + constructor(max: number); + set(key: K, value: V): this; + get(key: K): V | undefined; + keys(): K[]; + describe(label: string): string; +} diff --git a/tests/fixtures/npm/node_modules/gtminicache/dist/index.js b/tests/fixtures/npm/node_modules/gtminicache/dist/index.js new file mode 100644 index 000000000..31c6deef9 --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtminicache/dist/index.js @@ -0,0 +1 @@ +"use strict";var n=(s,e)=>()=>(e||s((e={exports:{}}).exports,e),e.exports);var u=n(i=>{"use strict";Object.defineProperty(i,"__esModule",{value:!0});i.metrics=void 0;var o=require("node:diagnostics_channel");i.metrics=(0,o.channel)("gtminicache:metrics")});var d=n(r=>{"use strict";Object.defineProperty(r,"__esModule",{value:!0});r.describe=a;var l=require("node:util");function a(s,e){return(0,l.format)("%s holds %d",s,e)}});Object.defineProperty(exports,"__esModule",{value:!0});exports.Cache=void 0;var h=u(),_=d(),c=class{#e=new Map;#t;constructor(e){this.#t=e}set(e,t){for(this.#e.delete(e),this.#e.set(e,t);this.#e.size>this.#t;)this.#e.delete(this.#e.keys().next().value);return h.metrics.hasSubscribers&&h.metrics.publish({key:e}),this}get(e){let t=this.#e.get(e);return t!==void 0&&(this.#e.delete(e),this.#e.set(e,t)),t}keys(){return[...this.#e.keys()]}describe(e){return(0,_.describe)(e,this.#e.size)}};exports.Cache=c; diff --git a/tests/fixtures/npm/node_modules/gtminicache/package.json b/tests/fixtures/npm/node_modules/gtminicache/package.json new file mode 100644 index 000000000..3f57e1ae3 --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtminicache/package.json @@ -0,0 +1,7 @@ +{ + "name": "gtminicache", + "version": "1.0.0", + "description": "tsc-style CommonJS modules bundled by esbuild 0.27.0 with --bundle --platform=node --format=cjs --minify: the __commonJS helper and the chunk parameters lose their names, the __esModule stamps read { value: !0 }, and calls into node:util and node:diagnostics_channel use the (0, ns.member)(...) form", + "main": "./dist/index.js", + "types": "./dist/index.d.ts" +} diff --git a/tests/harness/corpus-costs/7548-cjs-receiverless-builtin-calls/main.cjs.json b/tests/harness/corpus-costs/7548-cjs-receiverless-builtin-calls/main.cjs.json new file mode 100644 index 000000000..8a603b2f4 --- /dev/null +++ b/tests/harness/corpus-costs/7548-cjs-receiverless-builtin-calls/main.cjs.json @@ -0,0 +1 @@ +{ "plain": 12300, "san": 13600 } diff --git a/tests/harness/corpus-costs/7549-cjs-minified-esmodule-stamp/main.cjs.json b/tests/harness/corpus-costs/7549-cjs-minified-esmodule-stamp/main.cjs.json new file mode 100644 index 000000000..d00a49ec8 --- /dev/null +++ b/tests/harness/corpus-costs/7549-cjs-minified-esmodule-stamp/main.cjs.json @@ -0,0 +1 @@ +{ "plain": 16000, "san": 21200 } diff --git a/tests/harness/npm-static.test.ts b/tests/harness/npm-static.test.ts index 11e729f60..388da2138 100644 --- a/tests/harness/npm-static.test.ts +++ b/tests/harness/npm-static.test.ts @@ -1499,6 +1499,15 @@ describe(`npm-static pilots${sanitize ? " (sanitized)" : ""}`, () => { ]); }, 180_000); + // 7547: an identifier-minified esbuild CommonJS bundle: the __commonJS + // helper and chunk parameters renamed, { value: !0 } stamps, and + // (0, ns.member)(...) calls of builtin module functions. + test("the identifier-minified bundle 7547-minified-cjs-chunks compiles statically and matches Node", async () => { + await expectNodeParity(join(fixturesRoot, "npm/cases/7547-minified-cjs-chunks/main.ts"), [ + "gtminicache", + ]); + }, 180_000); + // A whitespace/syntax-minified bundle whose helpers keep their names is // judged by the recognizers, not refused up front as minified. test("--npm-static=auto opts a minified bundle with named helpers in", () => { From b8a6a6a939dbfdd666bbfc48558083dd61b68d48 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 06:59:05 -0500 Subject: [PATCH 16/53] Expect an untyped dependency of an opted-in library package to compile The test refused unchecked-dependency only because its one-line index.js looked minified. A dependency another package's JavaScript imports needs no declarations of its own (npmStaticIneligibleReason's untypedDependency), so with the minified refusal gone it compiles; the test now builds it and checks the library's result. --- tests/harness/library-static-types.test.ts | 23 +++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/tests/harness/library-static-types.test.ts b/tests/harness/library-static-types.test.ts index dfbd70bf9..36e2befac 100644 --- a/tests/harness/library-static-types.test.ts +++ b/tests/harness/library-static-types.test.ts @@ -81,7 +81,8 @@ test.each(["sample", "@scope/sample"])( expect(actual).toBe(reference); expect(actual).toBe("22\n"); } - // Opting in to one runtime package does not admit its untyped dependency. + // A dependency the package's JavaScript imports compiles without + // declarations of its own: no declaration checks that import. const dependency = join(dir, "node_modules/unchecked-dependency"); await mkdir(dependency, { recursive: true }); await writeFile( @@ -96,14 +97,18 @@ test.each(["sample", "@scope/sample"])( join(pkg, "index.js"), "import { adjust } from 'unchecked-dependency'; export function calculate(value) { return adjust(value) * 3 + 1; }\n", ); - const refused = await compileLibrary({ profilePath, outDir: join(dir, "refused") }); - expect(refused.ok).toBe(false); - if (!refused.ok) - expect( - refused.diagnostics.some( - (d) => d.code === "SC4020" && d.message.includes("unchecked-dependency"), - ), - ).toBe(true); + const withDependency = await compileLibrary({ profilePath, outDir: join(dir, "dependency") }); + if (!withDependency.ok) throw new Error(JSON.stringify(withDependency.diagnostics)); + if (process.env["SCRIPTC_PORTABLE_ONLY"] !== "1") { + const source = join(dir, "dependency-host.c"), + binary = join(dir, "dependency-host"); + await writeFile( + source, + '#include \nextern void st_init(void); extern double st_result(void);\nint main(void) { st_init(); printf("%.0f\\n", st_result()); }\n', + ); + execFileSync("clang", [source, withDependency.archivePath, "-lm", "-o", binary]); + expect(execFileSync(binary, { encoding: "utf8" })).toBe("22\n"); + } expect(await readFile(join(types, "index.d.ts"), "utf8")).toBe(declaration); } finally { await rm(dir, { recursive: true, force: true }); From da1dae0625fde0ef8168e07de7092d30213fc5db Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 07:20:08 -0500 Subject: [PATCH 17/53] Run two-counter loops and element copies over number arrays without per-element checks --- .../src/backend/llvm/dense-array-access.ts | 4 + .../backend/llvm/dense-array-loops.test.ts | 96 ++++++- .../src/backend/llvm/dense-array-loops.ts | 235 +++++++++++++++++- .../src/backend/llvm/emit-statements.ts | 17 +- packages/compiler/src/backend/llvm/emitter.ts | 8 +- .../src/backend/llvm/expr-primitives.ts | 4 +- .../src/backend/llvm/local-array-reads.ts | 28 ++- .../src/ir/array-length-stability.test.ts | 31 +++ .../compiler/src/ir/array-length-stability.ts | 32 ++- .../7264-array-dense-paired-loops.ts.json | 6 + tests/corpus/7264-array-dense-paired-loops.ts | 120 +++++++++ .../7264-array-dense-paired-loops.ts.json | 1 + 12 files changed, 561 insertions(+), 21 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7264-array-dense-paired-loops.ts.json create mode 100644 tests/corpus/7264-array-dense-paired-loops.ts create mode 100644 tests/harness/corpus-costs/7264-array-dense-paired-loops.ts.json diff --git a/packages/compiler/src/backend/llvm/dense-array-access.ts b/packages/compiler/src/backend/llvm/dense-array-access.ts index 95612e57c..df37d9c58 100644 --- a/packages/compiler/src/backend/llvm/dense-array-access.ts +++ b/packages/compiler/src/backend/llvm/dense-array-access.ts @@ -505,8 +505,12 @@ export function emitDenseArrayState( indexExpr: IrExpr | undefined, kind: "state" | "has", width: number, + /** The index is a dense array loop counter its entry test proved below + * the hole-free prefix (provenDenseIndex): the slot is a present value. */ + proven = false, ): string { const B = host.B; + if (proven) return kind === "state" ? f64Lit(SCR_ARR_VALUE) : "true"; const fn = kind === "state" ? "scr_arr_state" : "scr_arr_has"; const ty = kind === "state" ? "double" : "i1"; host.declare(`declare ${kind === "state" ? "double" : "zeroext i1"} @${fn}(ptr, double)`); diff --git a/packages/compiler/src/backend/llvm/dense-array-loops.test.ts b/packages/compiler/src/backend/llvm/dense-array-loops.test.ts index f7a4bc892..4a81a9bfe 100644 --- a/packages/compiler/src/backend/llvm/dense-array-loops.test.ts +++ b/packages/compiler/src/backend/llvm/dense-array-loops.test.ts @@ -1,9 +1,10 @@ import { expect, test } from "vitest"; import { BOOL, F64, arrayOf, type IrExpr, type IrLocal, type IrStmt } from "../../ir/ir.js"; -import type { IntegerCountedForLoop } from "../../ir/integer-loops.js"; +import type { IntegerCountedForLoop, IntegerPairedForLoop } from "../../ir/integer-loops.js"; import { countedDenseShape, matchDenseArrayLoop, + pairedDenseShape, type DenseLoopShape, } from "./dense-array-loops.js"; @@ -42,6 +43,8 @@ const locals = new Map([ ["a", { id: "a", name: "a", type: ARR, mutable: false }], ["b", { id: "b", name: "b", type: ARR, mutable: true }], ["c", { id: "c", name: "c", type: ARR, mutable: false }], + ["j", { id: "j", name: "j", type: F64, mutable: true }], + ["t", { id: "t", name: "t", type: F64, mutable: false }], ]); /** for (let i = 0; i < a.length; i++) body */ @@ -193,3 +196,94 @@ test("a loop to another invariant limit covers each array whose length it keeps" expect([...matchLimited(resizing, new Set(["b"]))!.arrays.keys()]).toEqual(["a"]); expect(matchLimited(resizing)).toBeNull(); }); + +test("a two-counter loop covers the arrays indexed at either counter", () => { + // for (let i = 1, j = k - 1; i < j; i++, j--) { t = a[i]; a[i] = a[j]; a[j] = t; } + const paired: IntegerPairedForLoop = { + low: { localId: "i", start: num(1), step: 1, startGuarded: false }, + high: { localId: "j", start: ref("k"), step: -1, startGuarded: true }, + inclusive: false, + guarded: true, + range: { min: -Number.MAX_SAFE_INTEGER, max: Number.MAX_SAFE_INTEGER }, + }; + const pairedShape = pairedDenseShape(paired); + const matchPaired = (body: IrStmt[]) => + matchDenseArrayLoop(loop(body), pairedShape, locals, new Set(), new Set(["a", "b"])); + const swap: IrStmt[] = [ + { kind: "varDecl", localId: "t", init: read("a"), loc }, + write("a", read("a", ref("j"))), + write("a", ref("t"), ref("j")), + ]; + const dense = matchPaired(swap); + expect([...dense!.arrays.keys()]).toEqual(["a"]); + expect(dense!.pair).toEqual({ high: "j", lowStart: num(1), highStart: ref("k") }); + expect(dense!.range).toEqual({ min: 0, max: 2147483646 }); + // Stores at any other index may extend the array. + expect(matchPaired([...swap, write("a", num(0), ref("k"))])).toBeNull(); +}); + +test("a present-undefined store is covered only where it stores a covered read's missing arm", () => { + // const v = b[i] (as number | undefined); + // if (v is undefined) c[i] = undefined; else c[i] = v is number ? v : throw; + const unionType = { kind: "union" as const, unionId: "u" }; + const v: IrExpr = { kind: "varRef", localId: "v", type: unionType, loc }; + const isTag = (tag: number): IrExpr => ({ + kind: "unionIsTag", + unionId: "u", + tag, + negated: false, + value: v, + type: BOOL, + loc, + }); + const reader = (array: string, index = ref("i")): IrExpr => ({ + kind: "call", + callee: "%arr.idxOr.f64", + args: [arr(array), index], + type: unionType, + loc, + }); + const undefinedStore: IrStmt = { kind: "arraySetUndefined", arr: arr("c"), index: ref("i"), loc }; + const guarded = (array: string): IrStmt[] => [ + { kind: "varDecl", localId: "v", init: reader(array), loc }, + { + kind: "if", + cond: isTag(1), + then: [undefinedStore], + else_: [ + write("c", { + kind: "ternary", + cond: isTag(0), + then: { kind: "unionNarrow", unionId: "u", tag: 0, value: v, type: F64, loc }, + else_: { + kind: "libCall", + fn: "error.nodeThrow", + args: [num(1), { kind: "strLit", value: "", type: { kind: "string" }, loc }], + type: F64, + loc, + }, + type: F64, + loc, + }), + ], + loc, + }, + ]; + const withV = new Map([ + ...locals, + ["v", { id: "v", name: "v", type: unionType, mutable: false }], + ]); + const matchStores = (body: IrStmt[], initialized: string[]) => + matchDenseArrayLoop(loop(body), shape, withV, new Set(), new Set(initialized)); + const dense = matchStores(guarded("b"), ["a", "b", "c"]); + expect([...dense!.arrays.keys()].sort()).toEqual(["b", "c"]); + // A read of an array the entry test does not cover may miss in the fast + // copy, and its undefined store would run there. + expect(matchStores(guarded("b"), ["a", "c"])).toBeNull(); + // A present undefined stored on its own leaves its array out. + const alone = matchStores( + [{ kind: "exprStmt", expr: read("b"), loc }, undefinedStore], + ["a", "b", "c"], + ); + expect([...alone!.arrays.keys()]).toEqual(["b"]); +}); diff --git a/packages/compiler/src/backend/llvm/dense-array-loops.ts b/packages/compiler/src/backend/llvm/dense-array-loops.ts index 184ff1353..135d5eaf2 100644 --- a/packages/compiler/src/backend/llvm/dense-array-loops.ts +++ b/packages/compiler/src/backend/llvm/dense-array-loops.ts @@ -20,16 +20,24 @@ * A counted loop with another invariant limit, such as `j < n` after * `const n = a.length` or `j < 5`, is covered the same way: the test bounds * the loop's exclusive limit by 2^31 - 1 and by each indexed array's - * `filled`, and the loop must provably never change any of their lengths. */ + * `filled`, and the loop must provably never change any of their lengths. + * + * A two-counter loop closing in on itself, such as + * `for (let i = 1, j = k - 1; i < j; i++, j--)`, indexes only between its + * low counter's start and its high counter's: the low counter only rises + * and stays below the high one, which only falls. Its test bounds the low + * start below by 0 and the high start by 2^31 - 2 and below each indexed + * array's `filled`, both starts being exact integers. */ import type { IrExpr, IrLocal, IrStmt } from "../../ir/ir.js"; import type { IntegerRange } from "../../ir/integer-ranges.js"; import { preservesArrayLength } from "../../ir/array-length-stability.js"; import { everyExprChild, everyStmtChild } from "../../ir/traverse.js"; -import type { IntegerCountedForLoop } from "../../ir/integer-loops.js"; +import type { IntegerCountedForLoop, IntegerPairedForLoop } from "../../ir/integer-loops.js"; import type { LlvmEmitterContext } from "./expr-context.js"; import { loadArrCount, SCR_ARR_FIELD } from "./common.js"; import { emitCountedLoopLimit } from "./counted-loops.js"; +import { f64Lit } from "./common.js"; export const INT32_MAX = 2147483647; @@ -47,6 +55,17 @@ export interface DenseArrayLoop { arrays: ReadonlyMap; /** The counter's range on every pass of the fast copy. */ range: IntegerRange; + /** A two-counter loop's high counter and both starts, when the counter + * is its low one (`length` and `limit` are then null). */ + pair: DensePair | null; +} + +/** The high counter of a two-counter dense loop, and the starts its entry + * test bounds. */ +export interface DensePair { + high: string; + lowStart: IrExpr; + highStart: IrExpr; } /** The fast copy duplicates the loop; keep the duplicated code small. */ @@ -62,6 +81,8 @@ export interface DenseLoopShape { /** The counter's proven range on every pass of the ordinary loop; the * fast copy keeps its minimum, which must be at least 0. */ range: IntegerRange; + /** A two-counter loop's high counter and starts. */ + pair?: DensePair; } /** `length.length - c` (c an integer literal, 0 when absent). */ @@ -107,6 +128,124 @@ export function countedDenseShape( return { counter: counted.localId, length: start.receiver, range: counted.range }; } +/** The shape of a two-counter loop whose low counter rises and high + * counter falls: both stay between the two starts, which the entry test + * bounds, so the fast copy's counters lie in [0, 2^31 - 2]. */ +export function pairedDenseShape(loop: IntegerPairedForLoop): DenseLoopShape { + return { + counter: loop.low.localId, + length: null, + range: { min: 0, max: INT32_MAX - 1 }, + pair: { high: loop.high.localId, lowStart: loop.low.start, highStart: loop.high.start }, + }; +} + +interface ElementRead { + arr: IrExpr; + index: IrExpr; +} + +/** The array and index of a call of an element-reader helper + * (`%arr.idxOr.*`, ir/int32-arrays.ts isElementReader): `a[i]` as + * `number | undefined`. */ +function elementReaderCall(e: IrExpr): ElementRead | null { + if (e.kind !== "call" || !e.callee.startsWith("%arr.idxOr.") || e.args.length !== 2) return null; + return { arr: e.args[0]!, index: e.args[1]! }; +} + +/** The array and index of an element read as `T | undefined`: an + * element-reader call, or its body inlined for an int32 array + * (`arrayState(a, i) === 1 ? wrap(a[i]) : wrap(undefined)`). */ +function elementRead(e: IrExpr): ElementRead | null { + const call = elementReaderCall(e); + if (call) return call; + if (e.kind !== "ternary") return null; + const cond = e.cond; + if (cond.kind !== "bin" || cond.op !== "===") return null; + const state = cond.left, + present = cond.right, + wrapped = e.then, + missing = e.else_; + if (state.kind !== "arrayState" || present.kind !== "numLit" || present.value !== 1) return null; + if (wrapped.kind !== "unionWrap" || missing.kind !== "unionWrap") return null; + const value = wrapped.value; + if (value.kind !== "arrayGet" || missing.value.kind !== "unitLit") return null; + const same = (x: IrExpr, y: IrExpr): boolean => + x.kind === "varRef" && y.kind === "varRef" && x.localId === y.localId; + return same(state.arr, value.arr) && same(state.index, value.index) + ? { arr: state.arr, index: state.index } + : null; +} + +/** The present-undefined stores of `a[i] = b[j]!` as lowering spells it, + * with the element read each one stores the missing arm of: + * + * const v = %arr.idxOr(b, j); + * if (v is undefined) a[i] = undefined; else a[i] = v is number ? v : throw; + * + * The read's local is declared once in the same list as the `if`, and only + * read after it. */ +function missingElementStores(body: IrStmt[]): Map { + const result = new Map(); + const writes = new Map(); + const count = (id: string): void => void writes.set(id, (writes.get(id) ?? 0) + 1); + const expr = (e: IrExpr): boolean => { + if (e.kind === "assignExpr" || e.kind === "incDec") count(e.localId); + return everyExprChild(e, expr, stmt); + }; + const stmt = (s: IrStmt): boolean => { + if (s.kind === "varDecl" || s.kind === "assign") count(s.localId); + return everyStmtChild(s, expr, stmt); + }; + body.every(stmt); + const list = (stmts: IrStmt[]): void => { + const reads = new Map(); + for (const s of stmts) { + if (s.kind === "varDecl" && s.init && writes.get(s.localId) === 1) { + const read = elementRead(s.init); + if (read) reads.set(s.localId, read); + } + if (s.kind === "if") { + const store = missingArmStore(s, reads); + if (store) result.set(store.stmt, store.read); + } + // Nested loops keep a loop out of the fast copy (matchDenseArrayLoop), + // and other statement lists leave their stores unmatched. + if (s.kind === "block") list(s.body); + if (s.kind === "if") { + list(s.then); + if (s.else_) list(s.else_); + } + } + }; + list(body); + return result; +} + +/** The present-undefined store of `if (v is undefined) a[i] = undefined; + * else a[i] = v is number ? v : throw;`, with the read `v` holds. */ +function missingArmStore( + s: IrStmt & { kind: "if" }, + reads: ReadonlyMap, +): { stmt: IrStmt; read: ElementRead } | null { + const cond = s.cond; + if (cond.kind !== "unionIsTag" || cond.negated || cond.value.kind !== "varRef") return null; + const local = cond.value.localId; + const read = reads.get(local); + if (!read || s.then.length !== 1 || s.else_?.length !== 1) return null; + const store = s.then[0]!, + other = s.else_[0]!; + if (store.kind !== "arraySetUndefined" || other.kind !== "arraySet") return null; + const value = other.value; + if (value.kind !== "ternary") return null; + const test = value.cond, + narrowed = value.then; + if (test.kind !== "unionIsTag" || test.negated || test.tag === cond.tag) return null; + if (test.value.kind !== "varRef" || test.value.localId !== local) return null; + if (narrowed.kind !== "unionNarrow" || narrowed.tag !== test.tag) return null; + return narrowed.type.kind === "f64" ? { stmt: store, read } : null; +} + /** Match the fast copy of a loop of the given shape. `initialized` holds the * bindings definitely assigned when the loop starts; `unaliased` the local * arrays no other binding holds. */ @@ -118,6 +257,8 @@ export function matchDenseArrayLoop( initialized: ReadonlySet, ): DenseArrayLoop | null { const { counter, length } = shape; + const high = shape.pair?.high ?? null; + const isCounter = (id: string): boolean => id === counter || id === high; if (shape.range.min < 0 || !loop.cond) return null; let nodes = 0, nested = false; @@ -126,13 +267,18 @@ export function matchDenseArrayLoop( // element write. const disturbed = new Set(); const access = (arr: IrExpr, index: IrExpr): void => { - if (arr.kind === "varRef" && index.kind === "varRef" && index.localId === counter) + if (arr.kind === "varRef" && index.kind === "varRef" && isCounter(index.localId)) candidates.set(arr.localId, arr); }; + const missing = missingElementStores(loop.body); + // The element reads behind the guarded present-undefined stores. + const guards: ElementRead[] = []; const expr = (e: IrExpr): boolean => { if (++nodes > MAX_LOOP_NODES) return false; if (e.kind === "assignExpr" || e.kind === "incDec") disturbed.add(e.localId); if (e.kind === "arrayGet") access(e.arr, e.index); + const reader = elementReaderCall(e); + if (reader) access(reader.arr, reader.index); if (e.kind === "arrIntrinsic") { if (e.method === "getNumber" && e.args[0]) access(e.receiver, e.args[0]); else if (e.method !== "length" && e.receiver.kind === "varRef") @@ -147,9 +293,21 @@ export function matchDenseArrayLoop( if (s.kind === "assign" || s.kind === "varDecl") disturbed.add(s.localId); if (s.kind === "arraySet") { access(s.arr, s.index); - if (s.arr.kind === "varRef" && (s.index.kind !== "varRef" || s.index.localId !== counter)) + if (s.arr.kind === "varRef" && (s.index.kind !== "varRef" || !isCounter(s.index.localId))) disturbed.add(s.arr.localId); } + if (s.kind === "arraySetUndefined") { + // A present undefined at a counter only where it stores a missing + // element read (`a[i] = b[j]!`): the fast copy reads a present value + // there, so the store never runs in it. + const read = missing.get(s); + if (read === undefined || s.index.kind !== "varRef" || !isCounter(s.index.localId)) { + if (s.arr.kind === "varRef") disturbed.add(s.arr.localId); + } else { + access(s.arr, s.index); + guards.push(read); + } + } return everyStmtChild(s, expr, stmt); }; if (!loop.body.every(stmt) || !expr(loop.cond) || nested) return null; @@ -180,18 +338,33 @@ export function matchDenseArrayLoop( // once: the loop must not change any of their lengths. if ( length === null && - !preservesArrayLength(nodesOfLoop, { receiver: id, inRange: counter }, locals, unaliased) + !preservesArrayLength( + nodesOfLoop, + { receiver: id, inRange: counter, inRangeHigh: high }, + locals, + unaliased, + ) ) continue; arrays.set(id, ref); } if (arrays.size === 0) return null; + // Each guarded store's read must be one the fast copy proves present. + for (const { arr, index } of guards) + if ( + arr.kind !== "varRef" || + !arrays.has(arr.localId) || + index.kind !== "varRef" || + !isCounter(index.localId) + ) + return null; return { counter, length, limit: length === null ? (shape.limit ?? null) : null, arrays, range: { min: shape.range.min, max: Math.min(shape.range.max, INT32_MAX - 1) }, + pair: shape.pair ?? null, }; } @@ -207,6 +380,7 @@ export function emitDenseLoopGuard(host: LlvmEmitterContext, loop: DenseArrayLoo host.markMemoryPointer(ptr, "array:header"); return ptr; }; + if (loop.pair !== null) return emitPairedDenseGuard(host, loop, loop.pair, header); if (loop.length === null) { // A limit at or below the start runs no pass, and passes the test. const limit = emitCountedLoopLimit(host, loop.limit!); @@ -254,3 +428,54 @@ export function emitDenseLoopGuard(host: LlvmEmitterContext, loop: DenseArrayLoo } return valid; } + +/** A two-counter loop's entry test: both starts are exact integers, the + * low one at least 0 and the high one at most 2^31 - 2 and below every + * indexed array's hole-free prefix bound. The starts are stable values + * (matchIntegerPairedForLoop), so evaluating them here again is unobservable. */ +function emitPairedDenseGuard( + host: LlvmEmitterContext, + loop: DenseArrayLoop, + pair: DensePair, + header: (array: string, field: number) => string, +): string { + const B = host.B; + const integer = (expr: IrExpr, min: number, max: number): { value: string; ok: string } => { + const start = host.emitExpr(expr); + const low = B.tmp(), + high = B.tmp(), + raw = B.tmp(), + value = B.tmp(), + back = B.tmp(), + exact = B.tmp(), + inRange = B.tmp(), + ok = B.tmp(); + B.line(`${low} = fcmp oge double ${start.name}, ${f64Lit(min)}`); + B.line(`${high} = fcmp ole double ${start.name}, ${f64Lit(max)}`); + B.line(`${inRange} = and i1 ${low}, ${high}`); + B.line(`${raw} = fptosi double ${start.name} to i64`); + B.line(`${value} = freeze i64 ${raw}`); + B.line(`${back} = sitofp i64 ${value} to double`); + B.line(`${exact} = fcmp oeq double ${back}, ${start.name}`); + B.line(`${ok} = and i1 ${inRange}, ${exact}`); + return { value, ok }; + }; + const low = integer(pair.lowStart, 0, INT32_MAX - 1); + const high = integer(pair.highStart, -1, INT32_MAX - 1); + let valid = B.tmp(); + B.line(`${valid} = and i1 ${low.ok}, ${high.ok}`); + for (const ref of loop.arrays.values()) { + const array = host.emitStableReceiver(ref, []); + const filledPtr = header(array.name, SCR_ARR_FIELD.filled), + filled = B.tmp(), + wide = B.tmp(), + covered = B.tmp(), + both = B.tmp(); + B.line(`${filled} = load i32, ptr ${filledPtr}${host.fieldAliasAttachment(filledPtr)}`); + B.line(`${wide} = zext i32 ${filled} to i64`); + B.line(`${covered} = icmp slt i64 ${high.value}, ${wide}`); + B.line(`${both} = and i1 ${valid}, ${covered}`); + valid = both; + } + return valid; +} diff --git a/packages/compiler/src/backend/llvm/emit-statements.ts b/packages/compiler/src/backend/llvm/emit-statements.ts index c2846a9d4..48f7a1d5f 100644 --- a/packages/compiler/src/backend/llvm/emit-statements.ts +++ b/packages/compiler/src/backend/llvm/emit-statements.ts @@ -52,7 +52,12 @@ import { matchIntegerPairedForLoop, } from "../../ir/integer-loops.js"; import { emitByteWindowGuard, matchByteWindow } from "./byte-windows.js"; -import { countedDenseShape, emitDenseLoopGuard, matchDenseArrayLoop } from "./dense-array-loops.js"; +import { + countedDenseShape, + emitDenseLoopGuard, + matchDenseArrayLoop, + pairedDenseShape, +} from "./dense-array-loops.js"; import { emitCountedLoopLimit, emitCountedLoopSafety, @@ -966,7 +971,9 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { } : countedLoop && !countedLoop.guarded && countedLoop.followers.length === 0 ? countedDenseShape(s, countedLoop) - : null; + : pairedLoop + ? pairedDenseShape(pairedLoop) + : null; const dense = denseShape && !window ? matchDenseArrayLoop( @@ -1144,11 +1151,15 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { B.line(`${start} = fptosi double ${value.name} to i64`); B.line(`store i64 ${start}, ptr ${slot}`); pairedSlots.push(slot); + const dense = LOOPS.of(emitter).denseLoop; emitter.integerLoopBindings.set(counter.localId, { slot, type: "i64", signed: true, - range: pairedLoop.range, + range: + dense?.pair && dense.counter === pairedLoop.low.localId + ? dense.range + : pairedLoop.range, }); } } else if (s.init) { diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 050c3fb90..3dc21f356 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -974,8 +974,8 @@ export class LlEmitter { } /** Whether an element access at `index` of `arr` lies in the hole-free - * prefix: the counter of a dense array loop's fast copy indexing one of - * the arrays its entry test covered. */ + * prefix: a counter of a dense array loop's fast copy indexing one of the + * arrays its entry test covered. */ provenDenseIndex(arr: IrExpr, index: IrExpr): boolean { const loop = LOOPS.of(this).denseLoop; return ( @@ -983,8 +983,8 @@ export class LlEmitter { arr.kind === "varRef" && loop.arrays.has(arr.localId) && index.kind === "varRef" && - index.localId === loop.counter && - this.integerLoopBindings.has(loop.counter) + (index.localId === loop.counter || index.localId === loop.pair?.high) && + this.integerLoopBindings.has(index.localId) ); } diff --git a/packages/compiler/src/backend/llvm/expr-primitives.ts b/packages/compiler/src/backend/llvm/expr-primitives.ts index 1eec6a965..2c6ce35db 100644 --- a/packages/compiler/src/backend/llvm/expr-primitives.ts +++ b/packages/compiler/src/backend/llvm/expr-primitives.ts @@ -1076,9 +1076,10 @@ export function emitContainerExpr( const idx = host.emitExpr(e.index); if (e.arr.type.kind !== "array") throw new InternalCompilerError("llvm emitter bug: arrayState on non-array"); + const proven = host.provenDenseIndex(e.arr, e.index); const t = host.checkedInt32Array(e.arr) ? host.emitByElementKind(arr.name, "double", (kind) => - emitDenseArrayState(host, arr.name, idx, e.index, "state", slotWidth(kind)), + emitDenseArrayState(host, arr.name, idx, e.index, "state", slotWidth(kind), proven), )! : emitDenseArrayState( host, @@ -1087,6 +1088,7 @@ export function emitContainerExpr( e.index, "state", host.int32Array(e.arr) ? slotWidth("i32") : denseSlotWidth(host, e.arr.type.elem), + proven, ); return { name: t, type: e.type }; } diff --git a/packages/compiler/src/backend/llvm/local-array-reads.ts b/packages/compiler/src/backend/llvm/local-array-reads.ts index 1c540295a..80ed2c440 100644 --- a/packages/compiler/src/backend/llvm/local-array-reads.ts +++ b/packages/compiler/src/backend/llvm/local-array-reads.ts @@ -797,7 +797,33 @@ export function emitLocalArrayRead( ); return null; }); - else if (inline) + else if ( + inline && + loopIndex !== null && + read.element.kind === "f64" && + host.provenDenseIndex(read.array, read.index) + ) { + // In a dense array loop's fast copy, the entry test proved the counter + // below the array's hole-free prefix: the slot holds a present value, + // and the read has no missing arm. + const dataPtr = B.tmp(), + data = B.tmp(), + valuePtr = B.tmp(), + value = B.tmp(), + width = int32 ? "i32" : "i64"; + B.line( + `${dataPtr} = getelementptr inbounds %ScrArr, ptr ${array.name}, i32 0, i32 ${SCR_ARR_FIELD.data}`, + ); + host.markMemoryPointer(dataPtr, "array:header"); + B.line(`${data} = load ptr, ptr ${dataPtr}${host.fieldAliasAttachment(dataPtr)}`); + B.line( + `${valuePtr} = getelementptr inbounds ${width}, ptr ${data}, ${host.sizeType} ${loopIndex}`, + ); + host.markMemoryPointer(valuePtr, "array:elements"); + B.line(`${value} = load ${width}, ptr ${valuePtr}${host.fieldAliasAttachment(valuePtr)}`); + storeValue(value); + B.br(join); + } else if (inline) emitDenseReferenceArrayRead( host, array, diff --git a/packages/compiler/src/ir/array-length-stability.test.ts b/packages/compiler/src/ir/array-length-stability.test.ts index f9b3cc9c2..97270e059 100644 --- a/packages/compiler/src/ir/array-length-stability.test.ts +++ b/packages/compiler/src/ir/array-length-stability.test.ts @@ -193,3 +193,34 @@ test("number and boolean writes to class fields run no user code", () => { // A reference stored into a field releases the old one. expect(preserves([fieldSet(arr("b"))], "i", new Set(), scope)).toBe(false); }); + +test("optional element reads, their assertion errors and present-undefined stores at an in-range index keep the length", () => { + const optional = (receiver: string, index: IrExpr): IrExpr => ({ + kind: "call", + callee: "%arr.idxOr.f64", + args: [arr(receiver), index], + type: F64, + loc, + }); + const undefinedAt = (receiver: string, index: IrExpr): IrStmt => ({ + kind: "arraySetUndefined", + arr: arr(receiver), + index, + loc, + }); + const thrown: IrExpr = { kind: "libCall", fn: "error.nodeThrow", args: [num(1)], type: F64, loc }; + expect(preserves([stmt(optional("a", ref("i"))), stmt(thrown)])).toBe(true); + expect(preserves([undefinedAt("a", ref("i"))])).toBe(true); + // Past the counter, a present undefined extends the array. + expect(preserves([undefinedAt("a", plus(ref("i"), num(1)))])).toBe(false); + // The high counter of a two-counter loop is in range as well. + const paired = (body: IrStmt[]): boolean => + preservesArrayLength( + [cond, ...body], + { receiver: "a", inRange: "i", inRangeHigh: "j" }, + locals, + new Set(), + ); + expect(paired([undefinedAt("a", ref("j")), set("a", ref("j"), num(0))])).toBe(true); + expect(preserves([set("a", ref("j"), num(0))])).toBe(false); +}); diff --git a/packages/compiler/src/ir/array-length-stability.ts b/packages/compiler/src/ir/array-length-stability.ts index ed57972b6..7715ea1e1 100644 --- a/packages/compiler/src/ir/array-length-stability.ts +++ b/packages/compiler/src/ir/array-length-stability.ts @@ -162,6 +162,10 @@ export interface ArrayLengthLoop { * long as the length itself has not changed). An element write at this * index replaces an existing element or sets a non-index property. */ inRange: string | null; + /** A second such binding: the high counter of a two-counter loop + * (`for (let i = lo, j = hi; i < j; i++, j--)`), whose values stay + * between the low counter's start and its own. */ + inRangeHigh?: string | null; } /** Whether running `nodes` (a loop's condition, body and update) can never @@ -191,7 +195,10 @@ export function preservesArrayLength( // escaping. const receiver = (e: IrExpr): boolean => (e.kind === "varRef" && e.localId === loop.receiver) || expr(e); - const written = (id: string): boolean => id === loop.receiver || id === loop.inRange; + const inRange = (id: string): boolean => + (loop.inRange !== null && id === loop.inRange) || + (loop.inRangeHigh != null && id === loop.inRangeHigh); + const written = (id: string): boolean => id === loop.receiver || inRange(id); const expr = (e: IrExpr): boolean => { switch (e.kind) { case "varRef": @@ -224,12 +231,21 @@ export function preservesArrayLength( case "call": // A checked union extraction (`x!`, coercions/unions.ts): the // asserted arm's payload or a TypeError, and no user code. - return e.callee.startsWith("%union.narrow.") && e.args.every(expr); + if (e.callee.startsWith("%union.narrow.")) return e.args.every(expr); + // An element read as `T | undefined` (int32-arrays.ts isElementReader). + return ( + e.callee.startsWith("%arr.idxOr.") && + e.args.length === 2 && + receiver(e.args[0]!) && + expr(e.args[1]!) + ); case "mapIntrinsic": return READING_MAP_METHODS.has(e.method) && expr(e.receiver) && e.args.every(expr); case "setIntrinsic": return READING_SET_METHODS.has(e.method) && expr(e.receiver) && e.args.every(expr); case "libCall": + // The TypeError of an assertion past a union arm leaves the loop. + if (e.fn === "error.nodeThrow") return e.args.every(expr); return ( isBorrowSafeMath(e.fn) && e.args.every((arg) => arg.type.kind === "f64") && @@ -254,14 +270,18 @@ export function preservesArrayLength( break; case "arraySet": return ( - (separate(s.arr) || - (s.index.kind === "varRef" && - loop.inRange !== null && - s.index.localId === loop.inRange)) && + (separate(s.arr) || (s.index.kind === "varRef" && inRange(s.index.localId))) && receiver(s.arr) && expr(s.index) && expr(s.value) ); + case "arraySetUndefined": + // A present undefined replacing an element below the length. + return ( + (separate(s.arr) || (s.index.kind === "varRef" && inRange(s.index.localId))) && + receiver(s.arr) && + expr(s.index) + ); case "return": return s.value === null || expr(s.value); case "fieldSet": diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7264-array-dense-paired-loops.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7264-array-dense-paired-loops.ts.json new file mode 100644 index 000000000..8396d237c --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7264-array-dense-paired-loops.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7264-array-dense-paired-loops.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7264-array-dense-paired-loops.ts b/tests/corpus/7264-array-dense-paired-loops.ts new file mode 100644 index 000000000..59cedf60b --- /dev/null +++ b/tests/corpus/7264-array-dense-paired-loops.ts @@ -0,0 +1,120 @@ +// Two-counter loops that index number arrays at both counters +// (`for (let i = lo, j = hi; i < j; i++, j--)`), and element copies through +// `a[i]!` reads, both inside and outside the ranges a single entry test can +// cover: filled prefixes, holes, present undefined, reads past the end, +// fractional and negative starts, int32 arrays and long checked arrays. + +function reversePrefix(a: number[], end: number): void { + for (let i = 0, j = end; i < j; i++, j--) { + const t = a[i]!; + a[i] = a[j]!; + a[j] = t; + } +} + +function show(label: string, a: number[]): void { + console.log(label, a.length, JSON.stringify(a), a); +} + +// The whole prefix is present: the entry test passes. +const plain: number[] = []; +for (let k = 0; k < 8; k++) plain[k] = k * 3; +reversePrefix(plain, 5); +show("plain", plain); +reversePrefix(plain, 7); +show("whole", plain); +reversePrefix(plain, 0); +show("empty range", plain); + +// The high start is past the end: the reads there miss, the stores extend. +const short = [1, 2, 3]; +reversePrefix(short, 5); +show("past the end", short); + +// A hole inside the range. +const holey: number[] = [10, 20]; +holey[3] = 40; +holey[4] = 50; +reversePrefix(holey, 4); +show("hole", holey); + +// A present undefined inside the range, copied by a mirror loop. +const loose: number[] = [7, 8, 9, 10]; +(loose as (number | undefined)[])[2] = undefined; +for (let i = 0, j = 3; i < j; i++, j--) loose[i] = loose[j]!; +show("undefined", loose); + +// Fractional and negative starts index non-element properties. +const fractional = [1, 2, 3, 4]; +reversePrefix(fractional, 2.5); +show("fractional", fractional); +const negative = [5, 6, 7]; +for (let i = -1, j = 2; i < j; i++, j--) negative[i] = negative[j]!; +show("negative", negative); + +// A local int32 array (every stored element is a small integer). +function int32Flips(start: number): void { + const counts = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]; + let flips = 0; + for (let first = counts[start]!; first !== 0; first = counts[0]!) { + for (let i = 0, j = first; i < j; i++, j--) { + const t = counts[i]!; + counts[i] = counts[j]!; + counts[j] = t; + } + flips++; + if (flips > 20) break; + } + console.log("int32", flips, counts.join(",")); +} +int32Flips(3); +int32Flips(9); + +// A local long array with checked int32 stores, widened by a fraction. +function longChecked(n: number, fraction: number): void { + const long = new Array(n).fill(0); + for (let k = 0; k < long.length; k++) long[k] = k; + for (let i = 0, j = n - 1; i < j; i++, j--) { + const t = long[i]!; + long[i] = long[j]!; + long[j] = t; + } + console.log("long", long[0], long[1], long[n - 2], long[n - 1]); + long[10] = fraction; + for (let i = 0, j = 20; i < j; i++, j--) { + const t = long[i]!; + long[i] = long[j]!; + long[j] = t; + } + console.log("widened", long.slice(0, 12).join(","), long[20]); +} +longChecked(5000, 0.5); + +// Element copies through `a[i]!`: the source is shorter than the range. +function copy(to: number[], from: number[], n: number): void { + for (let i = 0; i < n; i++) to[i] = from[i]!; +} +const into = [0, 0, 0, 0, 0, 0]; +copy(into, [1, 2, 3, 4, 5, 6], 6); +show("copy", into); +copy(into, [9, 8], 4); +show("short source", into); +const gappy: number[] = [1]; +gappy[2] = 3; +gappy[3] = 4; +copy(into, gappy, 4); +show("gappy source", into); + +// Two names for one array. +function aliased(n: number): void { + const same: number[] = []; + for (let k = 0; k < n; k++) same[k] = k + 1; + const alias = same; + for (let i = 0, j = n - 1; i < j; i++, j--) { + const t = alias[i]!; + same[i] = alias[j]!; + same[j] = t; + } + show("alias", same); +} +aliased(5); diff --git a/tests/harness/corpus-costs/7264-array-dense-paired-loops.ts.json b/tests/harness/corpus-costs/7264-array-dense-paired-loops.ts.json new file mode 100644 index 000000000..f591a71f3 --- /dev/null +++ b/tests/harness/corpus-costs/7264-array-dense-paired-loops.ts.json @@ -0,0 +1 @@ +{ "plain": 2900, "san": 4200 } From 5c015be6be05670a87b57d3a63b888f6f24bbae0 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 07:47:26 -0500 Subject: [PATCH 18/53] Try wider record arms first when a dyn value converts to a union Records are width-tolerant, and a union's arms were tried in canonical order, which sorts record arms by shape id. When a `{}` arm, or an arm whose fields are all optional, came first, it took every object, so a value converted to the union lost the fields only a wider arm has: const pairs = [["a", { publicKeyEncoding: 5 }], ["b", {}]]; for (const [, v] of pairs) console.log(Object.keys(v)); printed [] twice, where Node prints [ 'publicKeyEncoding' ] and []. A checked cast to `{ p: number; q: string } | { p?: number }` dropped `q` the same way. The checked conversion now tries record arms with more required fields first, then those with more fields, and keeps the canonical order otherwise. The union's tags are unchanged. --- packages/compiler/src/backend/llvm/dyn.ts | 39 +++++++++++++++++-- packages/compiler/src/ir/ir.ts | 6 ++- .../7447-union-record-arm-width.mjs.json | 6 +++ .../7448-union-record-arm-width.ts.json | 6 +++ tests/corpus/7447-union-record-arm-width.mjs | 37 ++++++++++++++++++ tests/corpus/7448-union-record-arm-width.ts | 21 ++++++++++ .../7447-union-record-arm-width.mjs.json | 1 + .../7448-union-record-arm-width.ts.json | 1 + 8 files changed, 111 insertions(+), 6 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7447-union-record-arm-width.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7448-union-record-arm-width.ts.json create mode 100644 tests/corpus/7447-union-record-arm-width.mjs create mode 100644 tests/corpus/7448-union-record-arm-width.ts create mode 100644 tests/harness/corpus-costs/7447-union-record-arm-width.mjs.json create mode 100644 tests/harness/corpus-costs/7448-union-record-arm-width.ts.json diff --git a/packages/compiler/src/backend/llvm/dyn.ts b/packages/compiler/src/backend/llvm/dyn.ts index 94581cfa0..86626d304 100644 --- a/packages/compiler/src/backend/llvm/dyn.ts +++ b/packages/compiler/src/backend/llvm/dyn.ts @@ -370,6 +370,36 @@ export class LlDyn { B.line(`call void @scr_dyn_build_add(ptr %d, ptr @${name}, ptr ${value})`); } + /** The order a dyn value tries a union's arms in: the canonical order, + * except that among the record arms, those with more required fields go + * first, then those with more fields. Records are width-tolerant, so an + * arm with fewer fields (`{}`, or one whose fields are all optional, + * matches every object) would otherwise take an object that a wider arm + * also matches and drop the fields only the wider arm has. */ + private unionArmOrder(def: IrUnionDef): number[] { + const width = (i: number): [number, number] | null => { + const arm = def.arms[i]!; + if (arm.kind !== "record") return null; + const shape = this.host.recordsById.get(arm.shapeId); + if (!shape || shape.tuple) return null; + const required = shape.fields.filter( + (field) => + field.type.kind !== "union" || + !this.host.unionsById.get(field.type.unionId)?.arms.some((x) => x.kind === "undefinedT"), + ).length; + return [required, shape.fields.length]; + }; + const order = def.arms.map((_, i) => i); + const records = order.filter((i) => width(i) !== null); + const widest = [...records].sort((x, y) => { + const [rx, fx] = width(x)!; + const [ry, fy] = width(y)!; + return ry - rx || fy - fx || x - y; + }); + records.forEach((slot, k) => (order[slot] = widest[k]!)); + return order; + } + /** Branch to a variant only after its literal discriminator and complete * structure match. Use the same path for predicates and builders. */ private unionArmMatch( @@ -2368,9 +2398,10 @@ export class LlDyn { throw new InternalCompilerError( `llvm emitter bug: dynCheck of unknown union ${t.unionId}`, ); - // Arms in CANONICAL order, first FULL match wins. The matched - // arm's builder can no longer fail. - def.arms.forEach((arm, i) => { + // Arms in unionArmOrder, first FULL match wins. The matched arm's + // builder can no longer fail. + for (const i of this.unionArmOrder(def)) { + const arm = def.arms[i]!; const lHit = B.newLabel("dcu.h"); const lNext = B.newLabel("dcu.n"); this.unionArmMatch(B, def, i, lHit, lNext, preserveRefs); @@ -2414,7 +2445,7 @@ export class LlDyn { B.terminate(`ret ptr ${u}`); } B.startBlock(lNext); - }); + } if (preserveRefs) { B.line( `call void @scr_dyn_check_fail(ptr %path, ptr ${host.cstr("reference with its original layout")}, ptr %d)`, diff --git a/packages/compiler/src/ir/ir.ts b/packages/compiler/src/ir/ir.ts index 6d0760b81..878088d65 100644 --- a/packages/compiler/src/ir/ir.ts +++ b/packages/compiler/src/ir/ir.ts @@ -3898,8 +3898,10 @@ export type IrExpr = * not shape equality; missing or wrong-typed fields throw, EXCEPT that a * missing key for an undefined-armed union field — an optional field — * builds the interned undefined arm instead); arrays check every element; - * unions try arms in canonical order and the first FULL match wins (no - * match → throw; an undefined arm matches no dyn value); JSON null matches + * unions try arms in canonical order, except that record arms with more + * required fields and then more fields go first, and the first FULL match + * wins (no match → throw; an undefined arm matches no dyn value); JSON + * null matches * exactly the nullT arm of a union target (bare null targets cannot * exist). MAY THROW: * backends' may-throw analyses must treat it like a `throw` statement. diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7447-union-record-arm-width.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7447-union-record-arm-width.mjs.json new file mode 100644 index 000000000..68bd750d9 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7447-union-record-arm-width.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7447-union-record-arm-width.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7448-union-record-arm-width.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7448-union-record-arm-width.ts.json new file mode 100644 index 000000000..498ee884a --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7448-union-record-arm-width.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7448-union-record-arm-width.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7447-union-record-arm-width.mjs b/tests/corpus/7447-union-record-arm-width.mjs new file mode 100644 index 000000000..dc12f8a59 --- /dev/null +++ b/tests/corpus/7447-union-record-arm-width.mjs @@ -0,0 +1,37 @@ +// Values read back from arrays whose elements mix object literals of +// different widths, an empty object among them, keep every field: the +// record arm with more fields is tried before `{}`, which matches every +// object. +const pairs = [ + ["a", { publicKeyEncoding: 5 }], + ["b", {}], + ["c", { publicKeyEncoding: 6, extra: true }], +]; +for (const [label, value] of pairs) console.log(label, Object.keys(value), JSON.stringify(value)); + +const options = [ + ["rsa", { modulusLength: 1024 }], + ["ec", { namedCurve: "P-256", paramEncoding: "named" }], + ["none", {}], +]; +for (const [type, value] of options) { + const { modulusLength, namedCurve, paramEncoding } = value; + console.log(type, modulusLength, namedCurve, paramEncoding); +} + +const mixed = [{ a: 1 }, {}, "s", { a: 2, b: "two" }]; +for (const value of mixed) { + console.log(typeof value, JSON.stringify(value), typeof value === "object" ? Object.keys(value).length : -1); +} + +function describe(value) { + return Object.entries(value) + .map(([key, field]) => `${key}=${field}`) + .join(";"); +} +const rows = [ + [1, { x: 1, y: 2 }], + [2, {}], + [3, { x: 3 }], +]; +for (const [id, value] of rows) console.log(id, describe(value), value.x, value.y); diff --git a/tests/corpus/7448-union-record-arm-width.ts b/tests/corpus/7448-union-record-arm-width.ts new file mode 100644 index 000000000..1518dd575 --- /dev/null +++ b/tests/corpus/7448-union-record-arm-width.ts @@ -0,0 +1,21 @@ +// A checked cast to a union of record types keeps every field of the +// value: arms with more required fields, then more fields, are tried +// before an arm whose fields are all optional, which matches every object. +type Full = { p: number; q: string }; +type Bare = { p?: number }; +type Other = { r: boolean }; + +const values = JSON.parse('[{"p":5,"q":"x"},{},{"r":true},{"p":7}]') as (Full | Bare | Other)[]; +for (const value of values) { + const q = "q" in value ? value.q : "-"; + const r = "r" in value ? value.r : "-"; + console.log(JSON.stringify(value), "q" in value, "r" in value, q, r); +} + +const one = JSON.parse('{"p":1,"q":"y"}') as Bare | Full; +console.log(JSON.stringify(one), "q" in one ? one.q : "no q"); + +type Point = { x: number; y?: number }; +type Labeled = { x: number; label: string }; +const points = JSON.parse('[{"x":1,"label":"a"},{"x":2,"y":3},{"x":4}]') as (Point | Labeled)[]; +for (const point of points) console.log(point.x, "label" in point ? point.label : "none", "y" in point ? point.y : "none"); diff --git a/tests/harness/corpus-costs/7447-union-record-arm-width.mjs.json b/tests/harness/corpus-costs/7447-union-record-arm-width.mjs.json new file mode 100644 index 000000000..5648bd255 --- /dev/null +++ b/tests/harness/corpus-costs/7447-union-record-arm-width.mjs.json @@ -0,0 +1 @@ +{ "plain": 5785, "san": 5722 } diff --git a/tests/harness/corpus-costs/7448-union-record-arm-width.ts.json b/tests/harness/corpus-costs/7448-union-record-arm-width.ts.json new file mode 100644 index 000000000..c60a813c2 --- /dev/null +++ b/tests/harness/corpus-costs/7448-union-record-arm-width.ts.json @@ -0,0 +1 @@ +{ "plain": 3204, "san": 4052 } From 8eedab9fdeb35a8221dbe36db51bce22e87afa6d Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 08:41:40 -0500 Subject: [PATCH 19/53] Cache untyped property access in programs that declare classes A program that declares a class reads and writes untyped properties through helpers that first select the receiver's property bag. The bag helpers return jsvals, so the helpers read the bag through js.toDyn; keyed helpers test each class's key table between the key's coercion and the read; and a program with subclasses normalizes the receiver (value = %dyn.class.normalize(value)) at the start of each helper. The inline-cache recognizer accepted none of these forms, so every property site in such a program called its helper. It now looks through dyn/jsval conversions of the receiver, accepts capsule branches before a keyed helper's final read, and accepts a rebinding of the receiver to a bag of itself. Plain objects and arrays then read and write through the site's cache, as they do in programs without classes. bench/dynamic-properties (new) exercises these sites. --- benchmarks/runtime/README.md | 1 + .../runtime/workloads/dynamic-properties.json | 6 + .../runtime/workloads/dynamic-properties.mjs | 56 ++++++++ packages/compiler/src/ir/js-ic.test.ts | 134 ++++++++++++++++++ packages/compiler/src/ir/js-ic.ts | 33 ++++- packages/compiler/test/js-ic-emission.test.ts | 44 ++++++ scripts/zero-cost-compare.mjs | 5 +- 7 files changed, 271 insertions(+), 8 deletions(-) create mode 100644 benchmarks/runtime/workloads/dynamic-properties.json create mode 100644 benchmarks/runtime/workloads/dynamic-properties.mjs diff --git a/benchmarks/runtime/README.md b/benchmarks/runtime/README.md index 4f152e4bb..04c90f7bb 100644 --- a/benchmarks/runtime/README.md +++ b/benchmarks/runtime/README.md @@ -13,6 +13,7 @@ Application-shaped workloads for measuring the speed of compiled executables. Ea | `cyclic-graphs` | cpu | object graphs with back references built and dropped: trees with parent and sibling links, rings of records sharing a long-lived registry, closures capturing their owner | | `dictionary-walks` | cpu | `Record` dictionaries walked with for-in and `Object.keys`/`values`/`entries` while entries are added, updated and deleted | | `dynamic-methods` | cpu | untyped JavaScript calling builtin methods on values it does not type: string `slice`/`indexOf`/`toUpperCase`/`padStart`, array `push`/`pop`/`includes`, `Map` and `Set`, and an object's own methods | +| `dynamic-properties` | cpu | untyped JavaScript reading and writing properties of values it does not type: records built by object literals in two key orders, nested records, and instances of a class and its subclass | | `functional-pipeline` | cpu | `filter`/`map`/`reduce`/`flatMap` chains, closures, small records | | `intern-table`, `intern-table-map`, `intern-table-1`, `intern-table-map-1` | cpu | worker threads interning the same type-like objects under string keys and records under number keys, then hits and misses: one shared `InternTable` per kind of key, or a module-level `Map` per thread (the `-map` entries), with 8 threads or 1 (the `-1` entries); under Node.js every thread has its own tables | | `json-records` | cpu | `JSON.stringify`, typed `JSON.parse`, Map aggregation | diff --git a/benchmarks/runtime/workloads/dynamic-properties.json b/benchmarks/runtime/workloads/dynamic-properties.json new file mode 100644 index 000000000..398981ef3 --- /dev/null +++ b/benchmarks/runtime/workloads/dynamic-properties.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/dynamic-properties.mjs", + "args": [ + "1000000" + ] +} diff --git a/benchmarks/runtime/workloads/dynamic-properties.mjs b/benchmarks/runtime/workloads/dynamic-properties.mjs new file mode 100644 index 000000000..d0ee3802c --- /dev/null +++ b/benchmarks/runtime/workloads/dynamic-properties.mjs @@ -0,0 +1,56 @@ +// Untyped JavaScript reading and writing properties of values it never +// declares types for: records built by object literals in two key orders, +// nested records, property updates, and instances of a class and its +// subclass read and written the same way. Argument: the number of +// iterations. +const rounds = Number(process.argv[2] ?? "1000000"); +class Cursor { + constructor(x, y) { + this.x = x; + this.y = y; + } + step(dx, dy) { + this.x += dx; + this.y += dy; + return this; + } +} +class Marker extends Cursor { + constructor(x, y, label) { + super(x, y); + this.label = label; + } +} +function makeItem(i) { + if (i % 3 === 0) + return { id: i, price: i % 17, qty: i % 5, meta: { tag: i % 7, weight: i % 11 } }; + return { id: i, qty: i % 5, price: i % 13, meta: { weight: i % 9, tag: i % 5 } }; +} +function cost(item) { + return item.price * item.qty + item.meta.weight; +} +function restock(item, round) { + item.qty = (item.qty + round) % 7; + item.meta.tag = (item.meta.tag + 1) % 7; +} +function distance(a, b) { + return Math.abs(a.x - b.x) + Math.abs(a.y - b.y); +} +const items = []; +for (let i = 0; i < 1024; i++) items.push(makeItem(i)); +const cursors = []; +for (let i = 0; i < 64; i++) + cursors.push(i % 2 === 0 ? new Cursor(i - 32, 7 - i) : new Marker(i - 32, 7 - i, "m" + i)); +const origin = { x: 0, y: 0 }; +let checksum = 0; +for (let r = 0; r < rounds; r++) { + const item = items[r & 1023]; + checksum += cost(item); + restock(item, r & 3); + checksum += item.meta.tag; + const cursor = cursors[r & 63]; + checksum += distance(cursor.step((r % 3) - 1, (r % 5) - 2), origin); + if (cursor.x > 100 || cursor.x < -100) cursor.x = 0; + if (cursor.y > 100 || cursor.y < -100) cursor.y = 0; +} +console.log(checksum, items[5].qty, items[6].meta.tag, cursors[3].x, cursors[3].y, cursors[3].label); diff --git a/packages/compiler/src/ir/js-ic.test.ts b/packages/compiler/src/ir/js-ic.test.ts index f71a1e9db..026600cc3 100644 --- a/packages/compiler/src/ir/js-ic.test.ts +++ b/packages/compiler/src/ir/js-ic.test.ts @@ -126,6 +126,140 @@ test("property helpers whose class branches only test capsules are cache sites", expect(sites.site("%dyn.class.property.1")).toEqual({ kind: "set", key: "y" }); }); +test("bags converted between dyn and jsval are still the receiver", () => { + // A program that declares a class returns its bags as jsvals, and the + // helpers read them through js.toDyn. + const toDyn = (value: IrExpr): IrExpr => ({ + kind: "libCall", + fn: "js.toDyn", + args: [value], + type: DYN, + loc, + }); + const fromDyn = (value: IrExpr): IrExpr => ({ + kind: "libCall", + fn: "js.fromDyn", + args: [value], + type: { kind: "jsval" }, + loc, + }); + const jsvalBag = helper( + "%dyn.class.readPropertiesOrEmpty.js", + ["v"], + [capsuleBranch("v"), ret(fromDyn(ref("v")))], + ); + const staticBag = helper("%dyn.class.staticProperties", ["v"], [ret(ref("v"))]); + const getter = getHelper( + "%dyn.class.property.0", + "x", + [capsuleBranch("p.0")], + call(staticBag.name, toDyn(call(jsvalBag.name, ref("p.0")))), + ); + const setter = setHelper("%dyn.class.property.1", "y"); + const write = setter.body[0] as Extract; + const keySet = write.expr as Extract; + keySet.args[0] = toDyn(call(jsvalBag.name, ref("p.0"))); + // A converted value that is not the receiver stays a helper call. + const foreign = getHelper( + "%dyn.class.property.2", + "x", + [], + toDyn(fromDyn({ kind: "dynObjLit", type: DYN, loc } as IrExpr)), + ); + const sites = sitesOf(jsvalBag, staticBag, getter, setter, foreign); + expect(sites.site("%dyn.class.property.0")).toEqual({ kind: "get", key: "x" }); + expect(sites.site("%dyn.class.property.1")).toEqual({ kind: "set", key: "y" }); + expect(sites.site("%dyn.class.property.2")).toBeNull(); +}); + +test("a bag helper may first rebind its parameter to a bag of it", () => { + // Programs with subclasses normalize the value before the class branches. + const normalize = helper("%dyn.class.normalize", ["v"], [capsuleBranch("v"), ret(ref("v"))]); + const rebind = (value: IrExpr): IrStmt => ({ kind: "assign", localId: "v", value, loc }); + const normalized = helper( + "%dyn.class.readPropertiesOrEmpty.normalized", + ["v"], + [rebind(call(normalize.name, ref("v"))), capsuleBranch("v"), ret(ref("v"))], + ); + const replaced = helper( + "%dyn.class.readPropertiesOrEmpty.replaced", + ["v"], + [rebind({ kind: "dynObjLit", type: DYN, loc } as IrExpr), ret(ref("v"))], + ); + const sites = sitesOf( + normalize, + normalized, + replaced, + getHelper("%dyn.class.property.0", "x", [], call(normalized.name, ref("p.0"))), + getHelper("%dyn.class.property.1", "x", [], call(replaced.name, ref("p.0"))), + // The property helper itself normalizes its receiver first, too. + getHelper("%dyn.class.property.2", "x", [ + { kind: "assign", localId: "p.0", value: call(normalize.name, ref("p.0")), loc }, + capsuleBranch("p.0"), + ]), + ); + expect(sites.site("%dyn.class.property.0")).toEqual({ kind: "get", key: "x" }); + expect(sites.site("%dyn.class.property.1")).toBeNull(); + expect(sites.site("%dyn.class.property.2")).toEqual({ kind: "get", key: "x" }); +}); + +test("keyed helpers may test capsules between the key's coercion and the read", () => { + const read = (object: IrExpr, key: string): IrExpr => ({ + kind: "libCall", + fn: "js.fromDyn", + args: [{ kind: "dynKeyGet", value: object, key: ref(key), type: DYN, loc }], + type: { kind: "jsval" }, + loc, + }); + const guard = (test: "symbol" | "nullish", tested: string, object: IrExpr): IrStmt => ({ + kind: "if", + cond: { kind: "dynTest", test, value: ref(tested), type: BOOL, loc }, + then: [ret(read(object, "p.key"))], + else_: null, + loc, + }); + const keyed = (name: string, between: IrStmt[]): IrFunction => + helper( + name, + ["p.0", "p.key"], + [ + capsuleBranch("p.0"), + guard("symbol", "p.key", call(identityBag.name, ref("p.0"))), + guard("nullish", "p.0", ref("p.0")), + { + kind: "varDecl", + localId: "key.string", + init: { + kind: "libCall", + fn: "dyn.toStringCoerce", + args: [ref("p.key")], + type: STRING, + loc, + }, + loc, + }, + ...between, + ret(read(call(identityBag.name, ref("p.0")), "key.string")), + ], + ["key.string"], + ); + const errorBranch: IrStmt = { + kind: "if", + cond: { kind: "dynTest", test: "error", value: ref("p.0"), type: BOOL, loc }, + then: [ret(ref("p.0"))], + else_: null, + loc, + }; + const sites = sitesOf( + keyed("%dyn.class.computed.0", []), + keyed("%dyn.class.computed.1", [capsuleBranch("p.0"), capsuleBranch("p.0", "dyn.classIs")]), + keyed("%dyn.class.computed.2", [errorBranch]), + ); + expect(sites.site("%dyn.class.computed.0")).toEqual({ kind: "keyed" }); + expect(sites.site("%dyn.class.computed.1")).toEqual({ kind: "keyed" }); + expect(sites.site("%dyn.class.computed.2")).toBeNull(); +}); + test("any other prefix or tail keeps the helper call", () => { const errorBranch: IrStmt = { kind: "if", diff --git a/packages/compiler/src/ir/js-ic.ts b/packages/compiler/src/ir/js-ic.ts index 752316db3..58c568bbd 100644 --- a/packages/compiler/src/ir/js-ic.ts +++ b/packages/compiler/src/ir/js-ic.ts @@ -62,7 +62,11 @@ export class JsIcSites { const receiver = fn.params[0]?.localId; if (receiver === undefined) return null; let i = 0; - while (i < fn.body.length && this.capsuleBranch(fn.body[i]!, receiver)) i++; + while ( + i < fn.body.length && + (this.capsuleBranch(fn.body[i]!, receiver) || this.rebinding(fn.body[i]!, receiver)) + ) + i++; const tail = fn.body.slice(i); if (fn.params.length === 1) return this.getTail(tail, receiver); if (fn.params.length !== 2) return null; @@ -109,10 +113,14 @@ export class JsIcSites { /** `if (typeof key === "symbol") return dynKeyGet(BAG(receiver), key); * if (receiver == null) return dynKeyGet(receiver, key); * const s = String(key); return dynKeyGet(BAG(receiver), s)`, each read - * possibly under js.fromDyn. */ + * possibly under js.fromDyn. Capsule branches may precede the final read: + * a program that declares classes looks the string key up in each class's + * key table there. */ private keyedTail(tail: IrStmt[], receiver: string, key: string): JsIcSite | null { - const [symbol, nullish, decl, ret] = tail; - if (tail.length !== 4) return null; + const [symbol, nullish, decl] = tail; + const ret = tail[tail.length - 1]; + if (tail.length < 4) return null; + if (!tail.slice(3, -1).every((s) => this.capsuleBranch(s, receiver))) return null; if (!this.guard(symbol, "symbol", key, (read) => this.bagOf(read, receiver), key)) return null; if (!this.guard(nullish, "nullish", receiver, (read) => isVar(read, receiver), key)) return null; @@ -159,9 +167,12 @@ export class JsIcSites { } /** Whether `e` is the receiver itself for every ordinary object or array: - * the receiver, or a chain of property-bag helpers applied to it. */ + * the receiver, or a chain of property-bag helpers applied to it. A + * conversion between the dyn and jsval forms keeps the object it holds. */ private bagOf(e: IrExpr, receiver: string): boolean { if (isVar(e, receiver)) return true; + if (e.kind === "libCall" && (e.fn === "js.toDyn" || e.fn === "js.fromDyn")) + return e.args.length === 1 && this.bagOf(e.args[0]!, receiver); return ( e.kind === "call" && e.args.length === 1 && @@ -182,6 +193,13 @@ export class JsIcSites { return ret.kind === "return" && ret.value !== null && this.bagOf(ret.value, param); } + /** `param = `, such as the subclass normalization + * `value = %dyn.class.normalize(value)`, which leaves an ordinary object + * or array in `param`. */ + private rebinding(s: IrStmt, param: string): boolean { + return s.kind === "assign" && s.localId === param && this.bagOf(s.value, param); + } + /** A property-bag helper that returns its argument unless a capsule * branch is taken. */ private identityHelper(name: string): boolean { @@ -204,7 +222,10 @@ export class JsIcSites { identity = fn.body .slice(0, -1) - .every((s) => this.capsuleBranch(s, param) || this.kindExit(s, param)) && + .every( + (s) => + this.capsuleBranch(s, param) || this.kindExit(s, param) || this.rebinding(s, param), + ) && last?.kind === "return" && last.value !== null && this.bagOf(last.value, param); diff --git a/packages/compiler/test/js-ic-emission.test.ts b/packages/compiler/test/js-ic-emission.test.ts index 47d641bff..54ea13d26 100644 --- a/packages/compiler/test/js-ic-emission.test.ts +++ b/packages/compiler/test/js-ic-emission.test.ts @@ -64,6 +64,50 @@ console.log(read(o), at(dyn([5, 6]), 1), call(o), o.n); } }); +test("property sites keep their caches in programs that declare classes and subclasses", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-js-ic-class-")); + try { + const entry = join(dir, "main.js"); + const outPath = join(dir, "main.ir.json"); + // Declared classes give every property helper class branches and + // property-bag lookups, and a subclass adds a normalization step; plain + // objects still read through the caches. + await writeFile( + entry, + ` +class Point { + constructor(x) { this.x = x; } +} +class Tagged extends Point { + constructor(x, tag) { super(x); this.tag = tag; } +} +function dyn(v) { return v; } +function read(o) { return o.x; } +function write(o, v) { o.x = v; } +function at(a, i) { return a[i]; } +var o = dyn({ x: 1 }); +var p = dyn(new Point(4)); +var t = dyn(new Tagged(6, "t")); +write(o, 3); +write(p, 5); +write(t, 7); +console.log(read(o), read(p), read(t), at(dyn([5, 6]), 1), at(p, "x"), at(t, "tag")); +`, + ); + const result = await compile(entry, { outDir: dir, outPath, outputKind: "ir" }); + if (!result.ok) + throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); + const mod = deserializeModule(await readFile(outPath, "utf8")); + expect(validateModule(mod)).toEqual([]); + const llvm = emitLlvmModule(mod); + expect(body(llvm, "read")).toContain("@scr_js_load_probe("); + expect(body(llvm, "write")).toContain("@scr_js_store_probe("); + expect(body(llvm, "at")).toContain("@scr_js_keyed_probe("); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + test("typed programs have no cache sites and keep the ordinary runtime", async () => { const dir = await mkdtemp(join(tmpdir(), "scriptc-js-ic-typed-")); try { diff --git a/scripts/zero-cost-compare.mjs b/scripts/zero-cost-compare.mjs index 160e33438..f202c2ab7 100644 --- a/scripts/zero-cost-compare.mjs +++ b/scripts/zero-cost-compare.mjs @@ -394,8 +394,8 @@ export function dynamicTierSymbols(names) { * not yet traced to its typed root. bench/thread-slots, * bench/channel-messages and bench/worker-messages, added later, link them * through worker messages: - * node:worker_threads values are dyn objects. bench/dynamic-methods is - * untyped JavaScript. Every other bench workload is fully typed and must link none. The gate and + * node:worker_threads values are dyn objects. bench/dynamic-methods and + * bench/dynamic-properties are untyped JavaScript. Every other bench workload is fully typed and must link none. The gate and * tests/harness/bench-dynamic-tier.test.ts enforce that, and the test also * fails when a listed workload stops linking them, so remove a workload here * when a leak fix makes its binary clean. */ @@ -410,6 +410,7 @@ export const DYNAMIC_WORKLOADS = new Set([ "bench/thread-slots", "bench/channel-messages", "bench/dynamic-methods", + "bench/dynamic-properties", "bench/worker-messages", "bench/typed-array-scan-workers", "bench/log-summary", From 17c069667e8508e945639dd1f49415efddd81337 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 09:03:59 -0500 Subject: [PATCH 20/53] Settle a var declared by a for-in or for-of loop only inside the loop body A for-in or for-of loop stores its var binding only when it iterates. The early-read analysis counted the loop as an unconditional store for every later statement, so after a loop that ran zero times a closure read the binding as initialized (typeof answered "string" where Node answers "undefined"). The loop now settles the reads in its body; later reads settle at the next unconditional store. --- .../src/frontend/lowering/var-hoisting.ts | 27 ++++++++++++------- ...7553-var-for-in-of-zero-iterations.js.json | 6 +++++ .../7553-var-for-in-of-zero-iterations.js | 18 +++++++++++++ ...7553-var-for-in-of-zero-iterations.js.json | 1 + 4 files changed, 43 insertions(+), 9 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7553-var-for-in-of-zero-iterations.js.json create mode 100644 tests/corpus/7553-var-for-in-of-zero-iterations.js create mode 100644 tests/harness/corpus-costs/7553-var-for-in-of-zero-iterations.js.json diff --git a/packages/compiler/src/frontend/lowering/var-hoisting.ts b/packages/compiler/src/frontend/lowering/var-hoisting.ts index 1a60e7fda..0b415bb3d 100644 --- a/packages/compiler/src/frontend/lowering/var-hoisting.ts +++ b/packages/compiler/src/frontend/lowering/var-hoisting.ts @@ -6,9 +6,11 @@ * * A binding settles at the first statement of its hoisting scope (the * function body or the source file) that stores it unconditionally: a - * direct `var x = e`, `x = e;`, or a loop declaring it (`for (var i = 0; - * ...)`, `for (var k in o)`), whose own condition, update and body run - * after the store. A read settles when it runs after that statement: + * direct `var x = e`, `x = e;`, or a `for` loop declaring it (`for (var + * i = 0; ...)`), whose own condition, update and body run after the store. + * A `for (var k in o)` or `for (var v of xs)` loop stores the binding only + * when it iterates, so it settles the reads in its body and no others. A + * read settles when it runs after the settling statement: * - a read in the scope's own code, in a TypeScript file: tsc's * definite-assignment analysis rejects any earlier one; * - a read in the scope's own code, in a JavaScript file: its statement @@ -201,6 +203,9 @@ interface Settle { index: number; list: ts.VariableDeclarationList | null; end: number; + /** The first for-in/of loop declaring the binding before `index`, whose + * body runs after its store, or -1. */ + loop: number; } class ScopeAnalysis { @@ -229,6 +234,7 @@ class ScopeAnalysis { settleIndex(scope: Scope, symbol: ts.Symbol): Settle { const statements = statementsOf(scope); const checker = this.lowerer.checker; + let loopIndex = -1; for (let i = 0; i < statements.length; i++) { const s = statements[i]!; const list = ts.isVariableStatement(s) @@ -244,8 +250,10 @@ class ScopeAnalysis { if ( (d.initializer !== undefined || loop) && boundIdentifiersOf(d.name).some((n) => checker.getSymbolAtLocation(n) === symbol) - ) - return { index: i, list: loop ? null : list, end: d.end }; + ) { + if (!loop) return { index: i, list, end: d.end, loop: loopIndex }; + if (loopIndex < 0) loopIndex = i; + } } } else if ( ts.isExpressionStatement(s) && @@ -254,10 +262,10 @@ class ScopeAnalysis { ts.isIdentifier(s.expression.left) && checker.getSymbolAtLocation(s.expression.left) === symbol ) { - return { index: i, list: null, end: -1 }; + return { index: i, list: null, end: -1, loop: loopIndex }; } } - return { index: Infinity, list: null, end: -1 }; + return { index: Infinity, list: null, end: -1, loop: loopIndex }; } /** Whether `ref` (inside `scope`, or in another module) runs only after @@ -290,12 +298,13 @@ class ScopeAnalysis { (index === settle.index && (settlesWithin(top, ref) || // A later declarator of the storing list (`var a = {}, b = { ...a }`). - (settle.list !== null && isInside(ref, settle.list) && ref.pos >= settle.end))) + (settle.list !== null && isInside(ref, settle.list) && ref.pos >= settle.end))) || + (index === settle.loop && settlesWithin(top, ref)) ); } private functionSettles(scope: Scope, settle: Settle, fn: ts.FunctionDeclaration): boolean { - const key = `${fn.pos}:${settle.index}:${settle.end}`; + const key = `${fn.pos}:${settle.index}:${settle.end}:${settle.loop}`; const known = this.functionMemo.get(key); if (known !== undefined) return known; this.functionMemo.set(key, true); diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7553-var-for-in-of-zero-iterations.js.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7553-var-for-in-of-zero-iterations.js.json new file mode 100644 index 000000000..9f470ca37 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7553-var-for-in-of-zero-iterations.js.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7553-var-for-in-of-zero-iterations.js" + ], + "diags": [] +} diff --git a/tests/corpus/7553-var-for-in-of-zero-iterations.js b/tests/corpus/7553-var-for-in-of-zero-iterations.js new file mode 100644 index 000000000..1a439f715 --- /dev/null +++ b/tests/corpus/7553-var-for-in-of-zero-iterations.js @@ -0,0 +1,18 @@ +// A for-in or for-of loop stores its var binding only when it iterates: +// after a loop that ran zero times, closures read undefined. +function f(o) { + for (var key in o) {} + const read = () => typeof key; + const same = () => key === undefined; + return [read(), same()]; +} +console.log(JSON.stringify(f({})), JSON.stringify(f({ a: 1 }))); +function g(xs) { + for (var x of xs) {} + const show = () => [typeof x, String(x)]; + return show(); +} +console.log(JSON.stringify(g([])), JSON.stringify(g([4]))); +for (var top in {}) {} +const readTop = () => typeof top; +console.log(readTop()); diff --git a/tests/harness/corpus-costs/7553-var-for-in-of-zero-iterations.js.json b/tests/harness/corpus-costs/7553-var-for-in-of-zero-iterations.js.json new file mode 100644 index 000000000..566096d24 --- /dev/null +++ b/tests/harness/corpus-costs/7553-var-for-in-of-zero-iterations.js.json @@ -0,0 +1 @@ +{ "plain": 19400, "san": 25900 } From a10c183b13f1f7b70ea200adf61198b2fb5ff4f7 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 09:05:55 -0500 Subject: [PATCH 21/53] Revert "Cache untyped property access in programs that declare classes (#1995)" --- benchmarks/runtime/README.md | 1 - .../runtime/workloads/dynamic-properties.json | 6 - .../runtime/workloads/dynamic-properties.mjs | 56 -------- packages/compiler/src/ir/js-ic.test.ts | 134 ------------------ packages/compiler/src/ir/js-ic.ts | 33 +---- packages/compiler/test/js-ic-emission.test.ts | 44 ------ scripts/zero-cost-compare.mjs | 5 +- 7 files changed, 8 insertions(+), 271 deletions(-) delete mode 100644 benchmarks/runtime/workloads/dynamic-properties.json delete mode 100644 benchmarks/runtime/workloads/dynamic-properties.mjs diff --git a/benchmarks/runtime/README.md b/benchmarks/runtime/README.md index 04c90f7bb..4f152e4bb 100644 --- a/benchmarks/runtime/README.md +++ b/benchmarks/runtime/README.md @@ -13,7 +13,6 @@ Application-shaped workloads for measuring the speed of compiled executables. Ea | `cyclic-graphs` | cpu | object graphs with back references built and dropped: trees with parent and sibling links, rings of records sharing a long-lived registry, closures capturing their owner | | `dictionary-walks` | cpu | `Record` dictionaries walked with for-in and `Object.keys`/`values`/`entries` while entries are added, updated and deleted | | `dynamic-methods` | cpu | untyped JavaScript calling builtin methods on values it does not type: string `slice`/`indexOf`/`toUpperCase`/`padStart`, array `push`/`pop`/`includes`, `Map` and `Set`, and an object's own methods | -| `dynamic-properties` | cpu | untyped JavaScript reading and writing properties of values it does not type: records built by object literals in two key orders, nested records, and instances of a class and its subclass | | `functional-pipeline` | cpu | `filter`/`map`/`reduce`/`flatMap` chains, closures, small records | | `intern-table`, `intern-table-map`, `intern-table-1`, `intern-table-map-1` | cpu | worker threads interning the same type-like objects under string keys and records under number keys, then hits and misses: one shared `InternTable` per kind of key, or a module-level `Map` per thread (the `-map` entries), with 8 threads or 1 (the `-1` entries); under Node.js every thread has its own tables | | `json-records` | cpu | `JSON.stringify`, typed `JSON.parse`, Map aggregation | diff --git a/benchmarks/runtime/workloads/dynamic-properties.json b/benchmarks/runtime/workloads/dynamic-properties.json deleted file mode 100644 index 398981ef3..000000000 --- a/benchmarks/runtime/workloads/dynamic-properties.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "entry": "workloads/dynamic-properties.mjs", - "args": [ - "1000000" - ] -} diff --git a/benchmarks/runtime/workloads/dynamic-properties.mjs b/benchmarks/runtime/workloads/dynamic-properties.mjs deleted file mode 100644 index d0ee3802c..000000000 --- a/benchmarks/runtime/workloads/dynamic-properties.mjs +++ /dev/null @@ -1,56 +0,0 @@ -// Untyped JavaScript reading and writing properties of values it never -// declares types for: records built by object literals in two key orders, -// nested records, property updates, and instances of a class and its -// subclass read and written the same way. Argument: the number of -// iterations. -const rounds = Number(process.argv[2] ?? "1000000"); -class Cursor { - constructor(x, y) { - this.x = x; - this.y = y; - } - step(dx, dy) { - this.x += dx; - this.y += dy; - return this; - } -} -class Marker extends Cursor { - constructor(x, y, label) { - super(x, y); - this.label = label; - } -} -function makeItem(i) { - if (i % 3 === 0) - return { id: i, price: i % 17, qty: i % 5, meta: { tag: i % 7, weight: i % 11 } }; - return { id: i, qty: i % 5, price: i % 13, meta: { weight: i % 9, tag: i % 5 } }; -} -function cost(item) { - return item.price * item.qty + item.meta.weight; -} -function restock(item, round) { - item.qty = (item.qty + round) % 7; - item.meta.tag = (item.meta.tag + 1) % 7; -} -function distance(a, b) { - return Math.abs(a.x - b.x) + Math.abs(a.y - b.y); -} -const items = []; -for (let i = 0; i < 1024; i++) items.push(makeItem(i)); -const cursors = []; -for (let i = 0; i < 64; i++) - cursors.push(i % 2 === 0 ? new Cursor(i - 32, 7 - i) : new Marker(i - 32, 7 - i, "m" + i)); -const origin = { x: 0, y: 0 }; -let checksum = 0; -for (let r = 0; r < rounds; r++) { - const item = items[r & 1023]; - checksum += cost(item); - restock(item, r & 3); - checksum += item.meta.tag; - const cursor = cursors[r & 63]; - checksum += distance(cursor.step((r % 3) - 1, (r % 5) - 2), origin); - if (cursor.x > 100 || cursor.x < -100) cursor.x = 0; - if (cursor.y > 100 || cursor.y < -100) cursor.y = 0; -} -console.log(checksum, items[5].qty, items[6].meta.tag, cursors[3].x, cursors[3].y, cursors[3].label); diff --git a/packages/compiler/src/ir/js-ic.test.ts b/packages/compiler/src/ir/js-ic.test.ts index 026600cc3..f71a1e9db 100644 --- a/packages/compiler/src/ir/js-ic.test.ts +++ b/packages/compiler/src/ir/js-ic.test.ts @@ -126,140 +126,6 @@ test("property helpers whose class branches only test capsules are cache sites", expect(sites.site("%dyn.class.property.1")).toEqual({ kind: "set", key: "y" }); }); -test("bags converted between dyn and jsval are still the receiver", () => { - // A program that declares a class returns its bags as jsvals, and the - // helpers read them through js.toDyn. - const toDyn = (value: IrExpr): IrExpr => ({ - kind: "libCall", - fn: "js.toDyn", - args: [value], - type: DYN, - loc, - }); - const fromDyn = (value: IrExpr): IrExpr => ({ - kind: "libCall", - fn: "js.fromDyn", - args: [value], - type: { kind: "jsval" }, - loc, - }); - const jsvalBag = helper( - "%dyn.class.readPropertiesOrEmpty.js", - ["v"], - [capsuleBranch("v"), ret(fromDyn(ref("v")))], - ); - const staticBag = helper("%dyn.class.staticProperties", ["v"], [ret(ref("v"))]); - const getter = getHelper( - "%dyn.class.property.0", - "x", - [capsuleBranch("p.0")], - call(staticBag.name, toDyn(call(jsvalBag.name, ref("p.0")))), - ); - const setter = setHelper("%dyn.class.property.1", "y"); - const write = setter.body[0] as Extract; - const keySet = write.expr as Extract; - keySet.args[0] = toDyn(call(jsvalBag.name, ref("p.0"))); - // A converted value that is not the receiver stays a helper call. - const foreign = getHelper( - "%dyn.class.property.2", - "x", - [], - toDyn(fromDyn({ kind: "dynObjLit", type: DYN, loc } as IrExpr)), - ); - const sites = sitesOf(jsvalBag, staticBag, getter, setter, foreign); - expect(sites.site("%dyn.class.property.0")).toEqual({ kind: "get", key: "x" }); - expect(sites.site("%dyn.class.property.1")).toEqual({ kind: "set", key: "y" }); - expect(sites.site("%dyn.class.property.2")).toBeNull(); -}); - -test("a bag helper may first rebind its parameter to a bag of it", () => { - // Programs with subclasses normalize the value before the class branches. - const normalize = helper("%dyn.class.normalize", ["v"], [capsuleBranch("v"), ret(ref("v"))]); - const rebind = (value: IrExpr): IrStmt => ({ kind: "assign", localId: "v", value, loc }); - const normalized = helper( - "%dyn.class.readPropertiesOrEmpty.normalized", - ["v"], - [rebind(call(normalize.name, ref("v"))), capsuleBranch("v"), ret(ref("v"))], - ); - const replaced = helper( - "%dyn.class.readPropertiesOrEmpty.replaced", - ["v"], - [rebind({ kind: "dynObjLit", type: DYN, loc } as IrExpr), ret(ref("v"))], - ); - const sites = sitesOf( - normalize, - normalized, - replaced, - getHelper("%dyn.class.property.0", "x", [], call(normalized.name, ref("p.0"))), - getHelper("%dyn.class.property.1", "x", [], call(replaced.name, ref("p.0"))), - // The property helper itself normalizes its receiver first, too. - getHelper("%dyn.class.property.2", "x", [ - { kind: "assign", localId: "p.0", value: call(normalize.name, ref("p.0")), loc }, - capsuleBranch("p.0"), - ]), - ); - expect(sites.site("%dyn.class.property.0")).toEqual({ kind: "get", key: "x" }); - expect(sites.site("%dyn.class.property.1")).toBeNull(); - expect(sites.site("%dyn.class.property.2")).toEqual({ kind: "get", key: "x" }); -}); - -test("keyed helpers may test capsules between the key's coercion and the read", () => { - const read = (object: IrExpr, key: string): IrExpr => ({ - kind: "libCall", - fn: "js.fromDyn", - args: [{ kind: "dynKeyGet", value: object, key: ref(key), type: DYN, loc }], - type: { kind: "jsval" }, - loc, - }); - const guard = (test: "symbol" | "nullish", tested: string, object: IrExpr): IrStmt => ({ - kind: "if", - cond: { kind: "dynTest", test, value: ref(tested), type: BOOL, loc }, - then: [ret(read(object, "p.key"))], - else_: null, - loc, - }); - const keyed = (name: string, between: IrStmt[]): IrFunction => - helper( - name, - ["p.0", "p.key"], - [ - capsuleBranch("p.0"), - guard("symbol", "p.key", call(identityBag.name, ref("p.0"))), - guard("nullish", "p.0", ref("p.0")), - { - kind: "varDecl", - localId: "key.string", - init: { - kind: "libCall", - fn: "dyn.toStringCoerce", - args: [ref("p.key")], - type: STRING, - loc, - }, - loc, - }, - ...between, - ret(read(call(identityBag.name, ref("p.0")), "key.string")), - ], - ["key.string"], - ); - const errorBranch: IrStmt = { - kind: "if", - cond: { kind: "dynTest", test: "error", value: ref("p.0"), type: BOOL, loc }, - then: [ret(ref("p.0"))], - else_: null, - loc, - }; - const sites = sitesOf( - keyed("%dyn.class.computed.0", []), - keyed("%dyn.class.computed.1", [capsuleBranch("p.0"), capsuleBranch("p.0", "dyn.classIs")]), - keyed("%dyn.class.computed.2", [errorBranch]), - ); - expect(sites.site("%dyn.class.computed.0")).toEqual({ kind: "keyed" }); - expect(sites.site("%dyn.class.computed.1")).toEqual({ kind: "keyed" }); - expect(sites.site("%dyn.class.computed.2")).toBeNull(); -}); - test("any other prefix or tail keeps the helper call", () => { const errorBranch: IrStmt = { kind: "if", diff --git a/packages/compiler/src/ir/js-ic.ts b/packages/compiler/src/ir/js-ic.ts index 58c568bbd..752316db3 100644 --- a/packages/compiler/src/ir/js-ic.ts +++ b/packages/compiler/src/ir/js-ic.ts @@ -62,11 +62,7 @@ export class JsIcSites { const receiver = fn.params[0]?.localId; if (receiver === undefined) return null; let i = 0; - while ( - i < fn.body.length && - (this.capsuleBranch(fn.body[i]!, receiver) || this.rebinding(fn.body[i]!, receiver)) - ) - i++; + while (i < fn.body.length && this.capsuleBranch(fn.body[i]!, receiver)) i++; const tail = fn.body.slice(i); if (fn.params.length === 1) return this.getTail(tail, receiver); if (fn.params.length !== 2) return null; @@ -113,14 +109,10 @@ export class JsIcSites { /** `if (typeof key === "symbol") return dynKeyGet(BAG(receiver), key); * if (receiver == null) return dynKeyGet(receiver, key); * const s = String(key); return dynKeyGet(BAG(receiver), s)`, each read - * possibly under js.fromDyn. Capsule branches may precede the final read: - * a program that declares classes looks the string key up in each class's - * key table there. */ + * possibly under js.fromDyn. */ private keyedTail(tail: IrStmt[], receiver: string, key: string): JsIcSite | null { - const [symbol, nullish, decl] = tail; - const ret = tail[tail.length - 1]; - if (tail.length < 4) return null; - if (!tail.slice(3, -1).every((s) => this.capsuleBranch(s, receiver))) return null; + const [symbol, nullish, decl, ret] = tail; + if (tail.length !== 4) return null; if (!this.guard(symbol, "symbol", key, (read) => this.bagOf(read, receiver), key)) return null; if (!this.guard(nullish, "nullish", receiver, (read) => isVar(read, receiver), key)) return null; @@ -167,12 +159,9 @@ export class JsIcSites { } /** Whether `e` is the receiver itself for every ordinary object or array: - * the receiver, or a chain of property-bag helpers applied to it. A - * conversion between the dyn and jsval forms keeps the object it holds. */ + * the receiver, or a chain of property-bag helpers applied to it. */ private bagOf(e: IrExpr, receiver: string): boolean { if (isVar(e, receiver)) return true; - if (e.kind === "libCall" && (e.fn === "js.toDyn" || e.fn === "js.fromDyn")) - return e.args.length === 1 && this.bagOf(e.args[0]!, receiver); return ( e.kind === "call" && e.args.length === 1 && @@ -193,13 +182,6 @@ export class JsIcSites { return ret.kind === "return" && ret.value !== null && this.bagOf(ret.value, param); } - /** `param = `, such as the subclass normalization - * `value = %dyn.class.normalize(value)`, which leaves an ordinary object - * or array in `param`. */ - private rebinding(s: IrStmt, param: string): boolean { - return s.kind === "assign" && s.localId === param && this.bagOf(s.value, param); - } - /** A property-bag helper that returns its argument unless a capsule * branch is taken. */ private identityHelper(name: string): boolean { @@ -222,10 +204,7 @@ export class JsIcSites { identity = fn.body .slice(0, -1) - .every( - (s) => - this.capsuleBranch(s, param) || this.kindExit(s, param) || this.rebinding(s, param), - ) && + .every((s) => this.capsuleBranch(s, param) || this.kindExit(s, param)) && last?.kind === "return" && last.value !== null && this.bagOf(last.value, param); diff --git a/packages/compiler/test/js-ic-emission.test.ts b/packages/compiler/test/js-ic-emission.test.ts index 54ea13d26..47d641bff 100644 --- a/packages/compiler/test/js-ic-emission.test.ts +++ b/packages/compiler/test/js-ic-emission.test.ts @@ -64,50 +64,6 @@ console.log(read(o), at(dyn([5, 6]), 1), call(o), o.n); } }); -test("property sites keep their caches in programs that declare classes and subclasses", async () => { - const dir = await mkdtemp(join(tmpdir(), "scriptc-js-ic-class-")); - try { - const entry = join(dir, "main.js"); - const outPath = join(dir, "main.ir.json"); - // Declared classes give every property helper class branches and - // property-bag lookups, and a subclass adds a normalization step; plain - // objects still read through the caches. - await writeFile( - entry, - ` -class Point { - constructor(x) { this.x = x; } -} -class Tagged extends Point { - constructor(x, tag) { super(x); this.tag = tag; } -} -function dyn(v) { return v; } -function read(o) { return o.x; } -function write(o, v) { o.x = v; } -function at(a, i) { return a[i]; } -var o = dyn({ x: 1 }); -var p = dyn(new Point(4)); -var t = dyn(new Tagged(6, "t")); -write(o, 3); -write(p, 5); -write(t, 7); -console.log(read(o), read(p), read(t), at(dyn([5, 6]), 1), at(p, "x"), at(t, "tag")); -`, - ); - const result = await compile(entry, { outDir: dir, outPath, outputKind: "ir" }); - if (!result.ok) - throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); - const mod = deserializeModule(await readFile(outPath, "utf8")); - expect(validateModule(mod)).toEqual([]); - const llvm = emitLlvmModule(mod); - expect(body(llvm, "read")).toContain("@scr_js_load_probe("); - expect(body(llvm, "write")).toContain("@scr_js_store_probe("); - expect(body(llvm, "at")).toContain("@scr_js_keyed_probe("); - } finally { - await rm(dir, { recursive: true, force: true }); - } -}); - test("typed programs have no cache sites and keep the ordinary runtime", async () => { const dir = await mkdtemp(join(tmpdir(), "scriptc-js-ic-typed-")); try { diff --git a/scripts/zero-cost-compare.mjs b/scripts/zero-cost-compare.mjs index f202c2ab7..160e33438 100644 --- a/scripts/zero-cost-compare.mjs +++ b/scripts/zero-cost-compare.mjs @@ -394,8 +394,8 @@ export function dynamicTierSymbols(names) { * not yet traced to its typed root. bench/thread-slots, * bench/channel-messages and bench/worker-messages, added later, link them * through worker messages: - * node:worker_threads values are dyn objects. bench/dynamic-methods and - * bench/dynamic-properties are untyped JavaScript. Every other bench workload is fully typed and must link none. The gate and + * node:worker_threads values are dyn objects. bench/dynamic-methods is + * untyped JavaScript. Every other bench workload is fully typed and must link none. The gate and * tests/harness/bench-dynamic-tier.test.ts enforce that, and the test also * fails when a listed workload stops linking them, so remove a workload here * when a leak fix makes its binary clean. */ @@ -410,7 +410,6 @@ export const DYNAMIC_WORKLOADS = new Set([ "bench/thread-slots", "bench/channel-messages", "bench/dynamic-methods", - "bench/dynamic-properties", "bench/worker-messages", "bench/typed-array-scan-workers", "bench/log-summary", From 0f8dd276d91d11095b73c53393dfcd1251af7a4c Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 09:24:28 -0500 Subject: [PATCH 22/53] Reach a class's methods from its class expression's inner name in static initializers A named class expression's own name (const C = class Impl { static x = Impl.m(); }) names the class inside its body as the binding it is stored in names it outside. The reachability walk followed only stored binding names, so a static initializer calling a method through the inner name did not reach that method, and its early reads of the other cycle member's bindings lost their TDZ checks: the program read the binding before its declaration ran (a crash), where Node throws ReferenceError. The inner name now maps to its class expression. --- packages/compiler/src/frontend/program.ts | 12 +++++++++ .../main.ts.json | 8 ++++++ .../alpha.ts | 11 ++++++++ .../beta.ts | 26 +++++++++++++++++++ .../main.ts | 9 +++++++ .../main.ts.json | 1 + 6 files changed, 67 insertions(+) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7383-cycle-class-expression-name-tdz/main.ts.json create mode 100644 tests/corpus/7383-cycle-class-expression-name-tdz/alpha.ts create mode 100644 tests/corpus/7383-cycle-class-expression-name-tdz/beta.ts create mode 100644 tests/corpus/7383-cycle-class-expression-name-tdz/main.ts create mode 100644 tests/harness/corpus-costs/7383-cycle-class-expression-name-tdz/main.ts.json diff --git a/packages/compiler/src/frontend/program.ts b/packages/compiler/src/frontend/program.ts index 34a993c76..ce8f7d822 100644 --- a/packages/compiler/src/frontend/program.ts +++ b/packages/compiler/src/frontend/program.ts @@ -4187,11 +4187,23 @@ function cycleReachedDeferredReferences7( unitsBySymbol.set(symbol, [...(unitsBySymbol.get(symbol) ?? []), ...units]); allUnits.push(...units); } + // A named class expression's inner name (`class Impl { static x = + // Impl.m(); }`) names the class inside its own body, as the binding it + // is stored in names it outside. + const innerNames: ts.Identifier[] = []; + for (const unit of allUnits) { + if (!ts.isClassExpression(unit) || unit.name === undefined) continue; + const symbol = checker.getSymbolAtLocation(unit.name); + if (symbol === undefined) continue; + innerNames.push(unit.name); + unitsBySymbol.set(symbol, [...(unitsBySymbol.get(symbol) ?? []), unit]); + } // The identifier texts worth resolving: declared names, and the local // names of imports (named, default and namespace) in cycle members. const names = new Set(); for (const c of candidates.values()) names.add(c.name.text); for (const name of storedNames) names.add(name.text); + for (const name of innerNames) names.add(name.text); for (const sf of evaluation) { for (const stmt of sf.statements.nodes) { if (!ts.isImportDeclaration(stmt) || stmt.importClause === undefined) continue; diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7383-cycle-class-expression-name-tdz/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7383-cycle-class-expression-name-tdz/main.ts.json new file mode 100644 index 000000000..69b6d63bb --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7383-cycle-class-expression-name-tdz/main.ts.json @@ -0,0 +1,8 @@ +{ + "order": [ + "/tests/corpus/7383-cycle-class-expression-name-tdz/beta.ts", + "/tests/corpus/7383-cycle-class-expression-name-tdz/alpha.ts", + "/tests/corpus/7383-cycle-class-expression-name-tdz/main.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7383-cycle-class-expression-name-tdz/alpha.ts b/tests/corpus/7383-cycle-class-expression-name-tdz/alpha.ts new file mode 100644 index 000000000..ff8eb4a87 --- /dev/null +++ b/tests/corpus/7383-cycle-class-expression-name-tdz/alpha.ts @@ -0,0 +1,11 @@ +import { Probe, Lazy } from "./beta.ts"; + +export { Probe }; +export const label = "ready"; +export const other = "other"; + +export function later(): string { + return `${Lazy.describe()} ${other}`; +} + +console.log("alpha body", Probe.result, Lazy.value); diff --git a/tests/corpus/7383-cycle-class-expression-name-tdz/beta.ts b/tests/corpus/7383-cycle-class-expression-name-tdz/beta.ts new file mode 100644 index 000000000..d1d89ad1a --- /dev/null +++ b/tests/corpus/7383-cycle-class-expression-name-tdz/beta.ts @@ -0,0 +1,26 @@ +import { label, other } from "./alpha.ts"; + +function guard(read: () => string): string { + try { + return read(); + } catch (error) { + return error instanceof ReferenceError ? `ReferenceError: ${error.message}` : "other error"; + } +} + +export const Probe = class ProbeImpl { + static helper(): string { + return guard(() => `label ${label}`); + } + static result = ProbeImpl.helper(); +}; + +export const Lazy = class LazyImpl { + static read(): string { + return guard(() => `other ${other}`); + } + static describe(): string { + return `lazy ${other}`; + } + static value = [0].map(() => LazyImpl.read()).join(""); +}; diff --git a/tests/corpus/7383-cycle-class-expression-name-tdz/main.ts b/tests/corpus/7383-cycle-class-expression-name-tdz/main.ts new file mode 100644 index 000000000..93b7a47ee --- /dev/null +++ b/tests/corpus/7383-cycle-class-expression-name-tdz/main.ts @@ -0,0 +1,9 @@ +// The first cycle member to run builds classes from named class +// expressions whose static initializers call a method through the inner +// name, directly and from an arrow function; both methods read constants +// of the other member before it declared them: Node's temporal-dead-zone +// ReferenceError. A third method reads a constant only after the cycle +// finished. +import { label, Probe, later } from "./alpha.ts"; + +console.log(label, Probe.result, later()); diff --git a/tests/harness/corpus-costs/7383-cycle-class-expression-name-tdz/main.ts.json b/tests/harness/corpus-costs/7383-cycle-class-expression-name-tdz/main.ts.json new file mode 100644 index 000000000..b24c0c970 --- /dev/null +++ b/tests/harness/corpus-costs/7383-cycle-class-expression-name-tdz/main.ts.json @@ -0,0 +1 @@ +{ "plain": 1800, "san": 2600 } From 7c33b55435e9684dd14ec8373c4238e6be31e7ca Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Fri, 23 Oct 2026 07:39:29 -0700 Subject: [PATCH 23/53] Keep fixed layouts for JavaScript classes next to freezes and unrelated accessors Two rules sent every JavaScript class family to field-creation tracking: - Any Object.freeze, Object.seal or Object.preventExtensions of a value other than a fresh literal. These never create or remove a property, and a class instance keeps its integrity state beside its fields: a typed call sets the CLASS_ATTRIBUTES word that TypeScript classes use with the same fixed layouts, and through an untyped value the runtime refuses the operation on every class instance (SC2020). Only Reflect.deleteProperty stays a removal of any name. - Any getter or setter. An accessor only matters when it shares a name with a field of the instances that see it, or when a subclass accessor can intercept a base constructor's field assignment. Other accessors live on the prototype beside the fields. In zod 3, ParseStatus and ParseInputLazyPath now keep fixed layouts (2 of the 39 tracked classes; the ZodType family stays tracked because its constructor creates own properties before _def). z.array(Order).parse over 500 orders: 270.7M -> 235.8M instructions per round (-12.9%). Co-Authored-By: Claude Opus 5.5 --- .../src/frontend/lowering/js-class-fields.ts | 56 +++++-- .../src/frontend/lowering/lower-classes.ts | 9 +- .../compiler/test/js-class-layouts.test.ts | 45 ++++++ .../7790-js-class-integrity-layouts.js.json | 6 + .../7791-js-class-accessor-layouts.js.json | 6 + .../corpus/7790-js-class-integrity-layouts.js | 139 ++++++++++++++++++ .../corpus/7791-js-class-accessor-layouts.js | 85 +++++++++++ .../7790-js-class-integrity-layouts.js.json | 1 + .../7791-js-class-accessor-layouts.js.json | 1 + 9 files changed, 334 insertions(+), 14 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7790-js-class-integrity-layouts.js.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7791-js-class-accessor-layouts.js.json create mode 100644 tests/corpus/7790-js-class-integrity-layouts.js create mode 100644 tests/corpus/7791-js-class-accessor-layouts.js create mode 100644 tests/harness/corpus-costs/7790-js-class-integrity-layouts.js.json create mode 100644 tests/harness/corpus-costs/7791-js-class-accessor-layouts.js.json diff --git a/packages/compiler/src/frontend/lowering/js-class-fields.ts b/packages/compiler/src/frontend/lowering/js-class-fields.ts index 2f726ce97..28a7ceeda 100644 --- a/packages/compiler/src/frontend/lowering/js-class-fields.ts +++ b/packages/compiler/src/frontend/lowering/js-class-fields.ts @@ -2,11 +2,19 @@ import * as ts from "../ts7/adapter.js"; import { isClassOwnEnumerableFieldName } from "../../ir/ir.js"; import type { ClassInfo } from "./lower-classes.js"; -/** Reflection that can remove a property or change its attributes. */ -const PROPERTY_REFLECTION = new Set(["deleteProperty", "freeze", "seal", "preventExtensions"]); +/* Object.freeze, Object.seal and Object.preventExtensions do not appear + * here: they never create or remove a property, and a class instance keeps + * its integrity state outside the field layout. A statically typed call + * sets the instance's CLASS_ATTRIBUTES word (class-attributes.ts), which + * TypeScript classes use with the same fixed layouts; through an untyped + * value the runtime refuses the operation on any class instance + * (scr_dyn_class_integrity_fence). Lifting that fence must revisit this. */ -/** `Object.freeze({ ... })` and the like apply to a new literal, which is - * not a class instance. */ +/** Reflection that can remove any property. */ +const PROPERTY_REFLECTION = new Set(["deleteProperty"]); + +/** `Reflect.deleteProperty({ ... }, key)` applies to a new literal, which + * is not a class instance. */ function freshLiteralArgument(callee: ts.PropertyAccessExpression): boolean { const call = callee.parent; if (!ts.isCallExpression(call) || call.expression !== callee) return false; @@ -141,12 +149,35 @@ export function constructionPrefix( return { created, sealed }; } -/** Whether a class declares a getter or setter, which can intercept a - * base constructor's field assignment. */ -function declaresAccessor(info: ClassInfo): boolean { - return [...info.methods.keys()].some( - (name) => name.startsWith("get:") || name.startsWith("set:"), - ); +/** The names of the getters and setters among a class's own methods. */ +export function accessorNames(methods: Iterable): Set { + const names = new Set(); + for (const name of methods) + if (name.startsWith("get:") || name.startsWith("set:")) names.add(name.slice(4)); + return names; +} + +/** Whether one of `accessors` can intercept an assignment the base + * constructions make: a base constructor's `this.name = value` runs a + * subclass accessor of that name instead of creating the field. Only field + * names can reach the created prefix; any other assignment already ends + * it. */ +export function accessorInterceptsBase(accessors: ReadonlySet, base: ClassInfo): boolean { + for (const name of accessors) if (base.fields.has(name)) return true; + return false; +} + +/** Whether an accessor that instances of `info` inherit or declare shares + * a name with one of their fields. Such a field and accessor disagree on + * what the instance's property is; any other accessor lives on the + * prototype beside the fields and never touches the layout. */ +function accessorShadowsField(info: ClassInfo): boolean { + for (let current: ClassInfo | null = info; current; current = current.base) { + if (current.def.runtime) break; + for (const name of accessorNames(current.methods.keys())) + if (info.fields.has(name)) return true; + } + return false; } /** `constructionPrefix` for a collected class and its bases. */ @@ -156,7 +187,8 @@ export function construction(info: ClassInfo): Construction | null { if (info.base) { base = construction(info.base); if (!base) return null; - if (declaresAccessor(info)) base = { created: base.created, sealed: false }; + if (accessorInterceptsBase(accessorNames(info.methods.keys()), info.base)) + base = { created: base.created, sealed: false }; } return constructionPrefix(info.decl, base, (name) => info.fields.has(name)); } @@ -169,7 +201,7 @@ export function construction(info: ClassInfo): Construction | null { export function fixedFieldFamily(root: ClassInfo, deleted: Set | null): boolean { if (deleted === null) return false; const fixed = (info: ClassInfo): boolean => { - if (declaresAccessor(info)) return false; + if (accessorShadowsField(info)) return false; const built = construction(info); if (!built) return false; for (const field of info.fieldOrder) { diff --git a/packages/compiler/src/frontend/lowering/lower-classes.ts b/packages/compiler/src/frontend/lowering/lower-classes.ts index 29afbb4fe..765f24884 100644 --- a/packages/compiler/src/frontend/lowering/lower-classes.ts +++ b/packages/compiler/src/frontend/lowering/lower-classes.ts @@ -177,7 +177,12 @@ import { hasLexicalPrivateReference, hasStaticThis, rejectStaticThis } from "./s import { lowerUrlNew } from "./lower-url.js"; import { isNativeProxyInitializer, lowerNativeProxy } from "./expressions/native-proxy.js"; import { classStaticDataFor } from "./class-static-data.js"; -import { construction, constructionPrefix } from "./js-class-fields.js"; +import { + accessorInterceptsBase, + accessorNames, + construction, + constructionPrefix, +} from "./js-class-fields.js"; import { lowerObjectFactoryNew } from "./object-factory-new.js"; import { tryLowerExpression } from "./expressions/try-lower-expression.js"; import { lowerInstanceConstructorNew } from "./class-instance-constructor.js"; @@ -4114,7 +4119,7 @@ export function collectClassShapeInner( // A subclass field created before anything can observe the instance // keeps its type: super() cannot run code that reads it earlier. const baseConstruction = - base && ![...methods.keys()].some((m) => m.startsWith("get:") || m.startsWith("set:")) + base && !accessorInterceptsBase(accessorNames(methods.keys()), base) ? construction(base) : null; const sealedPrefix = baseConstruction diff --git a/packages/compiler/test/js-class-layouts.test.ts b/packages/compiler/test/js-class-layouts.test.ts index c8491f3e8..110d007ee 100644 --- a/packages/compiler/test/js-class-layouts.test.ts +++ b/packages/compiler/test/js-class-layouts.test.ts @@ -101,6 +101,51 @@ delete loose.label; expect(layout(deleted, "Shape").tracks).toBe(true); }); +test("freezing values other than class instances keeps JavaScript class layouts fixed", async () => { + const module = await lower(`${SHAPES} +/** @param {{ value: any }} data */ +function settle(data) { + data.value = Object.freeze(data.value); + Object.seal(data); + return Object.preventExtensions(data.value); +} +settle({ value: { sides: 1 } }); +const square = new Square(); +Object.freeze(square); +`); + expect(layout(module, "Shape").tracks).toBe(false); + expect(layout(module, "Square").tracks).toBe(false); +}); + +test("accessors that share no field's name keep JavaScript class layouts fixed", async () => { + const module = await lower(` +class Path { + constructor(key) { + this.cache = []; + this.key = key; + } + get path() { + if (!this.cache.length) this.cache.push(this.key); + return this.cache; + } +} +class Named extends Path { + constructor(key) { + super(key); + this.label = "named"; + } + /** @param {string} value */ + set label2(value) { + this.label = value; + } +} +console.log(new Named("k").path.length); +`); + expect(layout(module, "Path")).toEqual({ fields: "cache:dyn key:dyn", tracks: false }); + expect(layout(module, "Named").tracks).toBe(false); + expect(layout(module, "Named").fields).toBe("cache:dyn key:dyn label:string"); +}); + test("JavaScript subclass fields keep their types when the base constructor only assigns fields", async () => { const module = await lower(` class Base { diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7790-js-class-integrity-layouts.js.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7790-js-class-integrity-layouts.js.json new file mode 100644 index 000000000..526b86044 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7790-js-class-integrity-layouts.js.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7790-js-class-integrity-layouts.js" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7791-js-class-accessor-layouts.js.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7791-js-class-accessor-layouts.js.json new file mode 100644 index 000000000..5350539f0 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7791-js-class-accessor-layouts.js.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7791-js-class-accessor-layouts.js" + ], + "diags": [] +} diff --git a/tests/corpus/7790-js-class-integrity-layouts.js b/tests/corpus/7790-js-class-integrity-layouts.js new file mode 100644 index 000000000..f889c02f8 --- /dev/null +++ b/tests/corpus/7790-js-class-integrity-layouts.js @@ -0,0 +1,139 @@ +"use strict"; +// Object.freeze, Object.seal and Object.preventExtensions next to JavaScript +// classes whose fields all exist from construction. The integrity state of +// a class instance lives beside its fields, so classes keep fixed layouts +// whether or not a program freezes values; frozen instances still refuse +// writes and additions as in Node. + +class Point { + constructor(x, y) { + this.x = x; + this.y = y; + } + norm() { + return Math.abs(this.x) + Math.abs(this.y); + } +} + +class Point3 extends Point { + constructor(x, y, z) { + super(x, y); + this.z = z; + } + norm() { + return super.norm() + Math.abs(this.z); + } +} + +class Status { + constructor() { + this.value = "valid"; + } + dirty() { + if (this.value === "valid") this.value = "dirty"; + } +} + +/** @param {any} target @param {string} key @param {unknown} value */ +function put(target, key, value) { + target[key] = value; +} + +/** @param {any} value */ +function describe(value) { + const keys = Object.keys(value); + const parts = []; + for (const key in value) parts.push(`${key}=${String(value[key])}`); + return `${keys.join(",")} | ${parts.join(" ")} | ${JSON.stringify(value)}`; +} + +/** @param {() => void} fn */ +function attempt(label, fn) { + try { + fn(); + console.log(label, "ok"); + } catch (e) { + console.log(label, e instanceof TypeError ? "TypeError" : "other", e.message); + } +} + +// The zod shape: freeze whatever value a result carries. Here it is always +// plain data, never a class instance. +/** @param {{ status: string, value: any }} data */ +function freezeResult(data) { + if (data.status === "valid") data.value = Object.freeze(data.value); + return data; +} + +const results = [ + freezeResult({ status: "valid", value: { a: 1, list: [1, 2] } }), + freezeResult({ status: "valid", value: [3, 4] }), + freezeResult({ status: "aborted", value: { b: 2 } }), + freezeResult({ status: "valid", value: "text" }), +]; +for (const result of results) console.log(result.status, Object.isFrozen(result.value), JSON.stringify(result.value)); +/** @param {() => void} fn */ +function throws(fn) { + try { + fn(); + return false; + } catch (e) { + return e instanceof TypeError; + } +} +console.log(throws(() => put(results[0].value, "a", 9)), throws(() => put(results[2].value, "b", 3))); +console.log(JSON.stringify(results[2].value)); + +// Class instances that only ever travel through untyped code. +const status = new Status(); +status.dirty(); +const p = new Point(1, -2); +const q = new Point3(3, 4, -5); +put(p, "tag", "p"); +put(q, "w", 6); +console.log(describe(status), status.value); +console.log(describe(p), p.norm()); +console.log(describe(q), q.norm()); +console.log(Object.isFrozen(p), Object.isExtensible(q), Object.isSealed(status)); + +// Statically typed freeze, seal and preventExtensions of instances. +const frozen = new Point(5, 6); +Object.freeze(frozen); +attempt("frozen field write", () => { + /** @type {Point} */ (frozen).x = 50; +}); +attempt("frozen field write untyped", () => put(frozen, "y", 60)); +attempt("frozen addition", () => put(frozen, "extra", 1)); +console.log(describe(frozen), Object.isFrozen(frozen), Object.isSealed(frozen), Object.isExtensible(frozen)); +console.log(JSON.stringify(Object.getOwnPropertyDescriptor(frozen, "x"))); + +const sealed = new Point3(1, 2, 3); +put(sealed, "before", true); +Object.seal(sealed); +attempt("sealed field write", () => { + sealed.z = 30; +}); +attempt("sealed added write", () => put(sealed, "before", false)); +attempt("sealed addition", () => put(sealed, "after", 1)); +console.log(describe(sealed), Object.isFrozen(sealed), Object.isSealed(sealed), Object.isExtensible(sealed)); +console.log(JSON.stringify(Object.getOwnPropertyDescriptor(sealed, "z"))); + +const closed = new Status(); +Object.preventExtensions(closed); +closed.dirty(); +attempt("closed addition", () => put(closed, "more", 1)); +console.log(describe(closed), Object.isFrozen(closed), Object.isExtensible(closed)); + +// A subclass instance frozen through its base type. +/** @type {Point} */ +const asBase = new Point3(7, 8, 9); +Object.freeze(asBase); +attempt("frozen subclass field", () => put(asBase, "z", 90)); +console.log(describe(asBase), asBase.norm(), Object.isFrozen(asBase)); + +// Fresh instances of the same classes stay writable. +const fresh = new Point3(0, 0, 0); +fresh.x = 1; +put(fresh, "y", 2); +put(fresh, "added", 3); +console.log(describe(fresh), fresh.norm(), Object.isFrozen(fresh)); diff --git a/tests/corpus/7791-js-class-accessor-layouts.js b/tests/corpus/7791-js-class-accessor-layouts.js new file mode 100644 index 000000000..6c87989cb --- /dev/null +++ b/tests/corpus/7791-js-class-accessor-layouts.js @@ -0,0 +1,85 @@ +"use strict"; +// Getters and setters on JavaScript classes. An accessor whose name no +// field uses lives on the prototype beside the fields; an accessor that +// shares a field's name decides what an assignment does. Both behave as in +// Node, for direct use and for instances passed to untyped code. + +class LazyPath { + constructor(parent, value, path, key) { + this._cachedPath = []; + this.parent = parent; + this.data = value; + this._path = path; + this._key = key; + } + get path() { + if (!this._cachedPath.length) { + if (Array.isArray(this._key)) this._cachedPath.push(...this._path, ...this._key); + else this._cachedPath.push(...this._path, this._key); + } + return this._cachedPath; + } +} + +class Temperature { + constructor(celsius) { + this.celsius = celsius; + } + get fahrenheit() { + return this.celsius * 1.8 + 32; + } + set fahrenheit(value) { + this.celsius = (value - 32) / 1.8; + } +} + +class Labeled extends Temperature { + constructor(celsius, label) { + super(celsius); + this.label = label; + } + get description() { + return `${this.label}: ${this.celsius.toFixed(1)}C`; + } +} + +/** @param {any} target @param {string} key @param {unknown} value */ +function put(target, key, value) { + target[key] = value; +} + +/** @param {any} value */ +function describe(value) { + const parts = []; + for (const key in value) parts.push(key); + return `${Object.keys(value).join(",")} | ${parts.join(",")} | ${JSON.stringify(value)}`; +} + +/** @param {any} value @param {string} key */ +function probe(value, key) { + return `${key}: in=${key in value} own=${Object.prototype.hasOwnProperty.call(value, key)} value=${JSON.stringify(value[key])}`; +} + +const root = { common: true }; +const paths = []; +for (let i = 0; i < 3; i++) paths.push(new LazyPath(root, { n: i }, ["items"], i)); +paths.push(new LazyPath(root, null, ["a"], ["b", "c"])); +for (const item of paths) console.log(item.path.join("."), item.data && item.data.n, item.parent === root); +console.log(describe(paths[0])); +console.log(probe(paths[1], "path"), probe(paths[1], "_key")); +put(paths[2], "extra", 1); +console.log(describe(paths[2]), paths[2].path.length); + +const t = new Temperature(100); +console.log(t.fahrenheit, describe(t)); +t.fahrenheit = 32; +console.log(t.celsius, t.fahrenheit); +put(t, "fahrenheit", 212); +console.log(t.celsius, probe(t, "fahrenheit"), probe(t, "celsius")); +put(t, "note", "added"); +console.log(describe(t)); + +const l = new Labeled(20, "room"); +console.log(l.description, l.fahrenheit, describe(l)); +l.fahrenheit = 50; +console.log(l.description, probe(l, "description")); diff --git a/tests/harness/corpus-costs/7790-js-class-integrity-layouts.js.json b/tests/harness/corpus-costs/7790-js-class-integrity-layouts.js.json new file mode 100644 index 000000000..af44b31af --- /dev/null +++ b/tests/harness/corpus-costs/7790-js-class-integrity-layouts.js.json @@ -0,0 +1 @@ +{ "plain": 4400, "san": 5200 } diff --git a/tests/harness/corpus-costs/7791-js-class-accessor-layouts.js.json b/tests/harness/corpus-costs/7791-js-class-accessor-layouts.js.json new file mode 100644 index 000000000..eb36dcefe --- /dev/null +++ b/tests/harness/corpus-costs/7791-js-class-accessor-layouts.js.json @@ -0,0 +1 @@ +{ "plain": 5100, "san": 4700 } From a23ce3043c0ab0d8c8b3ff495c0832fc4bc7a654 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 09:53:05 -0500 Subject: [PATCH 24/53] Keep loop-carried locals written from int32 element reads int32 The int32 array proof assumes its candidates' elements are int32 and asks the integer range analysis whether every stored value is one. A local assigned in a loop from an element read of a candidate array (`const t = next[at]!; next[at] = at; at = t`) never closed under the int32 bound, because the loop-carried closure had no case for element reads, so arrays that store such cursors kept double slots and every index converted from a double. Element reads now take the slot range in that closure, as field reads and calls already did. The index-cursors runtime workload walks an array of indices with such a cursor. --- .../runtime/workloads/index-cursors.json | 6 ++++ benchmarks/runtime/workloads/index-cursors.ts | 25 +++++++++++++++ .../compiler/src/ir/integer-ranges.test.ts | 31 +++++++++++++++++++ packages/compiler/src/ir/integer-ranges.ts | 1 + .../test/int32-element-array-emission.test.ts | 24 ++++++++++++++ 5 files changed, 87 insertions(+) create mode 100644 benchmarks/runtime/workloads/index-cursors.json create mode 100644 benchmarks/runtime/workloads/index-cursors.ts diff --git a/benchmarks/runtime/workloads/index-cursors.json b/benchmarks/runtime/workloads/index-cursors.json new file mode 100644 index 000000000..24f2a263c --- /dev/null +++ b/benchmarks/runtime/workloads/index-cursors.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/index-cursors.ts", + "args": [ + "20000" + ] +} diff --git a/benchmarks/runtime/workloads/index-cursors.ts b/benchmarks/runtime/workloads/index-cursors.ts new file mode 100644 index 000000000..b65ded46b --- /dev/null +++ b/benchmarks/runtime/workloads/index-cursors.ts @@ -0,0 +1,25 @@ +// Arrays of indices walked by loop-carried cursors: each step reads the +// next index from the array and relinks the visited entries, so the +// cursor and every stored value are small integers read back from the +// array itself. +const N = 4096; + +function relink(rounds: number): number { + const next: number[] = []; + for (let i = 0; i < N; i++) next.push(i + 1 < N ? i + 1 : 0); + let at = next[0]!; + let total = 0; + for (let r = 0; r < rounds; r++) { + for (let s = 0; s < N; s++) { + const t = next[at]!; + next[at] = next[t]!; + next[t] = at; + at = t; + total = (total + at) | 0; + } + } + return total; +} + +const rounds = Number(process.argv[2] ?? "1"); +console.log("relink", relink(rounds)); diff --git a/packages/compiler/src/ir/integer-ranges.test.ts b/packages/compiler/src/ir/integer-ranges.test.ts index fcb040275..5ae04d28f 100644 --- a/packages/compiler/src/ir/integer-ranges.test.ts +++ b/packages/compiler/src/ir/integer-ranges.test.ts @@ -440,6 +440,37 @@ test("int32-closed locals keep the int32 range across loops and switches", () => expect(switched.get(afterSwitch)).toEqual(INT32); }); +test("loop-carried locals written from Int32[] element reads keep the int32 range", () => { + const arrType = { kind: "array", elem: F64 } as const; + const loop = (localId: string): { after: IrExpr; f: IrFunction } => { + const after = ref(); + // `x = a[x]!` in a loop: a cursor that follows indices stored in the array. + const read: IrExpr = { + loc, + kind: "arrayGet", + arr: { loc, kind: "varRef", localId, type: arrType }, + index: ref(), + type: F64, + }; + const f = fn([ + assign(num(0)), + { + loc, + kind: "while", + cond: condition("<", ref("unknown"), num(1)), + body: [assign(read)], + }, + observe(after), + ]); + return { after, f }; + }; + const elements = { elements: (arr: IrExpr) => arr.kind === "varRef" && arr.localId === "ints" }; + const ints = loop("ints"); + expect(analyzeIntegerRanges(ints.f, elements).get(ints.after)).toEqual(INT32); + const floats = loop("floats"); + expect(analyzeIntegerRanges(floats.f, elements).get(floats.after) ?? null).toBeNull(); +}); + test("uint32-closed locals keep the uint32 range across loop back edges", () => { // for (let i = 0; i < n; i++) { v = unknown >>> 0; acc = (acc + v) >>> 0; ... } const loop = (start: IrExpr | null, writes: IrExpr[]) => { diff --git a/packages/compiler/src/ir/integer-ranges.ts b/packages/compiler/src/ir/integer-ranges.ts index 0e5ca426d..683ceab75 100644 --- a/packages/compiler/src/ir/integer-ranges.ts +++ b/packages/compiler/src/ir/integer-ranges.ts @@ -480,6 +480,7 @@ export function analyzeIntegerRangeFacts( return boxedRange(e.localId); } case "fieldGet": + case "arrayGet": case "call": return slotRange(e); default: diff --git a/packages/compiler/test/int32-element-array-emission.test.ts b/packages/compiler/test/int32-element-array-emission.test.ts index 0b68a5d3e..bb6623d5f 100644 --- a/packages/compiler/test/int32-element-array-emission.test.ts +++ b/packages/compiler/test/int32-element-array-emission.test.ts @@ -77,6 +77,19 @@ function literalCounter(n: number): number { for (let i = 0; i < n; i++) a[0] = a[0]! + 1; return a[0]!; } +function walk(n: number, steps: number): number { + const next: number[] = []; + for (let i = 0; i < n; i++) next.push(i + 1 < n ? i + 1 : 0); + let at = next[0]!; + let total = 0; + for (let s = 0; s < steps; s++) { + const t = next[at]!; + next[at] = at; + at = t; + total = (total + at) | 0; + } + return total; +} function joined(n: number): string { const a = [n | 0, 2]; return a.join(","); @@ -91,6 +104,7 @@ console.log( shortCounter(), literalCounter(n), joined(n), + walk(n | 0, 5), ); `; @@ -119,6 +133,16 @@ test("arrays storing only int32 elements get 4-byte slots", async () => { expect(counts).not.toContain("popArray"); }); +test("an array that stores a cursor read back from its own elements gets 4-byte slots", async () => { + const llvm = await emitLlvm(source); + const walk = functions(llvm, "walk"); + expect(walk).not.toBe(""); + // `at` is assigned `next[at]!` in the loop and stored back into `next`: + // it holds an int32 whenever the array's elements are int32s. + expect(walk).toMatch(/@scr_arr_new_small\(i32 7\)/); + expect(walk).toMatch(/getelementptr inbounds i32, ptr %\w+, i64 %\w+\n\s+%\w+ = load i32/); +}); + test("arrays with a store that has no exact integer range, or that escape, keep doubles", async () => { const llvm = await emitLlvm(source); for (const name of ["fraction", "negativeZero", "joined"]) { From 056dda3633b6c01e9df77ef9fbca683ef4e4e800 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Fri, 23 Oct 2026 07:35:50 -0700 Subject: [PATCH 25/53] Read Web Headers pairs without copying them, and cache their sorted list Headers kept its name/value pairs in a string array and read every element through scr_arr_get_ref (index checks, retain, release). forEach and the iterators re-sorted and rescanned the whole list for every step, and every Response or Headers built from another Headers object re-validated and re-normalized each copied pair. The Hono fixture spends 12-15% of a request there. - Pair loops read borrowed elements of the dense array (sf_pair_at). - The sort-and-combine list is cached on the Headers object and dropped by every mutation, so live iteration keeps Node's semantics. - get() of a single value returns the stored string. - Copying a Headers object whose pairs are canonical (built by the constructor or a Response) into an empty list copies the pairs; wire response headers keep the per-pair path because they may repeat names. Co-Authored-By: Claude Opus 5.5 --- .../corpus/fetch-headers-copy-cache.ts.json | 6 + packages/runtime/src/scr_fetch.c | 268 ++++++++++-------- tests/corpus/fetch-headers-copy-cache.ts | 85 ++++++ .../fetch-headers-copy-cache.ts.json | 1 + 4 files changed, 236 insertions(+), 124 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/fetch-headers-copy-cache.ts.json create mode 100644 tests/corpus/fetch-headers-copy-cache.ts create mode 100644 tests/harness/corpus-costs/fetch-headers-copy-cache.ts.json diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/fetch-headers-copy-cache.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/fetch-headers-copy-cache.ts.json new file mode 100644 index 000000000..bd69651b1 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/fetch-headers-copy-cache.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/fetch-headers-copy-cache.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_fetch.c b/packages/runtime/src/scr_fetch.c index 6049a4936..48b60942e 100644 --- a/packages/runtime/src/scr_fetch.c +++ b/packages/runtime/src/scr_fetch.c @@ -172,7 +172,14 @@ struct SfStream { struct SfHeaders { size_t rc; ScrArr *pairs; + /* Fetch's "sort and combine" list of `pairs` as flat name/value pairs, + * built on the first iteration step and dropped by every mutation. */ + ScrArr *sorted; bool immutable; + /* `pairs` is in the form sf_add_converted_headers builds: lowercase valid + * names, trimmed valid UTF-8 values, one combined pair per name except + * set-cookie. Headers built from wire lines are not (duplicates stay). */ + bool canonical; }; struct SfResponse { @@ -2164,6 +2171,24 @@ static ScrDyn *sf_controller_get(void *ptr, const char *key, size_t len) { static bool sf_header_name_ok(const char *s, size_t len); +/* A borrowed element of a runtime-built header pair array. These arrays are + * only ever appended to or rewritten whole, so they are dense. */ +static inline ScrStr *sf_pair_at(ScrArr *pairs, size_t i) { + if (i < pairs->filled) return (ScrStr *)(uintptr_t)scr_arr_dense_get(pairs, i); + return (ScrStr *)scr_arr_peek_ref(pairs, (double)i); +} + +static ScrStr *sf_header_name_lower(const char *data, size_t len); + +/* `name` in lowercase, +1: the same string when it has no uppercase byte. */ +static ScrStr *sf_header_name_lower_str(ScrStr *name) { + for (size_t i = 0; i < name->len; i++) { + char c = name->data[i]; + if (c >= 'A' && c <= 'Z') return sf_header_name_lower(name->data, name->len); + } + return scr_str_retain(name); +} + static ScrStr *sf_header_name_lower(const char *data, size_t len) { char *lower = malloc(len ? len : 1); if (!lower) sf_oom(); @@ -2232,13 +2257,8 @@ static SfHeaders *sf_headers_new_response(ScrArr *raw_pairs) { size_t n = (size_t)scr_arr_len(raw_pairs); ScrArr *pairs = scr_arr_new(SCR_ELEM_STR, n); for (size_t i = 0; i + 1 < n; i += 2) { - ScrStr *raw_name = scr_arr_get_ref(raw_pairs, (double)i); - scr_arr_push_ref( - pairs, sf_header_name_lower(raw_name->data, raw_name->len)); - scr_str_release(raw_name); - ScrStr *raw = scr_arr_get_ref(raw_pairs, (double)(i + 1)); - scr_arr_push_ref(pairs, sf_latin1_to_utf8(raw)); - scr_str_release(raw); + scr_arr_push_ref(pairs, sf_header_name_lower_str(sf_pair_at(raw_pairs, i))); + scr_arr_push_ref(pairs, sf_latin1_to_utf8(sf_pair_at(raw_pairs, i + 1))); } scr_arr_release(raw_pairs); return sf_headers_new(pairs, true); @@ -2252,9 +2272,17 @@ static SfHeaders *sf_headers_retain(SfHeaders *h) { static void sf_headers_release(SfHeaders *h) { if (!h || --h->rc > 0) return; scr_arr_release(h->pairs); + if (h->sorted) scr_arr_release(h->sorted); free(h); } +/* Every change to h->pairs goes through here first. */ +static void sf_headers_changed(SfHeaders *h) { + if (!h->sorted) return; + scr_arr_release(h->sorted); + h->sorted = NULL; +} + static void *sf_headers_retain_v(void *p) { return sf_headers_retain((SfHeaders *)p); } @@ -2294,7 +2322,7 @@ static ScrStr *sf_headers_name(const ScrDyn *value, const char *method) { scr_str_release(raw); return NULL; } - ScrStr *out = sf_header_name_lower(raw->data, raw->len); + ScrStr *out = sf_header_name_lower_str(raw); scr_str_release(raw); return out; } @@ -2305,40 +2333,36 @@ static bool sf_headers_key_eq(const ScrStr *left, const ScrStr *right) { } static ScrStr *sf_headers_get_value(SfHeaders *h, const ScrStr *name) { - size_t n = (size_t)scr_arr_len(h->pairs); + ScrArr *pairs = h->pairs; + size_t n = (size_t)scr_arr_len(pairs); size_t count = 0; size_t total = 0; + ScrStr *first = NULL; for (size_t i = 0; i + 1 < n; i += 2) { - ScrStr *key = scr_arr_get_ref(h->pairs, (double)i); - ScrStr *value = scr_arr_get_ref(h->pairs, (double)(i + 1)); - if (sf_headers_key_eq(key, name)) { - if (count > 0) total += 2; - total += value->len; - count++; - } - scr_str_release(key); - scr_str_release(value); + if (!sf_headers_key_eq(sf_pair_at(pairs, i), name)) continue; + ScrStr *value = sf_pair_at(pairs, i + 1); + if (count > 0) total += 2; + else first = value; + total += value->len; + count++; } if (count == 0) return NULL; + if (count == 1) return scr_str_retain(first); char *joined = malloc(total > 0 ? total : 1); if (!joined) sf_oom(); size_t at = 0; size_t emitted = 0; for (size_t i = 0; i + 1 < n; i += 2) { - ScrStr *key = scr_arr_get_ref(h->pairs, (double)i); - ScrStr *value = scr_arr_get_ref(h->pairs, (double)(i + 1)); - if (sf_headers_key_eq(key, name)) { - if (emitted > 0) { - joined[at++] = ','; - joined[at++] = ' '; - } - memcpy(joined + at, value->data, value->len); - at += value->len; - emitted++; + if (!sf_headers_key_eq(sf_pair_at(pairs, i), name)) continue; + ScrStr *value = sf_pair_at(pairs, i + 1); + if (emitted > 0) { + joined[at++] = ','; + joined[at++] = ' '; } - scr_str_release(key); - scr_str_release(value); + memcpy(joined + at, value->data, value->len); + at += value->len; + emitted++; } ScrStr *out = scr_str_new(joined, total); free(joined); @@ -2348,17 +2372,17 @@ static ScrStr *sf_headers_get_value(SfHeaders *h, const ScrStr *name) { static void sf_headers_delete_name(SfHeaders *h, const ScrStr *name) { ScrArr *old = h->pairs; size_t n = (size_t)scr_arr_len(old); + bool present = false; + for (size_t i = 0; i + 1 < n && !present; i += 2) + present = sf_headers_key_eq(sf_pair_at(old, i), name); + if (!present) return; + sf_headers_changed(h); ScrArr *next = scr_arr_new(SCR_ELEM_STR, n); for (size_t i = 0; i + 1 < n; i += 2) { - ScrStr *key = scr_arr_get_ref(old, (double)i); - ScrStr *value = scr_arr_get_ref(old, (double)(i + 1)); - if (sf_headers_key_eq(key, name)) { - scr_str_release(key); - scr_str_release(value); - continue; - } - scr_arr_push_ref(next, key); - scr_arr_push_ref(next, value); + ScrStr *key = sf_pair_at(old, i); + if (sf_headers_key_eq(key, name)) continue; + scr_arr_push_ref(next, scr_str_retain(key)); + scr_arr_push_ref(next, scr_str_retain(sf_pair_at(old, i + 1))); } h->pairs = next; scr_arr_release(old); @@ -2399,15 +2423,13 @@ static bool sf_headers_convert_pair( * Fetch-specific semicolon separator for Cookie. */ static void sf_headers_append_pair( SfHeaders *h, ScrStr *name, ScrStr *value) { + sf_headers_changed(h); if (!(name->len == 10 && memcmp(name->data, "set-cookie", 10) == 0)) { size_t n = (size_t)scr_arr_len(h->pairs); for (size_t i = 0; i + 1 < n; i += 2) { - ScrStr *key = scr_arr_get_ref(h->pairs, (double)i); - bool match = sf_headers_key_eq(key, name); - scr_str_release(key); - if (!match) continue; + if (!sf_headers_key_eq(sf_pair_at(h->pairs, i), name)) continue; - ScrStr *previous = scr_arr_get_ref(h->pairs, (double)(i + 1)); + ScrStr *previous = sf_pair_at(h->pairs, i + 1); const char *separator = name->len == 6 && memcmp(name->data, "cookie", 6) == 0 ? "; " @@ -2420,7 +2442,6 @@ static void sf_headers_append_pair( memcpy(joined_data + previous->len + 2, value->data, value->len); ScrStr *joined = scr_str_new(joined_data, joined_len); free(joined_data); - scr_str_release(previous); scr_str_release(name); scr_str_release(value); scr_arr_set_ref(h->pairs, (double)(i + 1), joined); @@ -2447,7 +2468,7 @@ static ScrStr **sf_headers_sorted_names(SfHeaders *h, size_t *count_out) { if (capacity && !names) sf_oom(); size_t count = 0; for (size_t i = 0; i + 1 < n; i += 2) { - ScrStr *key = scr_arr_get_ref(h->pairs, (double)i); + ScrStr *key = sf_pair_at(h->pairs, i); bool seen = false; for (size_t j = 0; j < count; j++) { if (sf_headers_key_eq(names[j], key)) { @@ -2455,8 +2476,7 @@ static ScrStr **sf_headers_sorted_names(SfHeaders *h, size_t *count_out) { break; } } - if (seen) scr_str_release(key); - else names[count++] = key; + if (!seen) names[count++] = key; } if (count > 1) { qsort(names, count, sizeof *names, &sf_headers_name_cmp); @@ -2469,44 +2489,44 @@ static bool sf_headers_is_set_cookie(const ScrStr *name) { return name->len == 10 && memcmp(name->data, "set-cookie", 10) == 0; } -/* The `index`th entry of Fetch's "sort and combine" list: names in - * byte order, each with its combined value, except that every set-cookie - * value is its own entry (in insertion order), as Node's Headers - * iterator yields them. False past the end. Outputs are +1. */ +/* Fetch's "sort and combine" list as flat name/value pairs: names in byte + * order, each with its combined value, except that every set-cookie value + * is its own entry (in insertion order), as Node's Headers iterator yields + * them. Cached until the next mutation; borrowed. */ +static ScrArr *sf_headers_sorted(SfHeaders *h) { + if (h->sorted) return h->sorted; + size_t count = 0; + ScrStr **names = sf_headers_sorted_names(h, &count); /* borrowed names */ + size_t n = (size_t)scr_arr_len(h->pairs); + ScrArr *sorted = scr_arr_new(SCR_ELEM_STR, n); + for (size_t i = 0; i < count; i++) { + if (!sf_headers_is_set_cookie(names[i])) { + scr_arr_push_ref(sorted, scr_str_retain(names[i])); + scr_arr_push_ref(sorted, sf_headers_get_value(h, names[i])); + continue; + } + for (size_t j = 0; j + 1 < n; j += 2) { + if (!sf_headers_is_set_cookie(sf_pair_at(h->pairs, j))) continue; + scr_arr_push_ref(sorted, scr_str_retain(names[i])); + scr_arr_push_ref(sorted, scr_str_retain(sf_pair_at(h->pairs, j + 1))); + } + } + free(names); + h->sorted = sorted; + return sorted; +} + +/* The `index`th entry of the sort-and-combine list (sf_headers_sorted). + * False past the end. Outputs are +1. */ static bool sf_headers_sorted_entry(SfHeaders *h, size_t index, ScrStr **name_out, ScrStr **value_out) { - size_t count = 0; - ScrStr **names = sf_headers_sorted_names(h, &count); - bool found = false; - size_t at = 0; - for (size_t i = 0; i < count && !found; i++) { - if (!sf_headers_is_set_cookie(names[i])) { - if (at == index) { - *name_out = scr_str_retain(names[i]); - *value_out = sf_headers_get_value(h, names[i]); - found = true; - } - at++; - continue; - } - size_t n = (size_t)scr_arr_len(h->pairs); - for (size_t j = 0; j + 1 < n && !found; j += 2) { - ScrStr *key = scr_arr_get_ref(h->pairs, (double)j); - bool match = sf_headers_is_set_cookie(key); - scr_str_release(key); - if (!match) continue; - if (at == index) { - *name_out = scr_str_retain(names[i]); - *value_out = scr_arr_get_ref(h->pairs, (double)(j + 1)); - found = true; - } - at++; - } - } - for (size_t j = 0; j < count; j++) scr_str_release(names[j]); - free(names); - return found; + ScrArr *sorted = sf_headers_sorted(h); + size_t entries = (size_t)scr_arr_len(sorted) / 2; + if (index >= entries) return false; + *name_out = scr_str_retain(sf_pair_at(sorted, 2 * index)); + *value_out = scr_str_retain(sf_pair_at(sorted, 2 * index + 1)); + return true; } /* One step of a live Headers iterator (entries, keys or values): the @@ -2570,14 +2590,8 @@ static ScrDyn *sf_headers_invoke(void *ptr, ScrDyn *self, ScrDyn *out = scr_dyn_new_arr(); size_t n = (size_t)scr_arr_len(h->pairs); for (size_t i = 0; i + 1 < n; i += 2) { - ScrStr *key = scr_arr_get_ref(h->pairs, (double)i); - ScrStr *value = scr_arr_get_ref(h->pairs, (double)(i + 1)); - if (key->len == 10 && - memcmp(key->data, "set-cookie", 10) == 0) { - scr_dyn_arr_push(out, scr_dyn_new_str(value)); - } - scr_str_release(key); - scr_str_release(value); + if (sf_headers_is_set_cookie(sf_pair_at(h->pairs, i))) + scr_dyn_arr_push(out, scr_dyn_new_str(sf_pair_at(h->pairs, i + 1))); } return out; } @@ -3323,14 +3337,13 @@ static bool sf_pairs_have(ScrArr *pairs, const char *name) { size_t n = (size_t)scr_arr_len(pairs); size_t want = strlen(name); for (size_t i = 0; i + 1 < n; i += 2) { - ScrStr *key = (ScrStr *)scr_arr_get_ref(pairs, (double)i); + ScrStr *key = sf_pair_at(pairs, i); bool same = key->len == want; for (size_t j = 0; j < want && same; j++) { char c = key->data[j]; if (c >= 'A' && c <= 'Z') c = (char)(c - 'A' + 'a'); same = c == name[j]; } - scr_str_release(key); if (same) return true; } return false; @@ -3437,11 +3450,14 @@ static ScrStr *sf_trim_header_value(const ScrStr *value) { (value->data[end - 1] == ' ' || value->data[end - 1] == '\t')) { end--; } + if (start == 0 && end == value->len) return scr_str_retain((ScrStr *)value); return scr_str_new(value->data + start, end - start); } -static bool sf_push_header(ScrArr *pairs, const char *name, size_t name_len, +static bool sf_push_header(ScrArr *pairs, ScrStr *name_str, ScrStr *value, bool request_guard) { + const char *name = name_str->data; + size_t name_len = name_str->len; if (!sf_header_name_ok(name, name_len)) { ScrStr *display = scr_str_new(name, name_len); sf_headers_invalid("append", display, "name"); @@ -3467,7 +3483,7 @@ static bool sf_push_header(ScrArr *pairs, const char *name, size_t name_len, if (!sf_header_name_ci(name, name_len, "set-cookie")) { size_t n = (size_t)scr_arr_len(pairs); for (size_t i = 0; i + 1 < n; i += 2) { - ScrStr *key = scr_arr_get_ref(pairs, (double)i); + ScrStr *key = sf_pair_at(pairs, i); bool match = key->len == name_len; for (size_t j = 0; j < name_len && match; j++) { unsigned char a = (unsigned char)key->data[j]; @@ -3476,11 +3492,9 @@ static bool sf_push_header(ScrArr *pairs, const char *name, size_t name_len, if (b >= 'A' && b <= 'Z') b = (unsigned char)(b - 'A' + 'a'); match = a == b; } - scr_str_release(key); if (!match) continue; - ScrStr *previous = - scr_arr_get_ref(pairs, (double)(i + 1)); + ScrStr *previous = sf_pair_at(pairs, i + 1); const char *separator = sf_header_name_ci(name, name_len, "cookie") ? "; " : ", "; size_t separator_len = 2; @@ -3494,7 +3508,6 @@ static bool sf_push_header(ScrArr *pairs, const char *name, size_t name_len, normalized->data, normalized->len); ScrStr *joined = scr_str_new(joined_data, joined_len); free(joined_data); - scr_str_release(previous); scr_str_release(normalized); if (request_guard && !sf_request_header_ok(name, name_len, joined)) { scr_str_release(joined); @@ -3513,8 +3526,8 @@ static bool sf_push_header(ScrArr *pairs, const char *name, size_t name_len, scr_arr_push_ref( pairs, request_guard - ? scr_str_new(name, name_len) - : sf_header_name_lower(name, name_len)); + ? scr_str_retain(name_str) + : sf_header_name_lower_str(name_str)); scr_arr_push_ref(pairs, normalized); return true; } @@ -3541,10 +3554,13 @@ static void sf_headers_pair_length_error(size_t len) { * leaves header-name/value syntax validation for initialization. Response * construction converts this snapshot before status/statusText and only * validates it after body extraction, matching Undici's two phases. */ -static ScrArr *sf_headers_convert_init(const ScrDyn *headers) { +/* *canonical_out reports that the result copies a canonical Headers + * object's pairs (SfHeaders.canonical). */ +static ScrArr *sf_headers_convert_init(const ScrDyn *headers, bool *canonical_out) { + *canonical_out = false; if (headers && headers->kind == SCR_DYN_TYPED_REF) { ScrDyn *materialized = scr_dyn_typed_ref_materialize(headers); - ScrArr *converted = sf_headers_convert_init(materialized); + ScrArr *converted = sf_headers_convert_init(materialized, canonical_out); scr_dyn_release(materialized); return converted; } @@ -3635,13 +3651,9 @@ static ScrArr *sf_headers_convert_init(const ScrDyn *headers) { headers->v.handle.tag == SCR_DYNH_FETCH_HEADERS) { SfHeaders *source = (SfHeaders *)headers->v.handle.ptr; size_t n = (size_t)scr_arr_len(source->pairs); - for (size_t i = 0; i + 1 < n; i += 2) { - ScrStr *name = scr_arr_get_ref(source->pairs, (double)i); - ScrStr *value = - scr_arr_get_ref(source->pairs, (double)(i + 1)); - scr_arr_push_ref(converted, name); - scr_arr_push_ref(converted, value); - } + for (size_t i = 0; i < n - n % 2; i++) + scr_arr_push_ref(converted, scr_str_retain(sf_pair_at(source->pairs, i))); + *canonical_out = source->canonical; return converted; } /* Functions are WebIDL objects and an ordinary function has no @@ -3656,25 +3668,29 @@ fail: } static bool sf_add_converted_headers( - ScrArr *pairs, ScrArr *converted, bool request_guard) { + ScrArr *pairs, ScrArr *converted, bool request_guard, bool canonical) { size_t n = (size_t)scr_arr_len(converted); + /* Pushing canonical pairs into an empty list one by one through + * sf_push_header would reproduce them exactly. */ + if (canonical && !request_guard && scr_arr_len(pairs) == 0) { + for (size_t i = 0; i < n; i++) + scr_arr_push_ref(pairs, scr_str_retain(sf_pair_at(converted, i))); + return true; + } for (size_t i = 0; i + 1 < n; i += 2) { - ScrStr *name = scr_arr_get_ref(converted, (double)i); - ScrStr *value = scr_arr_get_ref(converted, (double)(i + 1)); - bool ok = sf_push_header( - pairs, name->data, name->len, value, request_guard); - scr_str_release(name); - scr_str_release(value); - if (!ok) return false; + if (!sf_push_header(pairs, sf_pair_at(converted, i), + sf_pair_at(converted, i + 1), request_guard)) + return false; } return true; } static bool sf_add_headers( ScrArr *pairs, const ScrDyn *headers, bool request_guard) { - ScrArr *converted = sf_headers_convert_init(headers); + bool canonical = false; + ScrArr *converted = sf_headers_convert_init(headers, &canonical); if (!converted) return false; - bool ok = sf_add_converted_headers(pairs, converted, request_guard); + bool ok = sf_add_converted_headers(pairs, converted, request_guard, canonical); scr_arr_release(converted); return ok; } @@ -4854,6 +4870,7 @@ static ScrDyn *sf_response_construct(ScrDyn *body, ScrDyn *init, SfJsonSerialize serialize) { ScrArr *header_pairs = NULL; ScrArr *converted_headers = NULL; + bool converted_canonical = false; ScrBytes *body_bytes = NULL; ScrStr *status_text = NULL; SfStream *stream = NULL; @@ -4918,7 +4935,7 @@ static ScrDyn *sf_response_construct(ScrDyn *body, ScrDyn *init, * later response-initialization step, after body extraction. */ ScrDyn *init_headers = sf_response_init_get(init, "headers", 7); if (light_ok) light_form = sf_light_header_form(init_headers, json_body, &light_record); - converted_headers = sf_headers_convert_init(init_headers); + converted_headers = sf_headers_convert_init(init_headers, &converted_canonical); scr_dyn_release(init_headers); if (!converted_headers) goto fail; @@ -5013,7 +5030,7 @@ static ScrDyn *sf_response_construct(ScrDyn *body, ScrDyn *init, header_pairs = scr_arr_new(SCR_ELEM_STR, 8); if (!sf_add_converted_headers( - header_pairs, converted_headers, false)) goto fail; + header_pairs, converted_headers, false, converted_canonical)) goto fail; scr_arr_release(converted_headers); converted_headers = NULL; @@ -5044,6 +5061,7 @@ static ScrDyn *sf_response_construct(ScrDyn *body, ScrDyn *init, response->rc = 1; response->body = stream; response->headers = sf_headers_new(header_pairs, false); + response->headers->canonical = true; response->url = scr_str_new("", 0); response->status_text = status_text; response->status = status; @@ -5159,13 +5177,15 @@ static ScrDyn *sf_request_invoke(void *ptr, ScrDyn *self, const char *method, Sc } ScrDyn *scr_fetch_headers_new(ScrDyn *init) { - ScrArr *converted = sf_headers_convert_init(init); + bool canonical = false; + ScrArr *converted = sf_headers_convert_init(init, &canonical); if (!converted) return NULL; ScrArr *pairs = scr_arr_new(SCR_ELEM_STR, 8); - bool ok = sf_add_converted_headers(pairs, converted, false); + bool ok = sf_add_converted_headers(pairs, converted, false, canonical); scr_arr_release(converted); if (!ok) { scr_arr_release(pairs); return NULL; } SfHeaders *headers = sf_headers_new(pairs, false); + headers->canonical = true; ScrDyn *result = scr_dyn_new_handle(headers, SCR_DYNH_FETCH_HEADERS); sf_headers_release(headers); return result; diff --git a/tests/corpus/fetch-headers-copy-cache.ts b/tests/corpus/fetch-headers-copy-cache.ts new file mode 100644 index 000000000..686ecfc61 --- /dev/null +++ b/tests/corpus/fetch-headers-copy-cache.ts @@ -0,0 +1,85 @@ +// Headers copies and the cached sort-and-combine list: copies are +// independent of their source, reads see every mutation, and live +// iterators re-read the current list after a change. +function dump(label: string, h: Headers): void { + const parts: string[] = []; + h.forEach((value, name) => parts.push(`${name}=${value}`)); + console.log(`${label}: ${parts.join(" | ")}`); +} + +// The corpus lib has no DOM iterable typings for Headers, so iterators +// are reached through an untyped view. +const iter = (h: Headers): any => h; + +const source = new Headers({ "X-B": " two ", "x-a": "one", Accept: "text/html" }); +source.append("accept", "application/json"); +source.append("Set-Cookie", "a=1"); +source.append("set-cookie", "b=2"); +dump("source", source); + +// A Headers copy keeps the combined values and stays separate. +const copy = new Headers(source); +source.set("x-a", "changed"); +copy.append("x-c", "three"); +dump("source after set", source); +dump("copy after append", copy); +console.log(JSON.stringify(copy.get("accept")), JSON.stringify(copy.get("set-cookie"))); +console.log(JSON.stringify(copy.getSetCookie())); + +// Copying into an empty Response and a Request, then appending a repeat. +const response = new Response("body", { headers: copy, status: 201 }); +response.headers.append("accept", "*/*"); +response.headers.append("cookie", "c=3"); +response.headers.append("Cookie", "d=4"); +dump("response", response.headers); +dump("copy unchanged", copy); +const clone = new Response(response.body, response); +dump("response clone", clone.headers); +const request = new Request("http://example.test/", { headers: copy }); +console.log("request x-b", JSON.stringify(request.headers.get("X-B"))); + +// Every read after a mutation sees it: get, has, iteration, spread. +const h = new Headers([["x-z", "z"], ["x-m", "m"]]); +console.log([...iter(h).entries()].map(([n, v]) => `${n}:${v}`).join(",")); +h.delete("x-z"); +console.log([...iter(h).keys()].join(","), h.has("x-z")); +h.delete("x-missing"); +console.log([...iter(h).values()].join(",")); +h.append("x-a", "first"); +console.log([...iter(h).entries()].map(([n, v]) => `${n}:${v}`).join(",")); +h.set("X-M", "again"); +console.log(JSON.stringify(h.get("x-m")), [...iter(h).keys()].join(",")); + +// A live iterator: mutations between steps change what comes next. +const it = iter(h).entries(); +console.log(JSON.stringify(it.next().value)); +h.append("x-0", "before"); +console.log(JSON.stringify(it.next().value)); +h.delete("x-m"); +h.append("x-zz", "after"); +console.log(JSON.stringify(it.next()), JSON.stringify(it.next())); + +// forEach that mutates as it walks. +const walk = new Headers([["b", "2"], ["d", "4"]]); +const seen: string[] = []; +walk.forEach((value, name) => { + seen.push(`${name}=${value}`); + if (name === "b") { + walk.append("c", "3"); + walk.append("a", "1"); + } + if (name === "c") walk.set("d", "four"); +}); +console.log(seen.join(" | ")); + +// Invalid input in a copy source is still rejected (plain object source). +try { + new Response("x", { headers: { "bad name": "v" } }); +} catch (error) { + console.log((error as Error).name, (error as Error).message); +} +try { + new Headers(source).append("x", "a\nb"); +} catch (error) { + console.log((error as Error).name, (error as Error).message); +} diff --git a/tests/harness/corpus-costs/fetch-headers-copy-cache.ts.json b/tests/harness/corpus-costs/fetch-headers-copy-cache.ts.json new file mode 100644 index 000000000..6057503d9 --- /dev/null +++ b/tests/harness/corpus-costs/fetch-headers-copy-cache.ts.json @@ -0,0 +1 @@ +{ "plain": 6400, "san": 9200 } From 7b8a07e0f296d88b2c99b13158a1baa93451bb69 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Fri, 23 Oct 2026 07:37:41 -0700 Subject: [PATCH 26/53] Search proven-ASCII strings backwards by byte in lastIndexOf lastIndexOf built the receiver's UTF-16 index map and converted positions both ways even when the receiver is proven ASCII, where byte offsets are UTF-16 indexes (indexOf already has this fast path). A short id such as "bk_00042".lastIndexOf("_") cost about 200 instructions; the Hono fixture's list endpoint does it 80 times per request. Co-Authored-By: Claude Opus 5.5 --- .../corpus/string-last-index-of-ascii.ts.json | 6 ++++++ packages/runtime/src/scr_string.c | 16 ++++++++++++++ tests/corpus/string-last-index-of-ascii.ts | 21 +++++++++++++++++++ .../string-last-index-of-ascii.ts.json | 1 + 4 files changed, 44 insertions(+) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/string-last-index-of-ascii.ts.json create mode 100644 tests/corpus/string-last-index-of-ascii.ts create mode 100644 tests/harness/corpus-costs/string-last-index-of-ascii.ts.json diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/string-last-index-of-ascii.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/string-last-index-of-ascii.ts.json new file mode 100644 index 000000000..1d3107cd5 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/string-last-index-of-ascii.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/string-last-index-of-ascii.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_string.c b/packages/runtime/src/scr_string.c index f94554642..37b92e48c 100644 --- a/packages/runtime/src/scr_string.c +++ b/packages/runtime/src/scr_string.c @@ -1928,6 +1928,22 @@ static __attribute__((noinline)) double scr_str_unit_search(const ScrStr *s, con /* Search backwards from a clamped UTF-16 position. A low-surrogate position * maps to its scalar's first byte, so that scalar remains searchable. */ double scr_str_last_index_of_from(ScrStr *s, ScrStr *needle, double position) { + /* A proven-ASCII receiver: byte offsets are UTF-16 indexes, and a needle + * with other bytes (including lone surrogate halves) is never found. */ + if (s->ascii) { + size_t start = isnan(position) || position >= (double)s->len ? s->len + : position <= 0 ? 0 : (size_t)trunc(position); + if (needle->len == 0) return (double)start; + if (needle->len > s->len) return -1.0; + size_t max_start = s->len - needle->len; + if (start < max_start) max_start = start; + char first = needle->data[0]; + for (size_t i = max_start + 1; i-- > 0;) { + if (s->data[i] == first && memcmp(s->data + i, needle->data, needle->len) == 0) + return (double)i; + } + return -1.0; + } ScrSidx *e = scr_sidx_for(s); size_t len16 = scr_sidx_len(s, e); size_t start16 = isnan(position) || position >= (double)len16 ? len16 diff --git a/tests/corpus/string-last-index-of-ascii.ts b/tests/corpus/string-last-index-of-ascii.ts new file mode 100644 index 000000000..8dcf405f5 --- /dev/null +++ b/tests/corpus/string-last-index-of-ascii.ts @@ -0,0 +1,21 @@ +// lastIndexOf on ASCII receivers: every position form, empty and long +// needles, needles with non-ASCII text, and receivers built at run time. +const ids = ["bk_00042", "mb_00410", "plain", "_lead", "trail_", "a_b_c_d", ""]; +for (const id of ids) console.log(JSON.stringify(id), id.lastIndexOf("_"), Number(id.slice(id.lastIndexOf("_") + 1))); + +const text = "abcabcabc"; +const positions = [Infinity, -Infinity, NaN, -0, 0, 1, 2.9, 3, 5, 8, 9, 100, -5]; +for (const needle of ["abc", "c", "a", "", "abcabcabcabc", "cab", "x", "é", "\ud83d", "😀"]) { + const row = positions.map((position) => text.lastIndexOf(needle, position)); + console.log(JSON.stringify(needle), text.lastIndexOf(needle), row.join(",")); +} + +// Receivers built at run time (concatenation, repeat, join, slices). +const built = ["x", "y"].join("-").repeat(3) + String(42); +console.log(built, built.lastIndexOf("-"), built.lastIndexOf("x-", 3), built.lastIndexOf("42", 6)); +const slice = built.slice(2, 9); +console.log(slice, slice.lastIndexOf("y"), slice.lastIndexOf("y", 0), slice.lastIndexOf("-y-")); + +// A non-ASCII receiver keeps UTF-16 positions. +const mixed = "é_ü_😀_z"; +console.log(mixed.lastIndexOf("_"), mixed.lastIndexOf("_", 3), mixed.lastIndexOf("😀"), mixed.lastIndexOf("\ude00")); diff --git a/tests/harness/corpus-costs/string-last-index-of-ascii.ts.json b/tests/harness/corpus-costs/string-last-index-of-ascii.ts.json new file mode 100644 index 000000000..a970c2f16 --- /dev/null +++ b/tests/harness/corpus-costs/string-last-index-of-ascii.ts.json @@ -0,0 +1 @@ +{ "plain": 6000, "san": 9000 } From a705a3fd09b72b22ae744900fa90e8c88c45cfe0 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Fri, 23 Oct 2026 07:35:51 -0700 Subject: [PATCH 27/53] Reject other classes' capsules inline in class-table brand probes A class table (property bags, normalization, computed reads) tests the receiver's capsule key against every compiled class in turn. A capsule boxed by another class has another key pointer, and each such probe called scr_dyn_typed_ref_is_key to compare the names. The probe now rejects it inline when the key length or its last eight bytes differ, and calls the out-of-line comparison only for a likely match, so capsules boxed with an equal key elsewhere still match. Co-Authored-By: Claude Opus 5.5 --- .../compiler/src/backend/llvm/emitter.test.ts | 21 ++++++ .../compiler/src/backend/llvm/lib-dispatch.ts | 35 ++++++++++ .../corpus/class-brand-probe-keys.ts.json | 6 ++ packages/runtime/src/scr_dyn.h | 4 ++ tests/corpus/class-brand-probe-keys.ts | 68 +++++++++++++++++++ .../class-brand-probe-keys.ts.json | 1 + 6 files changed, 135 insertions(+) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/class-brand-probe-keys.ts.json create mode 100644 tests/corpus/class-brand-probe-keys.ts create mode 100644 tests/harness/corpus-costs/class-brand-probe-keys.ts.json diff --git a/packages/compiler/src/backend/llvm/emitter.test.ts b/packages/compiler/src/backend/llvm/emitter.test.ts index 4793663aa..e6334ab4c 100644 --- a/packages/compiler/src/backend/llvm/emitter.test.ts +++ b/packages/compiler/src/backend/llvm/emitter.test.ts @@ -499,6 +499,27 @@ test("class brand probes compare the program's key constant before the key text" const named = work(probe("dyn.typedRefIs"), dyn); expect(named).toContain("; ->v.typed_ref.type_key"); expect(named).toContain("@scr_dyn_typed_ref_is_key"); + // Another key pointer is rejected on its length and last eight bytes + // ("Example" plus the prefix's ':') before the out-of-line comparison. + expect(named).toContain("; ->v.typed_ref.type_key_len"); + expect(named).toContain("and i32 %t"); + expect(named).toMatch(/icmp eq i32 %t\d+, 14\b/); + const tail = Buffer.from("object:Example").readBigInt64LE(6); + expect(named).toContain(`icmp eq i64 %t`); + expect(named).toContain(`, ${tail}`); + const shortKey: IrExpr = { kind: "strLit", value: "o:E", type: STRING, loc: receiverLoc }; + const short = work( + { + kind: "libCall", + fn: "dyn.typedRefIs", + args: [ref(dyn), shortKey], + type: BOOL, + loc: receiverLoc, + }, + dyn, + ); + expect(short).toMatch(/icmp eq i32 %t\d+, 3\b/); + expect(short).not.toContain("load i64"); // The key-pointer test answers false for capsules boxed elsewhere. const exact = work(probe("dyn.typedRefKeyIs"), dyn); expect(exact).toContain("; ->v.typed_ref.type_key"); diff --git a/packages/compiler/src/backend/llvm/lib-dispatch.ts b/packages/compiler/src/backend/llvm/lib-dispatch.ts index eaf8f59f6..3f127e7e9 100644 --- a/packages/compiler/src/backend/llvm/lib-dispatch.ts +++ b/packages/compiler/src/backend/llvm/lib-dispatch.ts @@ -25,6 +25,9 @@ import { rawBytes } from "./lib-abi.js"; import { emitInlineAtomicsOp } from "./expr-bytes.js"; import { regexLiteralCaptures } from "./regex-matchers.js"; +/** ScrDyn.v.typed_ref.type_key_len: the low 27 bits of its bit-field word. */ +const TYPE_KEY_LEN_MASK = (1 << 27) - 1; + export function emitAssertInspectLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlValue { const B = host.B; if (e.fn === "assert.shapeStr" || e.fn === "assert.shapeRe") { @@ -281,6 +284,38 @@ export function emitGenericLibCall(host: LlvmEmitterContext, e: LibCallExpr): Ll B.line(`store i1 true, ptr ${slot}`); B.condBr(same, done, compare); B.startBlock(compare); + // A capsule with another key pointer usually belongs to another + // class: reject it on its key length and last eight bytes before + // the out-of-line name comparison, so a class table's chain of + // probes costs a few instructions per class it skips. + B.line(`store i1 false, ptr ${slot}`); + const bytes = Buffer.from(literal, "utf8"); + const lenSlot = B.tmp(), + lenWord = B.tmp(), + len = B.tmp(), + lenEq = B.tmp(); + B.line( + `${lenSlot} = getelementptr inbounds i8, ptr ${value.name}, i64 ${host.abiOffset(88, 52)} ; ->v.typed_ref.type_key_len`, + ); + B.line(`${lenWord} = load i32, ptr ${lenSlot}`); + B.line(`${len} = and i32 ${lenWord}, ${TYPE_KEY_LEN_MASK}`); + B.line(`${lenEq} = icmp eq i32 ${len}, ${bytes.length}`); + const full = B.newLabel("brand.full"); + if (bytes.length >= 8) { + const tail = B.newLabel("brand.tail"); + B.condBr(lenEq, tail, done); + B.startBlock(tail); + const tailPtr = B.tmp(), + word = B.tmp(), + wordEq = B.tmp(); + B.line(`${tailPtr} = getelementptr inbounds i8, ptr ${keyPtr}, i64 ${bytes.length - 8}`); + B.line(`${word} = load i64, ptr ${tailPtr}, align 1`); + B.line(`${wordEq} = icmp eq i64 ${word}, ${bytes.readBigInt64LE(bytes.length - 8)}`); + B.condBr(wordEq, full, done); + } else { + B.condBr(lenEq, full, done); + } + B.startBlock(full); } } if (e.fn === "dyn.typedRefIs") { diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/class-brand-probe-keys.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/class-brand-probe-keys.ts.json new file mode 100644 index 000000000..fd60e3ff9 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/class-brand-probe-keys.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/class-brand-probe-keys.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_dyn.h b/packages/runtime/src/scr_dyn.h index 669f4aedb..6f0c28371 100644 --- a/packages/runtime/src/scr_dyn.h +++ b/packages/runtime/src/scr_dyn.h @@ -540,6 +540,10 @@ _Static_assert(offsetof(ScrDyn, null_proto) == sizeof(size_t) + sizeof(ScrDynKin "dyn-to-record conversions read null_proto at byte 13 (9 on wasm32)"); _Static_assert(offsetof(ScrDyn, v.typed_ref.type_key) == 16 + 3 * sizeof(void *), "typed_ref.type_key offset"); +/* Emitted brand probes read type_key_len as the low 27 bits of the 32-bit + * word after `casts` (native 88, wasm32 52). */ +_Static_assert(offsetof(ScrDyn, v.typed_ref.casts) == 16 + 8 * sizeof(void *), + "typed_ref.casts offset"); /* The region links ride in payload space the capsule variant already * reserves, so dyn nodes keep their size (and allocation size class). */ _Static_assert(sizeof(((ScrDyn *)0)->v.arr) <= sizeof(((ScrDyn *)0)->v.typed_ref) && diff --git a/tests/corpus/class-brand-probe-keys.ts b/tests/corpus/class-brand-probe-keys.ts new file mode 100644 index 000000000..77b5ff138 --- /dev/null +++ b/tests/corpus/class-brand-probe-keys.ts @@ -0,0 +1,68 @@ +// Untyped reads of class instances dispatch on the capsule's class key. +// These classes have keys of equal length, keys that share their last +// eight bytes, and short keys, so every stage of the brand probe decides. +class FooHandlerA { + kind = "foo-a"; + label(): string { + return `FooHandlerA:${this.kind}`; + } +} +class BarHandlerA { + kind = "bar-a"; + label(): string { + return `BarHandlerA:${this.kind}`; + } +} +class FooHandlerB extends FooHandlerA { + override kind = "foo-b"; + extra = 2; +} +class Q { + kind = "q"; + label(): string { + return "Q"; + } +} +class R { + kind = "r"; + label(): string { + return "R"; + } +} +class LongerClassNameWithSharedTail { + kind = "long"; + label(): string { + return "long"; + } +} +class OtherClassNameWithSharedTail { + kind = "other"; + label(): string { + return "other"; + } +} + +const values: any[] = [ + new FooHandlerA(), + new BarHandlerA(), + new FooHandlerB(), + new Q(), + new R(), + new LongerClassNameWithSharedTail(), + new OtherClassNameWithSharedTail(), + { kind: "plain", label: () => "plain" }, +]; + +for (const value of values) { + const extra = value.extra; + console.log(value.kind, value.label(), extra === undefined ? "-" : extra, value.constructor.name); +} + +// Writes through the untyped view reach the right instance. +for (const value of values) value.kind = `${value.kind}!`; +console.log(values.map((value) => value.kind).join(",")); + +// Computed reads go through the same tables. +const key: string = ["kind", "label"][values.length % 2]; +console.log(values.map((value) => typeof value[key]).join(",")); +console.log(values.filter((value) => value instanceof FooHandlerA).length); diff --git a/tests/harness/corpus-costs/class-brand-probe-keys.ts.json b/tests/harness/corpus-costs/class-brand-probe-keys.ts.json new file mode 100644 index 000000000..9f464b312 --- /dev/null +++ b/tests/harness/corpus-costs/class-brand-probe-keys.ts.json @@ -0,0 +1 @@ +{ "plain": 8500, "san": 30000 } From 6ebdec28e22eb8111a5de0d82c44c7d250a33b94 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 10:14:37 -0500 Subject: [PATCH 28/53] Update tsc-ts to 0.0.9 tsc-ts 0.0.9 answers getDiagnosticsOfCheckedFiles, which checks only the listed files of a program and answers their semantic diagnostics and the global diagnostics. The release-age exclusions admit it with its platform packages. --- packages/compiler/package.json | 2 +- pnpm-lock.yaml | 66 +++++++++++++++++----------------- pnpm-workspace.yaml | 16 ++++----- 3 files changed, 42 insertions(+), 42 deletions(-) diff --git a/packages/compiler/package.json b/packages/compiler/package.json index a358ae545..a9e3d9616 100644 --- a/packages/compiler/package.json +++ b/packages/compiler/package.json @@ -40,7 +40,7 @@ }, "dependencies": { "@scriptc/runtime": "workspace:*", - "tsc-ts": "0.0.8", + "tsc-ts": "0.0.9", "typescript": "7.0.2", "typescript5": "npm:typescript@5.9.3" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 525994ba1..5fc32620a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -110,8 +110,8 @@ importers: specifier: workspace:* version: link:../runtime tsc-ts: - specifier: 0.0.8 - version: 0.0.8 + specifier: 0.0.9 + version: 0.0.9 typescript: specifier: 7.0.2 version: 7.0.2 @@ -1377,42 +1377,42 @@ packages: '@ts-morph/common@0.11.1': resolution: {integrity: sha512-7hWZS0NRpEsNV8vWJzg7FEz6V8MaLNeJOmwmghqUXTpzk16V1LLZhdo+4QvE/+zv4cVci0OviuJFnqhEfoV3+g==} - '@tsc-ts/darwin-arm64@0.0.8': - resolution: {integrity: sha512-+hB9yWk58lwZHGrVEzqEC4D7MQW7Vd2z7blV3TULIKZPTMwdf8aT7XFAmfP48PoGff0H9yuQKyRXJl9I+sQv9w==} + '@tsc-ts/darwin-arm64@0.0.9': + resolution: {integrity: sha512-aIjSulef3DIEX7F3RRWLXyHWFgjXiVffBkrfE83ay/tpbsQaniu16cFxYGYNaqqVGft/oivecpdTDBbpPlsb8Q==} cpu: [arm64] os: [darwin] - '@tsc-ts/darwin-x64@0.0.8': - resolution: {integrity: sha512-yz4UddicqMXe+T7nV7ftpf155lkiK6gIPNwSB8rH9dlwiV6a17CO/vpSk627QLMqnfGNicB6YwLMk/SJ9moPdw==} + '@tsc-ts/darwin-x64@0.0.9': + resolution: {integrity: sha512-LHE74tzIEDtrEN8ENvCH/ejnnZjT0D2Vaz2HmggcvJudgr0W300CcwGf2Kdfw8MVBBk6JN0HPF9+a5AUUtFZGg==} cpu: [x64] os: [darwin] - '@tsc-ts/linux-arm64-gnu@0.0.8': - resolution: {integrity: sha512-LHqfT9mQ4tusPdWmnanCvC8/YBOF5bHM5h5863a3t2HTgFVdps1Qwb0yzrW5DDbojIhQ0Z+GupxtXVUQcV0Q/Q==} + '@tsc-ts/linux-arm64-gnu@0.0.9': + resolution: {integrity: sha512-dNiQIKApLcKIJPzb3pk2gfGrrgmOWa+q3/qjrx5ia/SzIWz0+lENlVvHiIn9ukqDacDE30FW8xHzby8Kyd+10A==} cpu: [arm64] os: [linux] libc: [glibc] - '@tsc-ts/linux-arm64-musl@0.0.8': - resolution: {integrity: sha512-I0CC3xZMwoQM6OiVsubRHb0VI/RrJpneb33kDQhadYHq3taPos71KwP+CbCXNu88rNsDsyn7D1oCVfJb73g4Mw==} + '@tsc-ts/linux-arm64-musl@0.0.9': + resolution: {integrity: sha512-DolAqHtr1GUyG03Ch8+sJzpX1G+xzazAmj18kABzuOPre/6njrSWnmqJmtXQWvhwLBBEjajIkUJ4V+t5qigsrg==} cpu: [arm64] os: [linux] libc: [musl] - '@tsc-ts/linux-x64-gnu@0.0.8': - resolution: {integrity: sha512-PLERuwUuxT92x0BZAMQubiQP5jhlqoLmswaZWqcvfKAZyBp0TbWiVOmuNnDbJozvBaNUrfdGPb9suiQ+Me7M6A==} + '@tsc-ts/linux-x64-gnu@0.0.9': + resolution: {integrity: sha512-leANVZ4tKz3myBSVogb173ttbAurPOOGjy1WeBlRhkVJ7zOPMiyzJi3cPOAK/4O8RcxXOsCiC2AfKPKDYM8bEw==} cpu: [x64] os: [linux] libc: [glibc] - '@tsc-ts/linux-x64-musl@0.0.8': - resolution: {integrity: sha512-zDl36l1uyWc0mQniOAMZmkVD9TrlFExHlDT0fL8JGD4XLgx+bRe+To38yP0KSBehXBzjDNn8xrEoJ+3RP6mYwQ==} + '@tsc-ts/linux-x64-musl@0.0.9': + resolution: {integrity: sha512-a4FW16QXkrwNHAr4Qo/qDA5w9LxCJNrYbpqiz3TJ1sePTqvW/JTJc14tjx6avhW4mIaQp8AB52m7TqNAuDMm4Q==} cpu: [x64] os: [linux] libc: [musl] - '@tsc-ts/win32-x64-msvc@0.0.8': - resolution: {integrity: sha512-9gO+pFoLiIxITlAE2P6JL8ihZwns7z/2Raw5+38RqLzHmnSsRN47+H4p/4EqQsLW5/TTsAo9uSeSkf3EuRg+XA==} + '@tsc-ts/win32-x64-msvc@0.0.9': + resolution: {integrity: sha512-phThYnmOinlvY4dS4DEl1L9t/mFLiHR1zL5plSUfMB5NizA/jESfgMGRu+HW3uwOpCETSxgsuTNAXO7UuEIHIQ==} cpu: [x64] os: [win32] @@ -2577,8 +2577,8 @@ packages: ts-toolbelt@6.15.5: resolution: {integrity: sha512-FZIXf1ksVyLcfr7M317jbB67XFJhOO1YqdTcuGaq9q5jLUoTikukZ+98TPjKiP2jC5CgmYdWWYs0s2nLSU0/1A==} - tsc-ts@0.0.8: - resolution: {integrity: sha512-EiyDziYOryBg/YEvev/mKCn+hD4M6VFjH7VP9nv2WadDKYaPrdb+a3hP+wYr/UsDWNE7AdL7YGnjZppeXlXy6g==} + tsc-ts@0.0.9: + resolution: {integrity: sha512-7fDfC/o1YoITG6sSXj4ZznijQYg8AfoCagmYvwG1BFLR/hz4Lwnq7jqDwcZfoDuPYTkrs8jHUTAVG1turniJHA==} engines: {node: '>=20'} hasBin: true @@ -3454,25 +3454,25 @@ snapshots: mkdirp: 1.0.4 path-browserify: 1.0.1 - '@tsc-ts/darwin-arm64@0.0.8': + '@tsc-ts/darwin-arm64@0.0.9': optional: true - '@tsc-ts/darwin-x64@0.0.8': + '@tsc-ts/darwin-x64@0.0.9': optional: true - '@tsc-ts/linux-arm64-gnu@0.0.8': + '@tsc-ts/linux-arm64-gnu@0.0.9': optional: true - '@tsc-ts/linux-arm64-musl@0.0.8': + '@tsc-ts/linux-arm64-musl@0.0.9': optional: true - '@tsc-ts/linux-x64-gnu@0.0.8': + '@tsc-ts/linux-x64-gnu@0.0.9': optional: true - '@tsc-ts/linux-x64-musl@0.0.8': + '@tsc-ts/linux-x64-musl@0.0.9': optional: true - '@tsc-ts/win32-x64-msvc@0.0.8': + '@tsc-ts/win32-x64-msvc@0.0.9': optional: true '@tybys/wasm-util@0.10.3': @@ -4813,17 +4813,17 @@ snapshots: ts-toolbelt@6.15.5: {} - tsc-ts@0.0.8: + tsc-ts@0.0.9: dependencies: '@scriptc/threads': 0.4.2 optionalDependencies: - '@tsc-ts/darwin-arm64': 0.0.8 - '@tsc-ts/darwin-x64': 0.0.8 - '@tsc-ts/linux-arm64-gnu': 0.0.8 - '@tsc-ts/linux-arm64-musl': 0.0.8 - '@tsc-ts/linux-x64-gnu': 0.0.8 - '@tsc-ts/linux-x64-musl': 0.0.8 - '@tsc-ts/win32-x64-msvc': 0.0.8 + '@tsc-ts/darwin-arm64': 0.0.9 + '@tsc-ts/darwin-x64': 0.0.9 + '@tsc-ts/linux-arm64-gnu': 0.0.9 + '@tsc-ts/linux-arm64-musl': 0.0.9 + '@tsc-ts/linux-x64-gnu': 0.0.9 + '@tsc-ts/linux-x64-musl': 0.0.9 + '@tsc-ts/win32-x64-msvc': 0.0.9 tslib@2.8.1: optional: true diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index e74d40442..c9f873c4e 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -7,14 +7,14 @@ minimumReleaseAgeExclude: - "effect@4.0.0" - "@effect/platform-node@4.0.0" - "@effect/platform-node-shared@4.0.0" - - "@tsc-ts/darwin-arm64@0.0.8" - - "@tsc-ts/darwin-x64@0.0.8" - - "@tsc-ts/linux-arm64-gnu@0.0.8" - - "@tsc-ts/linux-arm64-musl@0.0.8" - - "@tsc-ts/linux-x64-gnu@0.0.8" - - "@tsc-ts/linux-x64-musl@0.0.8" - - "@tsc-ts/win32-x64-msvc@0.0.8" - - "tsc-ts@0.0.8" + - "@tsc-ts/darwin-arm64@0.0.9" + - "@tsc-ts/darwin-x64@0.0.9" + - "@tsc-ts/linux-arm64-gnu@0.0.9" + - "@tsc-ts/linux-arm64-musl@0.0.9" + - "@tsc-ts/linux-x64-gnu@0.0.9" + - "@tsc-ts/linux-x64-musl@0.0.9" + - "@tsc-ts/win32-x64-msvc@0.0.9" + - "tsc-ts@0.0.9" - "@scriptc/threads@0.4.2" allowBuilds: esbuild: true From 779893eef855b16825dabf486c1970360d8c66c5 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 10:15:37 -0500 Subject: [PATCH 29/53] Address review feedback on int32 arrays across calls --- packages/compiler/src/ir/int32-arrays.ts | 29 +++++++++++++++++-- .../corpus/7580-int32-arrays-across-calls.ts | 20 +++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/packages/compiler/src/ir/int32-arrays.ts b/packages/compiler/src/ir/int32-arrays.ts index e67f51956..353755c05 100644 --- a/packages/compiler/src/ir/int32-arrays.ts +++ b/packages/compiler/src/ir/int32-arrays.ts @@ -222,6 +222,25 @@ function writesLocal(e: IrExpr, id: string): boolean { return !expr(e); } +/** Expression kinds an index can be built from without running any code. */ +const PURE_INDEX_KINDS: ReadonlySet = new Set([ + "varRef", + "numLit", + "boolLit", + "bin", + "unary", + "ternary", + "logical", + "arrayGet", + "arrayState", +]); + +/** Whether evaluating an index runs no code (no call, accessor or + * conversion): plain reads, literals and their operators only. */ +function pureIndex(e: IrExpr): boolean { + return PURE_INDEX_KINDS.has(e.kind) && everyExprChild(e, pureIndex, () => false); +} + /** The parameters and globals a module's functions share as candidates, * still assumed to qualify. */ export interface SharedCandidates { @@ -319,8 +338,14 @@ export function int32ArrayCandidates( case "call": { if (readers.has(e.callee) && e.args.length === 2) { const [array, index] = e.args; - // Inlining evaluates the index before the array read. - if (array!.kind === "varRef" && writesLocal(index!, array!.localId)) + // Inlining evaluates the index before the array read: an index + // that rebinds the array keeps it out, and so does one that can run + // code when the array is a module global (a call can reassign it). + if ( + array!.kind === "varRef" && + (writesLocal(index!, array!.localId) || + (array!.localId.startsWith("%g.") && !pureIndex(index!))) + ) result.delete(array!.localId); receiver(array!); expr(index!); diff --git a/tests/corpus/7580-int32-arrays-across-calls.ts b/tests/corpus/7580-int32-arrays-across-calls.ts index 13df54609..df180b264 100644 --- a/tests/corpus/7580-int32-arrays-across-calls.ts +++ b/tests/corpus/7580-int32-arrays-across-calls.ts @@ -133,6 +133,25 @@ halve(halved); const pick: (values: number[]) => number = first; const viaValue: number[] = [9, 8, 7]; +// An index that calls a function reassigning the global: the read uses the +// array the global held before the index ran, as in Node. +let swapped: number[] = [1, 2, 3]; +function swapIn(): number { + swapped = []; + for (let i = 0; i < 4; i++) swapped.push(i + 10); + return 1; +} +function fillSwapped(n: number): number { + swapped = []; + for (let i = 0; i < n; i++) swapped.push(i & 15); + let s = 0; + for (let i = 0; i < swapped.length; i++) s += swapped[i]!; + return s; +} +function readSwapped(): number | undefined { + return swapped[swapIn()]; +} + console.log(`fnv ${fnv(data)} ${fnv(data)}`); const h = histogram(data, 10); console.log(`hist ${h.length} ${h[0]} ${h[9]} ${total(h)}`); @@ -143,4 +162,5 @@ console.log(`halved ${halved[0]} ${halved[2]} ${halved.length}`); console.log(`value ${pick(viaValue)} ${viaValue.length}`); console.log(`regrow ${regrow(40)} ${reassigned.length} ${reassigned[39]} ${regrow(5)}`); console.log(`span ${spanTotal()}`); +console.log(`swap ${fillSwapped(6)} ${readSwapped()} ${swapped[1]}`); void later().then((v) => console.log(`later ${v}`)); From b6e8e149b54250754d6c8651f45a600e83c7f40d Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 10:25:37 -0500 Subject: [PATCH 30/53] Trim combined header values again when copying after an empty append Appending an empty value to an existing header leaves the separator's space at the end of the combined value ("x, "). Node keeps that in the Headers object but trims it when the value is copied into another Headers, Request or Response ("x,"). The copy fast path copied a canonical object's pairs as they were, so the copy kept the space. Such an append now marks the pairs as no longer canonical, and copies normalize them pair by pair. --- packages/runtime/src/scr_fetch.c | 4 ++++ tests/corpus/fetch-headers-copy-cache.ts | 23 +++++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/packages/runtime/src/scr_fetch.c b/packages/runtime/src/scr_fetch.c index 48b60942e..d95807b6c 100644 --- a/packages/runtime/src/scr_fetch.c +++ b/packages/runtime/src/scr_fetch.c @@ -2442,6 +2442,10 @@ static void sf_headers_append_pair( memcpy(joined_data + previous->len + 2, value->data, value->len); ScrStr *joined = scr_str_new(joined_data, joined_len); free(joined_data); + /* Appending an empty value leaves the separator's space at the end of + * the combined value, which a copy trims again: the pairs are no + * longer in their canonical, trimmed form. */ + if (value->len == 0) h->canonical = false; scr_str_release(name); scr_str_release(value); scr_arr_set_ref(h->pairs, (double)(i + 1), joined); diff --git a/tests/corpus/fetch-headers-copy-cache.ts b/tests/corpus/fetch-headers-copy-cache.ts index 686ecfc61..24321d818 100644 --- a/tests/corpus/fetch-headers-copy-cache.ts +++ b/tests/corpus/fetch-headers-copy-cache.ts @@ -83,3 +83,26 @@ try { } catch (error) { console.log((error as Error).name, (error as Error).message); } + +// Appending an empty value leaves the separator's space at the end of the +// combined value; a copy trims it again, as Node's does. A leading +// separator has no whitespace to trim. +const spaced = new Headers(); +spaced.append("a", "x"); +spaced.append("a", ""); +spaced.append("b", ""); +spaced.append("b", "y"); +spaced.append("c", ""); +spaced.append("c", ""); +spaced.append("cookie", "k=v"); +spaced.append("cookie", ""); +const spacedList = (headers: Headers): string => { + const out: string[] = []; + headers.forEach((value, name) => out.push(`${name}=${JSON.stringify(value)}`)); + return out.join(" "); +}; +console.log(spacedList(spaced)); +console.log(spacedList(new Headers(spaced))); +console.log(spacedList(new Headers(new Headers(spaced)))); +console.log(spacedList(new Response("", { headers: spaced }).headers)); +console.log(spacedList(new Request("http://example.test/", { headers: spaced }).headers)); From 11426b4fc238a29743161f3836fa3ad31839ad29 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 10:26:22 -0500 Subject: [PATCH 31/53] Read string-keyed record entries as stack unions under projections A keyed read of a string-indexed record whose values are numbers or booleans (`counts[key]` on a `Record`, typed `number | undefined`) allocated a heap union for every read, also when the only consumer was `??`, `!` or a comparison with undefined. The runtime already returns the entry's value and whether it is present; the box came from wrapping that result. Such a read is now a stack union source: under a projection, the read writes the tag and bits of the declared field or index-signature entry the key names, or the undefined arm for a missing key, into a private stack box, and allocates nothing. This applies when every hit is a number or boolean arm of the result union, or an inline field of the same union, and also to an argument a callee only projects (the helper that `x!` lowers to). Other consumers keep the heap union. dictionary-walks allocates no union instead of 363,359 and runs 14.6% faster; no other runtime benchmark's code changes. Corpus 7390. --- .../compiler/src/backend/llvm/expr-calls.ts | 4 + .../compiler/src/backend/llvm/expr-records.ts | 109 ++++++++++++++++++ .../compiler/src/backend/llvm/receivers.ts | 6 +- .../7390-record-key-read-presence.ts.json | 6 + tests/corpus/7390-record-key-read-presence.ts | 49 ++++++++ .../7390-record-key-read-presence.ts.json | 1 + 6 files changed, 174 insertions(+), 1 deletion(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7390-record-key-read-presence.ts.json create mode 100644 tests/corpus/7390-record-key-read-presence.ts create mode 100644 tests/harness/corpus-costs/7390-record-key-read-presence.ts.json diff --git a/packages/compiler/src/backend/llvm/expr-calls.ts b/packages/compiler/src/backend/llvm/expr-calls.ts index a4e7cde2c..6ff52619e 100644 --- a/packages/compiler/src/backend/llvm/expr-calls.ts +++ b/packages/compiler/src/backend/llvm/expr-calls.ts @@ -44,6 +44,7 @@ import { LlvmUnsupportedError } from "./unsupported.js"; import type { LlvmEmitterContext, ExprOf, LlValue } from "./expr-context.js"; import { f64Lit, ffiNativeTypeLl, ffiNativeParamLl, ffiNativeReturnLl } from "./common.js"; import { canStackUnion, emitStackUnion } from "./stack-unions.js"; +import { emitStackRecordKeyGet, scalarRecordKeyGet } from "./expr-records.js"; import { emitCallArrayRead } from "./local-array-reads.js"; import { emitScalarOptional, isScalarOptionalSource } from "./local-union-storage.js"; import { emitStackMapRead } from "./map-read-lifetimes.js"; @@ -166,6 +167,9 @@ export function emitCallExpr( return { name: host.emitOwnedNullableStack(a).box, type: a.type }; // An inline union field read copies its tag and bits. if (host.inlineFieldGet(a)) return host.emitReadReceiver(a); + // A scalar keyed record read writes its tag and bits. + if (a.kind === "recordKeyGet" && scalarRecordKeyGet(host, a)) + return { name: emitStackRecordKeyGet(host, a), type: a.type }; } if (inputs[index]) return host.emitReadReceiver(a); } diff --git a/packages/compiler/src/backend/llvm/expr-records.ts b/packages/compiler/src/backend/llvm/expr-records.ts index bca728e9d..59ab625e5 100644 --- a/packages/compiler/src/backend/llvm/expr-records.ts +++ b/packages/compiler/src/backend/llvm/expr-records.ts @@ -7,6 +7,7 @@ import type { LlvmEmitterContext, LlValue } from "./expr-context.js"; import { f64Lit, llvmCommentText } from "./common.js"; import { borrowableInputs, emitBorrowedInputs, emitBorrowedInput } from "./borrowed-inputs.js"; import { regexLiteralTemplate } from "./expr-primitives.js"; +import { isScalarUnionDef, loadInlineUnion, presentBits } from "./inline-unions.js"; import { preservesRegexInputs } from "./reference-effects.js"; /** Widen one tagged read. The source is borrowed unless the overflow map @@ -365,6 +366,114 @@ export function emitRecordKeyGet( return host.own({ name: t, type: e.type }); } +/** A keyed read whose result is a scalar union (`r[k]` on a + * `Record`, at `number | undefined`) and whose every hit + * is a number or boolean arm of it, or an inline field of the same union. + * Projections (`??`, `!`, tag tests) read it as a private stack box, so + * the read allocates nothing. */ +export function scalarRecordKeyGet(host: LlvmEmitterContext, e: IrExpr): boolean { + if (e.kind !== "recordKeyGet" || e.missNaN === true || e.type.kind !== "union") return false; + const resultType = e.type; + const def = host.unionsById.get(resultType.unionId); + if (!isScalarUnionDef(def) || undefinedArmTag(resultType, host.unionsById) < 0) return false; + const isArm = (t: IrType): boolean => + (t.kind === "f64" || t.kind === "bool") && def!.arms.some((a) => typeEquals(a, t)); + const shape = host.recordShape(e.shapeId); + if (e.overflowOnly !== true) { + for (const f of shape.fields) { + const inline = host.recordFieldInline(e.shapeId, f.name); + if (!isArm(f.type) && !(inline && typeEquals(f.type, resultType))) return false; + } + } + return shape.indexValue === undefined || isArm(shape.indexValue); +} + +/** A scalarRecordKeyGet read as a private stack box: the tag and bits of + * the declared field or overflow entry the key names, or the undefined arm + * for a miss. The box must never reach an RC entry point. */ +export function emitStackRecordKeyGet( + host: LlvmEmitterContext, + e: IrExpr & { kind: "recordKeyGet" }, +): string { + const B = host.B; + if (e.type.kind !== "union") + throw new InternalCompilerError("llvm emitter bug: stack keyed read of a non-union"); + const resultType = e.type; + const def = host.unionsById.get(resultType.unionId)!; + const armTag = (t: IrType): number => def.arms.findIndex((a) => typeEquals(a, t)); + const obj = host.emitExpr(e.obj); + const key = host.emitExpr(e.key); + const box = B.slot(); + B.entryAllocas.push(`${box} = alloca %ScrUnion`); + const tagPtr = B.tmp(), + bitsPtr = B.tmp(); + B.line(`store ${host.sizeType} -1, ptr ${box}`); + B.line(`${tagPtr} = getelementptr inbounds %ScrUnion, ptr ${box}, i64 0, i32 1`); + B.line(`${bitsPtr} = getelementptr inbounds %ScrUnion, ptr ${box}, i64 0, i32 5`); + const join = B.newLabel("rks.j"); + const storeScalar = (t: IrType, value: string): void => { + const bits = B.tmp(); + if (t.kind === "f64") B.line(`${bits} = bitcast double ${value} to i64`); + else B.line(`${bits} = zext i1 ${value} to i64`); + B.line(`store i32 ${armTag(t)}, ptr ${tagPtr}`); + B.line(`store i64 ${bits}, ptr ${bitsPtr}`); + }; + const shape = host.recordShape(e.shapeId); + for (const f of e.overflowOnly === true ? [] : shape.fields) { + const hit = B.tmp(); + B.line( + `${hit} = call zeroext i1 ${host.literalEquals(f.name)}(ptr ${key.name}) ; ${llvmCommentText(f.name)}`, + ); + const lh = B.newLabel("rks.h"); + const ln = B.newLabel("rks.n"); + B.condBr(hit, lh, ln); + B.startBlock(lh); + const { ptr, type, inline } = host.recordFieldPtr(obj.name, e.shapeId, f.name); + if (inline) { + const { tag, bits } = loadInlineUnion(B, ptr); + const present = presentBits(B, tag, bits, undefinedArmTag(type, host.unionsById)); + B.line(`store i32 ${tag}, ptr ${tagPtr}`); + B.line(`store i64 ${present}, ptr ${bitsPtr}`); + } else { + storeScalar(type, host.loadRecordField(ptr, type, false)); + } + B.br(join); + B.startBlock(ln); + } + const iv = shape.indexValue; + if (iv) { + const ovf = host.recordOvfPtr(obj.name, e.shapeId); + const outTy = iv.kind === "f64" ? "double" : "i8"; + const outSlot = B.slot(); + B.entryAllocas.push(`${outSlot} = alloca ${outTy}`); + B.line(`store ${outTy} ${iv.kind === "f64" ? f64Lit(0) : "0"}, ptr ${outSlot}`); + const getter = `scr_map_get_str_${iv.kind === "f64" ? "f64" : "bool"}`; + host.declare(`declare zeroext i1 @${getter}(ptr, ptr, ptr)`); + const found = B.tmp(); + B.line(`${found} = call zeroext i1 @${getter}(ptr ${ovf}, ptr ${key.name}, ptr ${outSlot})`); + const lh = B.newLabel("rks.h"); + const ln = B.newLabel("rks.n"); + B.condBr(found, lh, ln); + B.startBlock(lh); + const raw = B.tmp(); + B.line(`${raw} = load ${outTy}, ptr ${outSlot}`); + let value = raw; + if (iv.kind === "bool") { + value = B.tmp(); + B.line(`${value} = trunc i8 ${raw} to i1`); + } + storeScalar(iv, value); + B.br(join); + B.startBlock(ln); + } + // A miss is the undefined arm. + B.line(`store i32 ${undefinedArmTag(resultType, host.unionsById)}, ptr ${tagPtr}`); + B.line(`store i64 0, ptr ${bitsPtr}`); + B.br(join); + B.startBlock(join); + return box; +} + export function keyedRecordReadInto( host: LlvmEmitterContext, slot: string, diff --git a/packages/compiler/src/backend/llvm/receivers.ts b/packages/compiler/src/backend/llvm/receivers.ts index 5e1077ee5..2fe9dd66b 100644 --- a/packages/compiler/src/backend/llvm/receivers.ts +++ b/packages/compiler/src/backend/llvm/receivers.ts @@ -18,6 +18,7 @@ import { canStackUnion, emitStackUnion } from "./stack-unions.js"; import { emitDowncastGuard, emitUnionNarrowGuard } from "./narrow-guards.js"; import { emitScalarOptional, isScalarOptionalSource } from "./local-union-storage.js"; import { borrowableInputs } from "./borrowed-inputs.js"; +import { emitStackRecordKeyGet, scalarRecordKeyGet } from "./expr-records.js"; import { type WalkBorrowHost, analyzeWalks } from "./walk-borrows.js"; /** Read a receiver whose value is consumed immediately by a field or tag @@ -95,6 +96,8 @@ export function emitReadReceiver(emitter: LlEmitter, e: IrExpr): LlValue { const narrowed = narrow ? emitCheckedNarrowReceiver(emitter, narrow) : null; if (narrowed) return narrowed; if (canStackUnion(e, emitter.boxedUnionsById)) return emitStackUnion(emitter, e).value; + if (e.kind === "recordKeyGet" && scalarRecordKeyGet(emitter, e)) + return { name: emitStackRecordKeyGet(emitter, e), type: e.type }; if (e.kind === "unionNarrow") { const union = emitter.emitUnionProjection(e.value); emitUnionNarrowGuard(emitter, union.name, e.unionId, e.tag); @@ -410,6 +413,7 @@ export function isStackUnionSource(emitter: LlEmitter, e: IrExpr): boolean { emitter.optionalArrayReads.get(e) !== null || isScalarOptionalSource(e, emitter.unionSources) || emitter.scalarResultCall(e) || - emitter.inlineFieldGet(e) + emitter.inlineFieldGet(e) || + scalarRecordKeyGet(emitter, e) ); } diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7390-record-key-read-presence.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7390-record-key-read-presence.ts.json new file mode 100644 index 000000000..74bd7e30d --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7390-record-key-read-presence.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7390-record-key-read-presence.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7390-record-key-read-presence.ts b/tests/corpus/7390-record-key-read-presence.ts new file mode 100644 index 000000000..76bb355ca --- /dev/null +++ b/tests/corpus/7390-record-key-read-presence.ts @@ -0,0 +1,49 @@ +// Keyed reads of string-indexed records whose values are numbers or +// booleans, consumed by `??`, `!`, comparisons with undefined and typeof: +// declared fields, an optional declared field, index-signature entries and +// missing keys. A stored NaN is a value, so `??` keeps it; -0 survives the +// read; a missing key is undefined. +interface Scores { + base: number; + bonus?: number; + [name: string]: number | undefined; +} + +const scores: Scores = { base: 10 }; +scores["alice"] = 3; +scores["nan"] = NaN; +scores["negzero"] = -0; + +const keys = ["base", "bonus", "alice", "nan", "negzero", "missing"]; +for (const key of keys) { + const read = scores[key]; + console.log( + key, + scores[key] ?? "default", + scores[key] === undefined, + typeof scores[key], + Object.is(scores[key] ?? 1, -0), + (scores[key] ?? 0) + 1, + read, + ); +} +scores.bonus = 5; +console.log("bonus", scores["bonus"] ?? "default", scores["bonus"]! * 2); + +const counts: Record = {}; +for (const word of "a b a c b a d".split(" ")) counts[word] = (counts[word] ?? 0) + 1; +let total = 0; +for (const word in counts) total += counts[word]!; +console.log(JSON.stringify(counts), total, counts["zzz"] ?? -1); + +const flags: Record = { on: true, off: false }; +for (const key of ["on", "off", "unset"]) { + console.log(key, flags[key] ?? "unset", flags[key] === undefined, typeof flags[key]); +} +let seen = 0; +for (let i = 0; i < 1000; i++) { + const key = "k" + (i % 7); + flags[key] = !(flags[key] ?? false); + if (flags[key]!) seen++; +} +console.log(seen, Object.keys(flags).length); diff --git a/tests/harness/corpus-costs/7390-record-key-read-presence.ts.json b/tests/harness/corpus-costs/7390-record-key-read-presence.ts.json new file mode 100644 index 000000000..3f7384f20 --- /dev/null +++ b/tests/harness/corpus-costs/7390-record-key-read-presence.ts.json @@ -0,0 +1 @@ +{ "plain": 3500, "san": 6000 } From 1f7677d246d92feefcd257456d2c3e4b3a26dde7 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Fri, 23 Oct 2026 08:23:28 -0700 Subject: [PATCH 32/53] Answer untyped property reads of plain values without the class tables Every untyped property read or write in a program with dynamic-tier classes calls a generated helper (%dyn.class.property.N). For a value that is not a class instance, the helper still called the normalization, the instance-bag and the static-bag tables, each of which returns the value itself, then compared the bag with the value and finally read the key. When every class branch of a helper tests a capsule key or a class key, its fallthrough is still the access it was created with, and every table that fallthrough calls has its kind exit, a value that is neither a class capsule nor a function now gets the same access on itself at the top of the helper. Helpers for `constructor` and `toString`, which answer primitives and errors specially, keep their full body. zod 3, z.array(Order).parse over 500 orders: 235.8M -> 199.8M instructions per round (-15.3%). Co-Authored-By: Claude Opus 5.5 --- .../lowering/class-dynamic-dispatch.ts | 110 ++++++++++++++++- ...ped-property-reads-mixed-receivers.js.json | 6 + ...-untyped-property-reads-mixed-receivers.js | 111 ++++++++++++++++++ ...ped-property-reads-mixed-receivers.js.json | 1 + 4 files changed, 227 insertions(+), 1 deletion(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7792-untyped-property-reads-mixed-receivers.js.json create mode 100644 tests/corpus/7792-untyped-property-reads-mixed-receivers.js create mode 100644 tests/harness/corpus-costs/7792-untyped-property-reads-mixed-receivers.js.json diff --git a/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts b/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts index ab3ebb34e..abf03811e 100644 --- a/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts +++ b/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts @@ -282,6 +282,11 @@ interface PropertyDispatch { fn: IrFunction; classes: Set; reflect?: boolean; + /** The helper's fallthrough statements as created, and the same access + * applied to the receiver itself: what the fallthrough reduces to for a + * value that is neither a class capsule nor a function, whose bag is the + * value. */ + plain?: { tail: readonly IrStmt[]; access: IrStmt[] }; } export function classInstanceOf( @@ -2724,7 +2729,28 @@ export class ClassDynamicDispatch { loc, }); } - dispatch = { name, write: !!write, fn: helper, classes: new Set() }; + const receiverAccess: IrStmt[] = read + ? [{ kind: "return", value: { ...read, value: varRef("p.0", DYN, loc) }, loc }] + : [ + { + kind: "exprStmt", + expr: { + ...write!, + args: [varRef("p.0", DYN, loc), write!.args[1]!, varRef("p.1", DYN, loc)], + }, + loc, + }, + { kind: "return", value: null, loc }, + ]; + dispatch = { + name, + write: !!write, + fn: helper, + classes: new Set(), + ...(name !== "constructor" && name !== "toString" + ? { plain: { tail: [...helper.body], access: receiverAccess } } + : {}), + }; this.properties.set(key, dispatch); this.generated.add(helper); lowerer.liftedFns.push(helper); @@ -4896,6 +4922,7 @@ export class ClassDynamicDispatch { ); } if (lowerer) this.guardClassTables(lowerer); + if (lowerer) this.guardPropertyHelpers(lowerer); } /** Each tracked class's bag initializer over a typed receiver (`this`), @@ -4985,6 +5012,87 @@ export class ClassDynamicDispatch { } } + /** Fast exits for the generated property helpers: when every class + * branch tests a capsule key or a class key and the fallthrough is still + * the bag access the helper was created with, a value of another kind + * (a plain object, array or primitive) normalizes to itself, has itself + * as its bag, and so gets the plain access directly, without the + * normalization, bag and static-bag calls. */ + private guardPropertyHelpers(lowerer: Lowerer): void { + // Every class table the fallthrough or normalization calls must answer + // a plain value with the value itself: its kind exit is in place. + const exits = (name: string): boolean => { + const helper = lowerer.liftedFnNamed(name); + const first = helper?.body[0]; + const param = helper?.params[0]?.localId; + return ( + param !== undefined && + first?.kind === "if" && + first.cond.kind === "dynTest" && + first.cond.negated === true && + first.then.length === 1 && + first.then[0]!.kind === "return" && + first.then[0]!.value !== null && + returnsParam(first.then[0]!.value, param) + ); + }; + const tables = new Set([ + "%dyn.class.properties", + READ_PROPERTIES, + READ_PROPERTIES_OR_EMPTY, + "%dyn.class.normalize", + "%dyn.class.staticProperties", + ]); + const callsOnlyGuardedTables = (statements: IrStmt[]): boolean => + everyStmtList(statements, { + stmt: () => true, + expr: (expression) => + expression.kind !== "call" || (tables.has(expression.callee) && exits(expression.callee)), + }); + for (const dispatch of this.properties.values()) { + const plain = dispatch.plain; + if (!plain) continue; + const fn = dispatch.fn; + const param = fn.params[0]?.localId; + if (param === undefined) continue; + const head = fn.body.length - plain.tail.length; + if (head < 0 || plain.tail.some((stmt, i) => fn.body[head + i] !== stmt)) continue; + let ok = true; + for (const stmt of fn.body.slice(0, head)) { + if ( + stmt.kind === "assign" && + stmt.localId === param && + stmt.value.kind === "call" && + stmt.value.callee === "%dyn.class.normalize" + ) + continue; + if (stmt.kind === "if" && stmt.else_ === null && branchKind(stmt.cond, param) !== null) + continue; + ok = false; + break; + } + // Class branches never run for a plain value; the rest always does. + if (!ok || !callsOnlyGuardedTables(fn.body.filter((stmt) => stmt.kind !== "if"))) continue; + const loc = fn.loc; + const not = (test: "classCapsule" | "funcKind"): IrExpr => ({ + kind: "dynTest", + test, + negated: true, + value: varRef(param, DYN, loc), + type: BOOL, + loc, + }); + fn.body.unshift({ + kind: "if", + cond: not("classCapsule"), + then: [{ kind: "if", cond: not("funcKind"), then: plain.access, else_: null, loc }], + else_: null, + loc, + }); + delete dispatch.plain; + } + } + private normalizedDispatches(): IrFunction[] { return [ ...(this.propertyBag ? [this.propertyBag] : []), diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7792-untyped-property-reads-mixed-receivers.js.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7792-untyped-property-reads-mixed-receivers.js.json new file mode 100644 index 000000000..1b28b3fbe --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7792-untyped-property-reads-mixed-receivers.js.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7792-untyped-property-reads-mixed-receivers.js" + ], + "diags": [] +} diff --git a/tests/corpus/7792-untyped-property-reads-mixed-receivers.js b/tests/corpus/7792-untyped-property-reads-mixed-receivers.js new file mode 100644 index 000000000..6a2785541 --- /dev/null +++ b/tests/corpus/7792-untyped-property-reads-mixed-receivers.js @@ -0,0 +1,111 @@ +"use strict"; +// One untyped property read site meets class instances (fixed and tracked +// layouts, getters, own properties added later), plain objects, arrays, +// strings, functions, class constructors with statics, null prototypes and +// optional chains. Each receiver kind answers as in Node. + +class Status { + constructor() { + this.value = "valid"; + } +} + +class Lazy { + constructor(key) { + this.key = key; + } + get value() { + return `lazy:${this.key}`; + } + get length() { + return 7; + } +} + +class Tracked { + constructor(list) { + list.push(this); + this.value = list.length; + } + static value = "static"; +} + +function plainFunction() {} +plainFunction.value = "fn"; + +/** @param {any} target */ +function readValue(target) { + return target.value; +} +/** @param {any} target */ +function readLength(target) { + return target.length; +} +/** @param {any} target */ +function readOptional(target) { + return target?.value; +} +/** @param {any} target @param {unknown} value */ +function writeValue(target, value) { + target.value = value; +} + +const list = []; +const added = new Status(); +writeValue(added, "dirty"); +/** @type {any} */ +const extra = new Lazy("x"); +extra.other = 1; +const nullProto = Object.create(null); +nullProto.value = "bare"; +const inherited = Object.create({ value: "from prototype" }); +const receivers = [ + new Status(), + added, + new Lazy("k"), + extra, + new Tracked(list), + { value: 1, length: 2 }, + { other: true }, + [1, 2, 3], + "text", + 42, + true, + plainFunction, + Tracked, + nullProto, + inherited, +]; +for (const receiver of receivers) { + console.log(typeof receiver, JSON.stringify(readValue(receiver)), JSON.stringify(readLength(receiver)), JSON.stringify(readOptional(receiver))); +} +console.log(readOptional(null), readOptional(undefined)); +for (const bad of [null, undefined]) { + try { + readValue(bad); + } catch (e) { + console.log(e instanceof TypeError, e.message); + } +} + +const plain = { value: 0 }; +const arr = []; +writeValue(plain, 5); +writeValue(arr, "on array"); +writeValue(inherited, "own now"); +writeValue(plainFunction, "fn2"); +const status = new Status(); +writeValue(status, "aborted"); +console.log(plain.value, arr.value, Object.keys(inherited), plainFunction.value, status.value, JSON.stringify(status)); +try { + writeValue("text", 1); +} catch (e) { + console.log(e instanceof TypeError, e.message); +} +const frozen = Object.freeze({ value: "frozen" }); +try { + writeValue(frozen, 2); +} catch (e) { + console.log(e instanceof TypeError); +} +console.log(frozen.value); diff --git a/tests/harness/corpus-costs/7792-untyped-property-reads-mixed-receivers.js.json b/tests/harness/corpus-costs/7792-untyped-property-reads-mixed-receivers.js.json new file mode 100644 index 000000000..d4cd0dbbb --- /dev/null +++ b/tests/harness/corpus-costs/7792-untyped-property-reads-mixed-receivers.js.json @@ -0,0 +1 @@ +{ "plain": 4500, "san": 5000 } From 14678cf4ebc81df929b8009d2c4da6cb53a738e9 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 10:29:35 -0500 Subject: [PATCH 33/53] Keep arrays a present-undefined store may reach out of the dense fast copy A present undefined that the fast copy stores leaves its slot out of the hole-free prefix the entry test read. The matcher left out only the array named by the store, so a store through another name for the same array, such as `const b = a` followed by `(b as (number | undefined)[])[i] = undefined`, kept `a` covered, and the fast copy read the slot as a number. Such a store now leaves out every covered array it may reach: its own, and every other array unless that array or the store's binding is a local array no other binding holds. A store into a field reaches every array except those local ones. --- .../src/backend/llvm/dense-array-loops.test.ts | 16 +++++++++++----- .../src/backend/llvm/dense-array-loops.ts | 15 ++++++++++++++- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/packages/compiler/src/backend/llvm/dense-array-loops.test.ts b/packages/compiler/src/backend/llvm/dense-array-loops.test.ts index 4a81a9bfe..a88c38575 100644 --- a/packages/compiler/src/backend/llvm/dense-array-loops.test.ts +++ b/packages/compiler/src/backend/llvm/dense-array-loops.test.ts @@ -280,10 +280,16 @@ test("a present-undefined store is covered only where it stores a covered read's // A read of an array the entry test does not cover may miss in the fast // copy, and its undefined store would run there. expect(matchStores(guarded("b"), ["a", "c"])).toBeNull(); - // A present undefined stored on its own leaves its array out. - const alone = matchStores( - [{ kind: "exprStmt", expr: read("b"), loc }, undefinedStore], - ["a", "b", "c"], + // A present undefined stored on its own leaves out its array and every + // array the other name may hold. + const alone: IrStmt[] = [{ kind: "exprStmt", expr: read("b"), loc }, undefinedStore]; + expect(matchStores(alone, ["a", "b", "c"])).toBeNull(); + const separate = matchDenseArrayLoop( + loop(alone), + shape, + withV, + new Set(["c"]), + new Set(["a", "b", "c"]), ); - expect([...alone!.arrays.keys()]).toEqual(["b"]); + expect([...separate!.arrays.keys()]).toEqual(["b"]); }); diff --git a/packages/compiler/src/backend/llvm/dense-array-loops.ts b/packages/compiler/src/backend/llvm/dense-array-loops.ts index 135d5eaf2..384c31b0c 100644 --- a/packages/compiler/src/backend/llvm/dense-array-loops.ts +++ b/packages/compiler/src/backend/llvm/dense-array-loops.ts @@ -273,6 +273,9 @@ export function matchDenseArrayLoop( const missing = missingElementStores(loop.body); // The element reads behind the guarded present-undefined stores. const guards: ElementRead[] = []; + // The arrays of the other present-undefined stores. The fast copy runs + // them, and the slot each one writes leaves the hole-free prefix. + const undefinedStores: IrExpr[] = []; const expr = (e: IrExpr): boolean => { if (++nodes > MAX_LOOP_NODES) return false; if (e.kind === "assignExpr" || e.kind === "incDec") disturbed.add(e.localId); @@ -302,7 +305,7 @@ export function matchDenseArrayLoop( // there, so the store never runs in it. const read = missing.get(s); if (read === undefined || s.index.kind !== "varRef" || !isCounter(s.index.localId)) { - if (s.arr.kind === "varRef") disturbed.add(s.arr.localId); + undefinedStores.push(s.arr); } else { access(s.arr, s.index); guards.push(read); @@ -322,6 +325,15 @@ export function matchDenseArrayLoop( ) ) return null; + // A store through another name, such as `const b = a` or a field, lands + // in `a` unless one of the two bindings is a local array no other binding + // holds. + const mayStoreUndefined = (id: string): boolean => + undefinedStores.some((arr) => + arr.kind === "varRef" + ? arr.localId === id || (!unaliased.has(id) && !unaliased.has(arr.localId)) + : !unaliased.has(id), + ); const arrays = new Map(); for (const [id, ref] of candidates) { const local = locals.get(id); @@ -331,6 +343,7 @@ export function matchDenseArrayLoop( local.boxed || local.tdz || disturbed.has(id) || + mayStoreUndefined(id) || (id !== length?.localId && !initialized.has(id)) ) continue; From d3c8155c8595cc394159d2dabad9c904afc3a3b4 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 10:32:15 -0500 Subject: [PATCH 34/53] Reuse the bundled declaration files' diagnostics across builds --- .../src/frontend/ts7/lib-diagnostics.test.ts | 221 ++++++++++++ .../src/frontend/ts7/lib-diagnostics.ts | 324 ++++++++++++++++++ .../compiler/src/frontend/ts7/program-host.ts | 35 +- packages/compiler/src/frontend/ts7/session.ts | 30 ++ packages/compiler/src/index.ts | 31 ++ packages/compiler/src/native/cli.ts | 17 +- packages/compiler/test/source-output.test.ts | 33 ++ 7 files changed, 688 insertions(+), 3 deletions(-) create mode 100644 packages/compiler/src/frontend/ts7/lib-diagnostics.test.ts create mode 100644 packages/compiler/src/frontend/ts7/lib-diagnostics.ts diff --git a/packages/compiler/src/frontend/ts7/lib-diagnostics.test.ts b/packages/compiler/src/frontend/ts7/lib-diagnostics.test.ts new file mode 100644 index 000000000..f041a6cb5 --- /dev/null +++ b/packages/compiler/src/frontend/ts7/lib-diagnostics.test.ts @@ -0,0 +1,221 @@ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, afterEach, expect, test } from "vitest"; +import type { SourceFile } from "./ast-types.js"; +import { + reusedLibDiagnostics, + setLibDiagnosticsStore, + type LibDiagnosticsStore, +} from "./lib-diagnostics.js"; +import type { Ts7DiagnosticData } from "./session-schema.generated.js"; +import type { Ts7CheckedFileDiagnosticData, Ts7SessionProgram } from "./session.js"; +import { closeSourceParser, parseSourceFile } from "./source-parser-node.js"; + +const dir = mkdtempSync(join(tmpdir(), "scriptc-lib-diagnostics-")); +const lib = join(dir, "lib.es5.d.ts"); +const fallback = join(dir, "scriptc-node-fallback.d.ts"); +const libText = "declare var NaN: number;\ninterface Array { length: number }\n"; +const fallbackText = "export {};\ndeclare global {\n var process: { argv: string[] };\n}\n"; +writeFileSync(lib, libText); +writeFileSync(fallback, fallbackText); +const main = join(dir, "main.ts"); +const util = join(dir, "\u{1F600}.ts"); +const other = join(dir, "�.ts"); + +afterEach(() => setLibDiagnosticsStore(null)); +afterAll(() => { + closeSourceParser(); + rmSync(dir, { recursive: true, force: true }); +}); + +function memoryStore(): LibDiagnosticsStore & { entries: Map } { + const entries = new Map(); + return { + entries, + read: (key) => entries.get(key) ?? null, + write: (key, text) => void entries.set(key, text), + }; +} + +function diagnostic(fileName: string | undefined, pos: number, code = 2322): Ts7DiagnosticData { + return { fileName, pos, end: pos + 1, code, category: 1, text: `at ${pos}` }; +} + +/** A session program whose full-check and checked-file answers are given; it + * records which requests it was asked. */ +function sessionProgram(answers: { + semantic: Map; + global?: Ts7DiagnosticData[]; + checked?: boolean; +}): Ts7SessionProgram & { asked: string[] } { + const asked: string[] = []; + const all = [...answers.semantic.values()].flat(); + return { + asked, + getCompilerOptions: () => ({ strict: true }), + diagnosticData: (method: string, file?: string) => { + asked.push(file === undefined ? method : `${method} ${file}`); + if (method === "getGlobalDiagnostics") return answers.global ?? []; + return file === undefined ? all : (answers.semantic.get(file) ?? []); + }, + checkedFileDiagnosticData: (files: readonly string[]): Ts7CheckedFileDiagnosticData | null => { + asked.push(`checked ${files.join(",")}`); + if (answers.checked === false) return null; + return { files: files.map((file) => answers.semantic.get(file) ?? []), global: [] }; + }, + } as unknown as Ts7SessionProgram & { asked: string[] }; +} + +/** The program's files, parsed: the supplied two and the given others. */ +function files(texts: Record): (name: string) => SourceFile | undefined { + const parsed = new Map([ + [lib, parseSourceFile(lib, libText, "ts")], + [fallback, parseSourceFile(fallback, fallbackText, "ts")], + ]); + for (const [name, text] of Object.entries(texts)) + parsed.set(name, parseSourceFile(name, text, "ts")); + return (name) => parsed.get(name); +} + +const names = [lib, fallback, main]; +const supplied = (name: string): boolean => name === lib || name === fallback; +const moduleFile = files({ [main]: "export const n: number = 'x';\n" }); + +test("without a store the full check answers", () => { + const program = sessionProgram({ semantic: new Map() }); + expect(reusedLibDiagnostics(program, names, supplied, moduleFile)).toBeNull(); + expect(program.asked).toEqual([]); +}); + +test("a stored answer stands in for checking the supplied files, merged by file name", () => { + const store = memoryStore(); + setLibDiagnosticsStore(store, "checker"); + const semantic = new Map([ + [lib, [diagnostic(lib, 1)]], + [fallback, []], + [main, [diagnostic(main, 5), diagnostic(main, 9)]], + ]); + const first = sessionProgram({ semantic }); + const full = reusedLibDiagnostics(first, names, supplied, moduleFile); + // The first build runs the full check and stores the supplied files' answers. + expect(first.asked).toEqual([ + "getGlobalDiagnostics", + "getSemanticDiagnostics", + "checked ", + `getSemanticDiagnostics ${lib}`, + `getSemanticDiagnostics ${fallback}`, + ]); + expect(store.entries.size).toBe(1); + const second = sessionProgram({ semantic }); + const reused = reusedLibDiagnostics(second, names, supplied, moduleFile); + expect(second.asked).toEqual([`checked ${main}`]); + // The files' diagnostics in the order of their names (lib.es5.d.ts, main.ts), as the full answer. + expect(reused).toEqual({ + semantic: [diagnostic(lib, 1), diagnostic(main, 5), diagnostic(main, 9)], + global: [], + }); + expect(full?.semantic).toEqual(reused?.semantic); +}); + +test("files merge in code point order, which differs from UTF-16 order", () => { + const store = memoryStore(); + setLibDiagnosticsStore(store, "checker"); + const all = [lib, fallback, util, other]; + const sources = files({ [util]: "export {};\n", [other]: "export {};\n" }); + const semantic = new Map([ + [util, [diagnostic(util, 1)]], + [other, [diagnostic(other, 2)]], + ]); + reusedLibDiagnostics(sessionProgram({ semantic }), all, supplied, sources); + const reused = reusedLibDiagnostics(sessionProgram({ semantic }), all, supplied, sources); + // U+FFFD sorts before U+1F600 by code point (its surrogates sort after it by UTF-16 unit). + expect(reused?.semantic).toEqual([diagnostic(other, 2), diagnostic(util, 1)]); +}); + +test("a full check with global diagnostics is not stored", () => { + const store = memoryStore(); + setLibDiagnosticsStore(store, "checker"); + const program = sessionProgram({ + semantic: new Map([[main, [diagnostic(main, 1)]]]), + global: [diagnostic(undefined, 0, 2318)], + }); + expect(reusedLibDiagnostics(program, names, supplied, moduleFile)?.global).toEqual([ + diagnostic(undefined, 0, 2318), + ]); + expect(store.entries.size).toBe(0); +}); + +test("scripts reuse the answer unless they declare a global the supplied files declare", () => { + const store = memoryStore(); + setLibDiagnosticsStore(store, "checker"); + const semantic = new Map([[main, []]]); + // A script declaring only new names: it stores the answer, and a later one reuses it. + const fresh = files({ + [main]: "const total = 1;\nfunction run() {}\nif (total) { var later = 2; }\n", + }); + reusedLibDiagnostics(sessionProgram({ semantic }), names, supplied, fresh); + expect(store.entries.size).toBe(1); + const again = sessionProgram({ semantic }); + expect(reusedLibDiagnostics(again, names, supplied, fresh)).not.toBeNull(); + expect(again.asked).toEqual([`checked ${main}`]); + // Scripts declaring a supplied global, at the top level, in a block, in a + // `declare global` of a supplied module, or an ambient module: the full check. + for (const text of [ + "interface Array { extra: T }\n", + "if (true) { var NaN = 1; }\n", + "declare var process: unknown;\n", + 'declare module "fs" {}\n', + ]) { + const program = sessionProgram({ semantic }); + const result = reusedLibDiagnostics(program, names, supplied, files({ [main]: text })); + expect(program.asked[0], text).not.toBe(`checked ${main}`); + if (result !== null) expect(program.asked[0]).toBe("getGlobalDiagnostics"); + } + expect(store.entries.size).toBe(1); +}); + +test("a program whose script declares a supplied global does not store its answer", () => { + const store = memoryStore(); + setLibDiagnosticsStore(store, "checker"); + const program = sessionProgram({ semantic: new Map([[main, []]]) }); + reusedLibDiagnostics(program, names, supplied, files({ [main]: "declare var NaN: number;\n" })); + expect(store.entries.size).toBe(0); +}); + +test("modules that can declare something global, and servers without the method, ask as before", () => { + const store = memoryStore(); + setLibDiagnosticsStore(store, "checker"); + const semantic = new Map([[main, []]]); + reusedLibDiagnostics(sessionProgram({ semantic }), names, supplied, moduleFile); + expect(store.entries.size).toBe(1); + for (const text of [ + "export {};\ndeclare global { var extra: number }\n", + 'export {};\ndeclare module "./other" { const extra: number }\n', + "export as namespace Lib;\nexport const n = 1;\n", + ]) { + const program = sessionProgram({ semantic }); + expect(reusedLibDiagnostics(program, names, supplied, files({ [main]: text }))).toBeNull(); + expect(program.asked).toEqual([]); + } + const old = sessionProgram({ semantic, checked: false }); + expect(reusedLibDiagnostics(old, names, supplied, moduleFile)).toBeNull(); + // Nor does a server without the method store an answer it could not use. + const fresh = memoryStore(); + setLibDiagnosticsStore(fresh, "checker"); + reusedLibDiagnostics(sessionProgram({ semantic, checked: false }), names, supplied, moduleFile); + expect(fresh.entries.size).toBe(0); +}); + +test("a supplied file that changes is a different entry", () => { + const store = memoryStore(); + setLibDiagnosticsStore(store, "checker"); + const semantic = new Map([[main, []]]); + reusedLibDiagnostics(sessionProgram({ semantic }), names, supplied, moduleFile); + writeFileSync(fallback, fallbackText + "// edited\n"); + const program = sessionProgram({ semantic }); + reusedLibDiagnostics(program, names, supplied, moduleFile); + expect(program.asked[0]).toBe("getGlobalDiagnostics"); + expect(store.entries.size).toBe(2); + writeFileSync(fallback, fallbackText); +}); diff --git a/packages/compiler/src/frontend/ts7/lib-diagnostics.ts b/packages/compiler/src/frontend/ts7/lib-diagnostics.ts new file mode 100644 index 000000000..827b337bd --- /dev/null +++ b/packages/compiler/src/frontend/ts7/lib-diagnostics.ts @@ -0,0 +1,324 @@ +/* The semantic diagnostics of the declaration files scriptc supplies (the + * TypeScript libs and scriptc's own declaration files), reused across + * builds. Checking them is most of the full check of a small program. When + * no other file of a program can change what checking those files reports + * (every one is a module without global or module augmentations, or a + * script declaring no global name the supplied files declare), the answer + * depends only on the checker, the compiler options and the files + * themselves, so a stored answer for the same three stands in for checking + * them again: the server checks only the program's other files + * (getDiagnosticsOfCheckedFiles), and the semantic diagnostics merge in the + * order the full check sorts them, by file name. An answer is stored only + * when the full check reports no global diagnostics: the global diagnostics + * of checking the other files are then the full check's. */ +import { createHash } from "node:crypto"; +import { statSync } from "node:fs"; +import type { Node, SourceFile } from "./ast-types.js"; +import type { Ts7DiagnosticData } from "./session-schema.generated.js"; +import type { Ts7SessionProgram } from "./session.js"; +import { NodeFlags, SyntaxKind } from "./enums.js"; +import { walkPreorder } from "./ast.js"; + +/** Key-value storage for reused diagnostics (best effort: failures miss). */ +export interface LibDiagnosticsStore { + read(key: string): string | null; + write(key: string, text: string): void; +} + +const SCHEMA = "scriptc.lib-diagnostics.v1"; + +let configured: { store: LibDiagnosticsStore; checker: string } | null = null; + +/** Has later builds of this process reuse the diagnostics of the supplied + * declaration files through `store`, for the checker identified by + * `checker` (its executable's identity), or stop reusing them (null). */ +export function setLibDiagnosticsStore(store: LibDiagnosticsStore | null, checker = ""): void { + configured = store === null ? null : { store, checker }; +} + +/** A checker executable's identity for the stored diagnostics' key: its + * path and file identity, or the path alone when it cannot be read. */ +export function checkerExecutableIdentity(path: string): string { + try { + const info = statSync(path); + return JSON.stringify([path, info.size, info.mtimeMs, info.ino]); + } catch { + return JSON.stringify([path]); + } +} + +/** The program's semantic and global diagnostics as the full check reports + * them, using stored diagnostics for the supplied declaration files; null + * when they cannot be reused for this program (then the caller asks for + * them as before). `supplied` tells whether a program file is one of the + * supplied declaration files. */ +export function reusedLibDiagnostics( + program: Ts7SessionProgram, + fileNames: readonly string[], + supplied: (fileName: string) => boolean, + sourceFile: (fileName: string) => SourceFile | undefined, +): { semantic: Ts7DiagnosticData[]; global: Ts7DiagnosticData[] } | null { + if (configured === null || process.env["SCRIPTC_LIB_DIAGNOSTICS"] === "0") return null; + const { store, checker } = configured; + const libs: string[] = []; + const own: string[] = []; + for (const name of fileNames) (supplied(name) ? libs : own).push(name); + if (libs.length === 0) return null; + // The global names each of the program's scripts declares; a module that + // declares nothing global contributes none. + const scripts: Set[] = []; + for (const name of own) { + const file = sourceFile(name); + if (file === undefined) return null; + if (file.externalModuleIndicator !== undefined) { + if (!declaresNothingGlobal(file)) return null; + continue; + } + const names = isJsFileName(name) ? null : scriptGlobalNames(file); + if (names === null) return null; + scripts.push(names); + } + // A script's globals cannot change what the supplied files report unless + // one of them is also a global the supplied files declare. + const collides = (globals: ReadonlySet): boolean => + scripts.some((names) => [...names].some((name) => globals.has(name))); + // The options without the configuration file's path: it names where the + // project is, which checking the supplied files does not read (the files a + // project's location brings in are part of the key). + const options = { ...program.getCompilerOptions(), configFilePath: undefined }; + const key = entryKey(checker, options, libs); + if (key === null) return null; + const stored = readEntry(store, key, libs); + if (stored !== null && !collides(stored.globals)) { + const checked = program.checkedFileDiagnosticData(own); + if (checked === null) return null; + const byFile = new Map(stored.files); + own.forEach((name, index) => byFile.set(name, checked.files[index] ?? [])); + return { semantic: inFileOrder(byFile), global: checked.global }; + } + // The full check, as before. Its answers for the supplied files are stored + // when it reports no global diagnostics and the program's scripts declare + // none of the supplied files' globals (so nothing of the program reached + // what those files report). + const global = program.diagnosticData("getGlobalDiagnostics"); + const semantic = program.diagnosticData("getSemanticDiagnostics"); + // A server that cannot check only some files could never use the answer. + if (global.length === 0 && stored === null && program.checkedFileDiagnosticData([]) !== null) { + const globals = suppliedGlobalNames(libs, sourceFile); + if (globals !== null && !collides(globals)) { + const files: Record = {}; + for (const name of libs) files[name] = program.diagnosticData("getSemanticDiagnostics", name); + try { + store.write(key, JSON.stringify({ schema: SCHEMA, globals: [...globals], files })); + } catch { + // Best effort: a failed write is a later miss. + } + } + } + return { semantic, global }; +} + +/** Whether a file cannot add to or change the global declarations the + * supplied files declare: a module with no `declare global`, no module + * augmentation and no UMD `export as namespace` declaration. */ +function declaresNothingGlobal(file: SourceFile): boolean { + if (file.externalModuleIndicator === undefined) return false; + if (file.moduleAugmentations.length > 0) return false; + for (const statement of file.statements.nodes) { + if (statement.kind === SyntaxKind.NamespaceExportDeclaration) return false; + // `declare global` (a module declaration named by the identifier `global`). + if ( + statement.kind === SyntaxKind.ModuleDeclaration && + statement.name?.kind === SyntaxKind.Identifier && + statement.name.text === "global" + ) + return false; + } + return true; +} + +function isJsFileName(name: string): boolean { + return /\.[cm]?jsx?$/.test(name); +} + +/** The names a script file declares in the global scope: its top-level + * declarations, and the `var` and function declarations of its top-level + * blocks; null when it declares an ambient module (`declare module "x"`), + * which can merge with one the supplied files declare. Over-approximates: + * a name more only makes the stored answer apply less often. */ +function scriptGlobalNames(file: SourceFile): Set | null { + const names = new Set(); + let ambientModule = false; + walkPreorder(file, (node) => { + if (node === file) return undefined; + if (node.kind === SyntaxKind.ModuleDeclaration && node.name?.kind === SyntaxKind.StringLiteral) + ambientModule = true; + const topLevel = node.parent === file; + if (topLevel || node.kind === SyntaxKind.FunctionDeclaration) addDeclaredNames(node, names); + else if ( + node.kind === SyntaxKind.VariableDeclarationList && + (node.flags & NodeFlags.BlockScoped) === 0 + ) + for (const declaration of node.declarations?.nodes ?? []) + addBindingNames(declaration.name, names); + // Functions, classes and namespaces have scopes of their own. + return GLOBAL_SCOPE_BOUNDARIES.has(node.kind) ? "skip" : undefined; + }); + return ambientModule ? null : names; +} + +/** The global names the supplied files declare: the declarations of their + * scripts (as scriptGlobalNames finds them, with ambient modules as their + * quoted names) and those of the `declare global` blocks of their modules; + * null when one of them is not available. */ +function suppliedGlobalNames( + libs: readonly string[], + sourceFile: (fileName: string) => SourceFile | undefined, +): Set | null { + const names = new Set(); + for (const name of libs) { + const file = sourceFile(name); + if (file === undefined) return null; + if (file.externalModuleIndicator === undefined) { + walkPreorder(file, (node) => { + if (node === file) return undefined; + if ( + node.kind === SyntaxKind.ModuleDeclaration && + node.name?.kind === SyntaxKind.StringLiteral + ) + names.add(JSON.stringify(node.name.text ?? "")); + if (node.parent === file || node.kind === SyntaxKind.FunctionDeclaration) + addDeclaredNames(node, names); + else if ( + node.kind === SyntaxKind.VariableDeclarationList && + (node.flags & NodeFlags.BlockScoped) === 0 + ) + for (const declaration of node.declarations?.nodes ?? []) + addBindingNames(declaration.name, names); + return GLOBAL_SCOPE_BOUNDARIES.has(node.kind) ? "skip" : undefined; + }); + continue; + } + for (const statement of file.statements.nodes) { + if ( + statement.kind !== SyntaxKind.ModuleDeclaration || + statement.name?.kind !== SyntaxKind.Identifier || + statement.name.text !== "global" + ) + continue; + for (const inner of statement.body?.statements?.nodes ?? []) addDeclaredNames(inner, names); + } + } + return names; +} + +const GLOBAL_SCOPE_BOUNDARIES = new Set([ + SyntaxKind.FunctionDeclaration, + SyntaxKind.FunctionExpression, + SyntaxKind.ArrowFunction, + SyntaxKind.MethodDeclaration, + SyntaxKind.Constructor, + SyntaxKind.GetAccessor, + SyntaxKind.SetAccessor, + SyntaxKind.ClassDeclaration, + SyntaxKind.ClassExpression, + SyntaxKind.ModuleDeclaration, +]); + +/** The names a statement declares in the scope it is in. */ +function addDeclaredNames(node: Node, names: Set): void { + switch (node.kind) { + case SyntaxKind.VariableStatement: + for (const declaration of node.declarationList?.declarations?.nodes ?? []) + addBindingNames(declaration.name, names); + return; + case SyntaxKind.FunctionDeclaration: + case SyntaxKind.ClassDeclaration: + case SyntaxKind.InterfaceDeclaration: + case SyntaxKind.TypeAliasDeclaration: + case SyntaxKind.EnumDeclaration: + case SyntaxKind.ModuleDeclaration: + case SyntaxKind.ImportEqualsDeclaration: + if (node.name?.kind === SyntaxKind.Identifier) names.add(node.name.text ?? ""); + return; + } +} + +function addBindingNames(name: Node | undefined, names: Set): void { + if (name === undefined) return; + if (name.kind === SyntaxKind.Identifier) { + names.add(name.text ?? ""); + return; + } + // Object and array binding patterns. + for (const element of name.elements?.nodes ?? []) addBindingNames(element.name, names); +} + +/** The stored diagnostics' key: the checker, the compiler options and the + * supplied files in program order with each one's file identity; null when + * one of them cannot be read. */ +function entryKey(checker: string, options: unknown, libs: readonly string[]): string | null { + const files: unknown[] = []; + for (const name of libs) { + try { + const info = statSync(name); + files.push([name, info.size, info.mtimeMs, info.ino]); + } catch { + return null; + } + } + return createHash("sha256") + .update(JSON.stringify({ schema: SCHEMA, checker, options, files })) + .digest("hex"); +} + +function readEntry( + store: LibDiagnosticsStore, + key: string, + libs: readonly string[], +): { files: Map; globals: Set } | null { + let text: string | null; + try { + text = store.read(key); + } catch { + return null; + } + if (text === null) return null; + let entry: { schema?: unknown; globals?: unknown; files?: Record }; + try { + entry = JSON.parse(text) as typeof entry; + } catch { + return null; + } + if (entry.schema !== SCHEMA || entry.files === undefined || !Array.isArray(entry.globals)) + return null; + const files = new Map(); + for (const name of libs) { + const list = entry.files[name]; + if (!Array.isArray(list)) return null; + files.set(name, list); + } + return { files, globals: new Set(entry.globals.filter((name) => typeof name === "string")) }; +} + +/** Each file's diagnostics (already in the checker's order) in the order the + * checker sorts files: by file name, compared by code point. */ +function inFileOrder(byFile: ReadonlyMap): Ts7DiagnosticData[] { + const names = [...byFile.keys()].sort(compareCodePoints); + const out: Ts7DiagnosticData[] = []; + for (const name of names) for (const d of byFile.get(name)!) out.push(d); + return out; +} + +function compareCodePoints(a: string, b: string): number { + let i = 0; + let j = 0; + while (i < a.length && j < b.length) { + const x = a.codePointAt(i)!; + const y = b.codePointAt(j)!; + if (x !== y) return x < y ? -1 : 1; + i += x > 0xffff ? 2 : 1; + j += y > 0xffff ? 2 : 1; + } + return i < a.length ? 1 : j < b.length ? -1 : 0; +} diff --git a/packages/compiler/src/frontend/ts7/program-host.ts b/packages/compiler/src/frontend/ts7/program-host.ts index 639086e94..ec143e401 100644 --- a/packages/compiler/src/frontend/ts7/program-host.ts +++ b/packages/compiler/src/frontend/ts7/program-host.ts @@ -46,6 +46,10 @@ import { trackedRealpath, } from "../input-tracker.js"; import { packageShimDeclarationText } from "../package-shims.js"; +import { ambientDtsPath, fallbackDtsPath, overridesDtsPath } from "../dts-paths.js"; +import { reusedLibDiagnostics } from "./lib-diagnostics.js"; +import { ts7Diagnostic } from "./diagnostic.js"; +import type { Ts7DiagnosticData } from "./session-schema.generated.js"; /** The compiler options our createProgram accepts: TS7's CompilerOptions * shape (numeric enums for target/module/moduleResolution — the enums module @@ -393,6 +397,22 @@ export class Ts7Program { return this.project.program.getSourceFileMetadata(resolve(fileName)); } + /** The semantic and global diagnostics with the stored diagnostics of the + * declaration files scriptc supplies (lib-diagnostics.ts), or null. */ + reusedLibDiagnostics(): { semantic: Ts7DiagnosticData[]; global: Ts7DiagnosticData[] } | null { + this.snapshot.ensureActive(); + const program = this.project.program; + const declarations = new Set([ambientDtsPath(), overridesDtsPath(), fallbackDtsPath()]); + return reusedLibDiagnostics( + program, + this.getSourceFileNames(), + (name) => + declarations.has(tsgoPath(name)) || + program.getSourceFileMetadataByPath(name)?.isDefaultLibrary === true, + (name) => this.getSourceFile(name), + ); + } + isSourceFileDefaultLibrary(file: SourceFile): boolean { return this.project.program.isSourceFileDefaultLibrary(file); } @@ -423,9 +443,20 @@ export class Ts7Program { * package dropped the aggregate). Order mirrors 5.9.3: config, options/ * program, global, syntactic, semantic. */ export function getPreEmitDiagnostics(program: Ts7Program): readonly Diagnostic[] { + const config = program.getConfigFileParsingDiagnostics(); + const options = program.getProgramDiagnostics(); + const reused = program.reusedLibDiagnostics(); + if (reused !== null) + return [ + ...config, + ...options, + ...reused.global.map(ts7Diagnostic), + ...program.getSyntacticDiagnostics(), + ...reused.semantic.map(ts7Diagnostic), + ]; return [ - ...program.getConfigFileParsingDiagnostics(), - ...program.getProgramDiagnostics(), + ...config, + ...options, ...program.getGlobalDiagnostics(), ...program.getSyntacticDiagnostics(), ...program.getSemanticDiagnostics(), diff --git a/packages/compiler/src/frontend/ts7/session.ts b/packages/compiler/src/frontend/ts7/session.ts index d3f2ae9fe..082bb79a6 100644 --- a/packages/compiler/src/frontend/ts7/session.ts +++ b/packages/compiler/src/frontend/ts7/session.ts @@ -55,6 +55,13 @@ interface ProgramQuery { snapshot?: number; project?: string; file?: SemanticDocument; + files?: string[]; +} + +/** The answer of getDiagnosticsOfCheckedFiles, as the server sends it. */ +export interface Ts7CheckedFileDiagnosticData { + files: Ts7DiagnosticData[][]; + global: Ts7DiagnosticData[]; } /** Platform-independent session over an already connected transport. The @@ -344,6 +351,29 @@ export class Ts7SessionProgram { result.push(ts7Diagnostic(data)); return result; } + /** The diagnostics of `method` for `file`, or for the whole program, as + * the server sends them. */ + diagnosticData(method: string, file?: SemanticDocument): Ts7DiagnosticData[] { + return this.session.request(method, this.query(file)) ?? []; + } + /** The semantic diagnostics of each of `files` and the global diagnostics, + * after the server checks only those files (getDiagnosticsOfCheckedFiles), + * or null when the server does not answer that method. */ + checkedFileDiagnosticData(files: readonly string[]): Ts7CheckedFileDiagnosticData | null { + try { + return this.session.request("getDiagnosticsOfCheckedFiles", { + ...this.query(), + files: [...files], + }); + } catch (error) { + if ( + error instanceof Error && + error.message.includes('unknown API method "getDiagnosticsOfCheckedFiles"') + ) + return null; + throw error; + } + } getSyntacticDiagnostics(file?: SemanticDocument): Ts7Diagnostic[] { return this.diagnostics("getSyntacticDiagnostics", file); } diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index 4f7b93cfd..4b988ab8f 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -9,6 +9,7 @@ import { } from "./library/prepare.js"; import { analyzeWithFrontend } from "./frontend/analysis.js"; import { checkerCacheIdentity } from "./frontend/ts7/checker-backend.js"; +import { setLibDiagnosticsStore } from "./frontend/ts7/lib-diagnostics.js"; import { analyzeWithVerdictCache, fileVerdictStore } from "./coverage/verdict-cache.js"; import { coverageCompilerIdentity } from "./coverage/verdict-startup.js"; import { commentOnlyReplay } from "./coverage/verdict-semantic.js"; @@ -450,6 +451,7 @@ export async function compile( ); return withCompileWarnings(result, warnings); } finally { + setLibDiagnosticsStore(null); frontendProcess?.release(); } } @@ -1161,10 +1163,36 @@ async function prepareExecutableInputInChild( * `requestPath` as `compileTracked` would in-process, and writes what it prepared, without the LLVM * text, to `resultPath`. */ +/** Has the build reuse the diagnostics of the declaration files scriptc + * supplies (lib-diagnostics.ts) through the build cache at `root`, when the + * build uses one (executables; source outputs create no cache) and the + * checker is tsc-ts. */ +function configureLibDiagnostics(root: string | null): void { + let checker: string | undefined; + try { + checker = root === null ? undefined : checkerCacheIdentity(); + } catch { + // A missing checker fails where the build starts it. + checker = undefined; + } + setLibDiagnosticsStore( + root === null || checker === undefined + ? null + : fileVerdictStore(join(root, "lib-diagnostics-v1")), + checker ?? "", + ); +} + export async function runFrontendProcess(requestPath: string, resultPath: string): Promise { const request = JSON.parse(await readFile(requestPath, "utf8")) as FrontendProcessRequest; clearCompileSessionCaches(); setProvenanceSources(request.provenance); + // The compile that started this process prepared the cache root. + configureLibDiagnostics( + (request.opts.outputKind ?? "exe") === "exe" && provenanceSources() === null + ? await prepareBuildCacheRoot(buildCacheRoot()) + : null, + ); const frontendInputs = new FrontendInputTracker(); const warnings: CompileWarningSink = { warnings: [], sourceTexts: null }; let result: FrontendProcessResult = { kind: "failed" }; @@ -1212,6 +1240,8 @@ export async function runFrontendProcess(requestPath: string, resultPath: string } } catch { result = { kind: "failed" }; + } finally { + setLibDiagnosticsStore(null); } await writeFile(resultPath, JSON.stringify(result), { mode: 0o600 }); } @@ -1449,6 +1479,7 @@ async function compileTracked( outputKind === "exe" && provenanceSources() === null ? await prepareBuildCacheRoot(buildCacheRoot()) : null; + configureLibDiagnostics(cacheRoot); let earlyCacheOptions: EarlyExecutableCacheOptions | null = null; if (outputKind === "exe") { const helperObjectRoute = diff --git a/packages/compiler/src/native/cli.ts b/packages/compiler/src/native/cli.ts index 21047b929..44d5eb9f0 100644 --- a/packages/compiler/src/native/cli.ts +++ b/packages/compiler/src/native/cli.ts @@ -16,7 +16,11 @@ import { createNativeTs7Api } from "../frontend/ts7/native-api.js"; import { resolveProvenanceSourcesWithParser } from "../frontend/provenance-core.js"; import { setCompilerReleaseVersion } from "../library/sidecar.js"; import { NativeCompiler } from "./compiler.js"; -import { openNativeCache } from "./cache.js"; +import { NATIVE_STRING_BYTES, openNativeCache } from "./cache.js"; +import { + checkerExecutableIdentity, + setLibDiagnosticsStore, +} from "../frontend/ts7/lib-diagnostics.js"; import { loadNativeToolchain } from "./toolchain.js"; import { buildSanitizedRuntime } from "./sanitizer.js"; import { runCompilerTask } from "./task.js"; @@ -53,6 +57,17 @@ async function main(): Promise { if (toolchain.declarationsRoot !== undefined) setDeclarationRoot(toolchain.declarationsRoot); setCompilerReleaseVersion(toolchain.compilerVersion); cache = openNativeCache((path, harden) => compilerNativePrivateDirectory(path, harden)); + if (cache !== null) { + const store = cache; + setLibDiagnosticsStore( + { + read: (key) => + store.read("lib-diagnostics", key, NATIVE_STRING_BYTES)?.toString("utf8") ?? null, + write: (key, text) => store.write("lib-diagnostics", key, text), + }, + checkerExecutableIdentity(toolchain.ts7Executable), + ); + } compiler = new NativeCompiler(toolchain, cache); return compiler; }; diff --git a/packages/compiler/test/source-output.test.ts b/packages/compiler/test/source-output.test.ts index 8b8e3883c..ce411f102 100644 --- a/packages/compiler/test/source-output.test.ts +++ b/packages/compiler/test/source-output.test.ts @@ -3,6 +3,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, expect, test } from "vitest"; import { compile, deserializeModule, validateModule } from "../src/index.js"; +import { setLibDiagnosticsStore } from "../src/frontend/ts7/lib-diagnostics.js"; const dirs: string[] = []; afterEach(async () => { @@ -188,6 +189,38 @@ test("source outputs ignore invalid external compiler selection and create no ex } }); +test("source outputs neither read nor create declaration diagnostics a previous build stored", async () => { + const { dir, entry, outDir } = await fixture(); + const oldCache = process.env["SCRIPTC_CACHE_DIR"]; + const cache = join(dir, "must-not-exist"); + process.env["SCRIPTC_CACHE_DIR"] = cache; + // A store an earlier build of this process left configured. + const reads: string[] = []; + setLibDiagnosticsStore( + { read: (key) => (reads.push(key), null), write: () => {} }, + "earlier-checker", + ); + try { + for (const [kind, name] of [ + ["ir", "main.ir.json"], + ["llvm", "main.ll"], + ] as const) { + const result = await compile(entry, { + outDir, + outPath: join(outDir, name), + outputKind: kind, + }); + expect(result.ok).toBe(true); + } + expect(reads).toEqual([]); + await expect(readdir(cache)).rejects.toMatchObject({ code: "ENOENT" }); + } finally { + setLibDiagnosticsStore(null); + if (oldCache === undefined) delete process.env["SCRIPTC_CACHE_DIR"]; + else process.env["SCRIPTC_CACHE_DIR"] = oldCache; + } +}); + test.each(["wasm64-wasi", "totally-invalid"])( "source output rejects unsupported target %s without writing an artifact", async (target) => { From 998a385bde23da4b06051f16b851d20f20674d6d Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 10:45:26 -0500 Subject: [PATCH 35/53] Mark constructed headers canonical only when every value is trimmed The Headers and Response constructors combine duplicate entries (a: x, a: "") into one value with the separator, which can end in its space ("x, "). They marked the pairs canonical anyway, so a copy took them unchanged where Node trims the copied value ("x,"). The constructors now mark the pairs canonical only when every value is already trimmed; otherwise copies normalize them pair by pair. --- packages/runtime/src/scr_fetch.c | 19 +++++++++++++++++-- tests/corpus/fetch-headers-copy-cache.ts | 17 +++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/packages/runtime/src/scr_fetch.c b/packages/runtime/src/scr_fetch.c index d95807b6c..d6471040c 100644 --- a/packages/runtime/src/scr_fetch.c +++ b/packages/runtime/src/scr_fetch.c @@ -3443,6 +3443,21 @@ static bool sf_request_header_ok( !sf_header_name_ci(name, name_len, "expect"); } +/* Whether every value in `pairs` is already trimmed (sf_trim_header_value + * would keep it): the form a copy can take unchanged (SfHeaders.canonical). + * Combining duplicate names with an empty value leaves the separator's + * space at the end of the combined value, which a copy trims again. */ +static bool sf_header_values_trimmed(ScrArr *pairs) { + size_t n = (size_t)scr_arr_len(pairs); + for (size_t i = 1; i < n; i += 2) { + const ScrStr *value = sf_pair_at(pairs, i); + if (value->len == 0) continue; + char first = value->data[0], last = value->data[value->len - 1]; + if (first == ' ' || first == '\t' || last == ' ' || last == '\t') return false; + } + return true; +} + static ScrStr *sf_trim_header_value(const ScrStr *value) { size_t start = 0; size_t end = value->len; @@ -5065,7 +5080,7 @@ static ScrDyn *sf_response_construct(ScrDyn *body, ScrDyn *init, response->rc = 1; response->body = stream; response->headers = sf_headers_new(header_pairs, false); - response->headers->canonical = true; + response->headers->canonical = sf_header_values_trimmed(header_pairs); response->url = scr_str_new("", 0); response->status_text = status_text; response->status = status; @@ -5189,7 +5204,7 @@ ScrDyn *scr_fetch_headers_new(ScrDyn *init) { scr_arr_release(converted); if (!ok) { scr_arr_release(pairs); return NULL; } SfHeaders *headers = sf_headers_new(pairs, false); - headers->canonical = true; + headers->canonical = sf_header_values_trimmed(pairs); ScrDyn *result = scr_dyn_new_handle(headers, SCR_DYNH_FETCH_HEADERS); sf_headers_release(headers); return result; diff --git a/tests/corpus/fetch-headers-copy-cache.ts b/tests/corpus/fetch-headers-copy-cache.ts index 24321d818..cbbc7fc6d 100644 --- a/tests/corpus/fetch-headers-copy-cache.ts +++ b/tests/corpus/fetch-headers-copy-cache.ts @@ -106,3 +106,20 @@ console.log(spacedList(new Headers(spaced))); console.log(spacedList(new Headers(new Headers(spaced)))); console.log(spacedList(new Response("", { headers: spaced }).headers)); console.log(spacedList(new Request("http://example.test/", { headers: spaced }).headers)); + +// The same through duplicate constructor entries, in a Headers object and +// in a Response's own headers. +const entries: [string, string][] = [ + ["a", "x"], + ["a", ""], + ["c", ""], + ["c", ""], + ["d", "plain"], +]; +const built = new Headers(entries); +console.log(spacedList(built)); +console.log(spacedList(new Headers(built))); +const builtResponse = new Response("", { headers: entries }); +console.log(spacedList(builtResponse.headers)); +console.log(spacedList(new Headers(builtResponse.headers))); +console.log(spacedList(new Request("http://example.test/", { headers: built }).headers)); From e84f656172aceab97a42205656d410e81daae614 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Fri, 23 Oct 2026 07:11:38 -0700 Subject: [PATCH 36/53] Clone Response and Request bodies by teeing their streams Response.clone() and Request.clone() on checked-dynamic handles answered 'res.clone is not a function' (hono's etag middleware hashes res.clone().body). The runtime now implements the streams spec's ReadableStreamDefaultTee over native streams and clones the body with it, so the original and the clone each read every chunk. A used or locked body throws undici's TypeErrors, the clone's headers are a copy with the same guard, and a cloned Request's signal follows the original's. Co-Authored-By: Claude Opus 5.5 --- .../corpus/7742-response-clone-tee.ts.json | 6 + packages/runtime/src/scr_fetch.c | 277 +++++++++++++++++- tests/corpus/7742-response-clone-tee.ts | 154 ++++++++++ .../7742-response-clone-tee.ts.json | 1 + 4 files changed, 437 insertions(+), 1 deletion(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7742-response-clone-tee.ts.json create mode 100644 tests/corpus/7742-response-clone-tee.ts create mode 100644 tests/harness/corpus-costs/7742-response-clone-tee.ts.json diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7742-response-clone-tee.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7742-response-clone-tee.ts.json new file mode 100644 index 000000000..a62bef2d0 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7742-response-clone-tee.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7742-response-clone-tee.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_fetch.c b/packages/runtime/src/scr_fetch.c index 6049a4936..37de698ac 100644 --- a/packages/runtime/src/scr_fetch.c +++ b/packages/runtime/src/scr_fetch.c @@ -1855,6 +1855,210 @@ static ScrPromise *sf_stream_collect(SfStream *s, int mode) { return p; } +/* ── ReadableStreamDefaultTee ────────────────────────────────────── */ + +/* + * The streams spec's ReadableStreamDefaultTee over a native stream: the + * tee locks the source with a reader and feeds two native branches whose + * pull and cancel algorithms are native functions. A pull on either branch + * reads one chunk from the source and enqueues it into every branch that + * was not canceled; canceling both branches cancels the source with both + * reasons. Response.clone and Request.clone use it (undici's cloneBody). + * + * Ownership: the tee owns the source, its reader and both branches; each + * branch's callbacks and every pending read own the tee. Closing, erroring + * or canceling a branch drops its callbacks (sf_stream_drop_source_callbacks), + * and fetch teardown drops the callbacks of branches nobody finished. + */ +typedef struct SfTee { + size_t rc; + SfStream *source; + SfReader *reader; + SfStream *branch[2]; + bool reading; + bool read_again; + bool canceled[2]; + ScrDyn *reason[2]; + ScrPromise *cancel_promise; +} SfTee; + +static SfTee *sf_tee_retain(SfTee *t) { + t->rc++; + return t; +} + +static void sf_tee_release(SfTee *t) { + if (!t || --t->rc > 0) return; + sf_stream_release(t->branch[0]); + sf_stream_release(t->branch[1]); + sf_reader_release(t->reader); + sf_stream_release(t->source); + scr_dyn_release(t->reason[0]); + scr_dyn_release(t->reason[1]); + scr_promise_release(t->cancel_promise); + free(t); +} + +static void *sf_tee_retain_v(void *p) { return sf_tee_retain((SfTee *)p); } +static void sf_tee_release_v(void *p) { sf_tee_release((SfTee *)p); } + +static ScrBox *sf_tee_box(SfTee *t) { + ScrBox *box = scr_box_new_obj(&sf_tee_retain_v, &sf_tee_release_v, NULL); + scr_box_set_ref(box, sf_tee_retain(t)); + return box; +} + +static void sf_tee_pull(SfTee *t); + +/* The read request's chunk steps run in a microtask (spec: "queue a + * microtask"), after the read that produced the chunk settles. */ +static void sf_tee_chunk_task(ScrClosure *cb) { + SfTee *t = (SfTee *)scr_box_get_ref(cb->caps[0]); + ScrDyn *chunk = (ScrDyn *)scr_box_get_ref(cb->caps[1]); + t->read_again = false; + for (int i = 0; i < 2; i++) { + if (t->canceled[i]) continue; + SfStream *b = t->branch[i]; + if (b->close_requested || b->closed || b->error) continue; + sf_stream_enqueue_value(b, chunk); + if (scr_exc_pending()) scr_exc_clear(); + } + t->reading = false; + if (t->read_again) sf_tee_pull(t); + scr_dyn_release(chunk); + sf_tee_release(t); +} + +static void sf_tee_sink_chunk(void *ctx, ScrDyn *value) { + SfTee *t = ctx; + ScrClosure *cb = scr_closure_new((void *)&sf_tee_chunk_task, 2); + cb->caps[0] = sf_tee_box(t); + cb->caps[1] = scr_box_new_obj(&scr_dyn_retain_v, &scr_dyn_release_v, NULL); + scr_box_set_ref(cb->caps[1], scr_dyn_retain(value)); + scr_queue_microtask(cb); + sf_tee_release(t); /* the read request's reference */ +} + +static void sf_tee_sink_close(void *ctx) { + SfTee *t = ctx; + t->reading = false; + for (int i = 0; i < 2; i++) { + if (!t->canceled[i]) sf_stream_close(t->branch[i]); + } + if (!t->canceled[0] || !t->canceled[1]) { + scr_promise_fulfill_void(t->cancel_promise); + } + sf_tee_release(t); +} + +static void sf_tee_sink_error(void *ctx, ScrDyn *reason) { + SfTee *t = ctx; + t->reading = false; + if (reason) { + /* The spec errors both branches when the source reader's closed + * promise rejects; a native read request learns it here. */ + sf_stream_error(t->branch[0], reason); + sf_stream_error(t->branch[1], reason); + if (!t->canceled[0] || !t->canceled[1]) { + scr_promise_fulfill_void(t->cancel_promise); + } + } + sf_tee_release(t); +} + +static const ScrFetchReadSink sf_tee_sink = { + &sf_tee_sink_chunk, &sf_tee_sink_close, &sf_tee_sink_error}; + +static void sf_tee_pull(SfTee *t) { + if (t->reading) { + t->read_again = true; + return; + } + t->reading = true; + sf_tee_retain(t); /* owned by the read request */ + if (!sf_reader_read_request(t->reader, NULL, &sf_tee_sink, t)) { + scr_exc_clear(); + t->reading = false; + sf_tee_release(t); + } +} + +static ScrDyn *sf_tee_pull_thunk(ScrClosure *cb, ScrDyn *const *args, + size_t argc) { + (void)args; + (void)argc; + SfTee *t = (SfTee *)scr_box_get_ref(cb->caps[0]); + sf_tee_pull(t); + sf_tee_release(t); + return scr_dyn_retain(scr_dyn_undefined()); +} + +static ScrDyn *sf_tee_cancel(SfTee *t, int i, ScrDyn *reason) { + t->canceled[i] = true; + scr_dyn_release(t->reason[i]); + t->reason[i] = scr_dyn_retain(reason); + if (t->canceled[0] && t->canceled[1]) { + ScrDyn *composite = scr_dyn_new_arr(); + scr_dyn_arr_push(composite, scr_dyn_retain(t->reason[0])); + scr_dyn_arr_push(composite, scr_dyn_retain(t->reason[1])); + ScrPromise *result = sf_stream_cancel(t->source, composite, true); + scr_dyn_release(composite); + ScrDyn *boxed = scr_dyn_new_promise(result); + scr_promise_release(result); + sf_then_adopt(t->cancel_promise, boxed); + scr_dyn_release(boxed); + } + return scr_dyn_new_promise(t->cancel_promise); +} + +static ScrDyn *sf_tee_cancel_thunk_0(ScrClosure *cb, ScrDyn *const *args, + size_t argc) { + SfTee *t = (SfTee *)scr_box_get_ref(cb->caps[0]); + ScrDyn *out = sf_tee_cancel(t, 0, argc ? args[0] : scr_dyn_undefined()); + sf_tee_release(t); + return out; +} + +static ScrDyn *sf_tee_cancel_thunk_1(ScrClosure *cb, ScrDyn *const *args, + size_t argc) { + SfTee *t = (SfTee *)scr_box_get_ref(cb->caps[0]); + ScrDyn *out = sf_tee_cancel(t, 1, argc ? args[0] : scr_dyn_undefined()); + sf_tee_release(t); + return out; +} + +static ScrDyn *sf_tee_func(SfTee *t, ScrDynThunk thunk, const char *name) { + ScrClosure *cb = scr_closure_new((void *)thunk, 1); + cb->caps[0] = sf_tee_box(t); + return scr_dyn_new_func(cb, thunk, 1, "(controller)", name); +} + +/* Tees an unlocked, undisturbed stream: +1 branches in out[0] and out[1]. + * The source stays locked by the tee. */ +static void sf_stream_tee(SfStream *s, SfStream *out[2]) { + SfTee *t = calloc(1, sizeof *t); + if (!t) sf_oom(); + t->rc = 1; + t->source = sf_stream_retain(s); + t->reader = sf_stream_get_reader(s); + t->cancel_promise = scr_promise_new(); + scr_promise_mark_handled(t->cancel_promise); + for (int i = 0; i < 2; i++) { + SfStream *b = sf_stream_new_native(); + b->pull_cb = sf_tee_func(t, &sf_tee_pull_thunk, "pull"); + b->cancel_cb = sf_tee_func( + t, i == 0 ? &sf_tee_cancel_thunk_0 : &sf_tee_cancel_thunk_1, "cancel"); + b->source_this = scr_dyn_retain(scr_dyn_undefined()); + sf_stream_track_callbacks(b); + t->branch[i] = sf_stream_retain(b); + out[i] = b; + } + /* The branches start at once; a branch with room then pulls. */ + sf_stream_schedule_initial_pull(out[0]); + sf_stream_schedule_initial_pull(out[1]); + sf_tee_release(t); +} + ScrDyn *scr_fetch_stream_new(ScrDyn *source) { ScrDyn *source_view = source; ScrDyn *materialized = NULL; @@ -2729,12 +2933,54 @@ static ScrPromise *sf_response_collect(SfResponse *r, int mode) { return p; } +/* The body and metadata copy of Response.clone and Request.clone (undici + * cloneResponse/cloneBody): the body tees, so the original and the clone + * each keep one branch, and the clone's headers are a copy with the same + * guard. NULL with a TypeError pending when the body is unusable. */ +static SfResponse *sf_response_clone(SfResponse *r, const char *unusable) { + if (!r->null_body && + (r->body->disturbed || r->body->reader || r->body->internal_lock)) { + sf_type_error(unusable); + return NULL; + } + SfResponse *c = calloc(1, sizeof *c); + if (!c) sf_oom(); + c->rc = 1; + if (r->null_body) { + c->body = sf_stream_new_native(); + sf_stream_close(c->body); + } else { + SfStream *branches[2]; + sf_stream_tee(r->body, branches); + sf_stream_release(r->body); + r->body = branches[0]; + c->body = branches[1]; + } + c->headers = sf_headers_new(scr_arr_slice(r->headers->pairs, 0, INFINITY), + r->headers->immutable); + c->url = scr_str_retain(r->url); + c->status_text = scr_str_retain(r->status_text); + c->status = r->status; + c->redirected = r->redirected; + c->null_body = r->null_body; + c->light_length = r->light_length; + return c; +} + static ScrDyn *sf_response_invoke(void *ptr, ScrDyn *self, const char *method, ScrDyn *const *args, size_t argc, const char *what) { SfResponse *r = ptr; sf_response_materialize(r); + if (strcmp(method, "clone") == 0) { + SfResponse *c = sf_response_clone( + r, "Response.clone: Body has already been consumed."); + if (!c) return NULL; + ScrDyn *out = scr_dyn_new_handle(c, SCR_DYNH_FETCH_RESPONSE); + sf_response_release(c); + return out; + } int mode = -1; if (strcmp(method, "json") == 0) mode = SF_COLLECT_JSON; else if (strcmp(method, "text") == 0) mode = SF_COLLECT_TEXT; @@ -5153,8 +5399,37 @@ static ScrDyn *sf_request_get(void *ptr, const char *key, size_t len) { if (sf_name(key, len, "body") || sf_name(key, len, "bodyUsed") || sf_name(key, len, "headers")) return sf_response_get(r->payload, key, len); return NULL; } +/* Request.clone: the payload clones as a Response does, and the clone's + * signal follows the original's (undici: an AbortController aborted with + * the original signal's reason, at once when it already aborted). */ +static ScrDyn *sf_request_clone(SfRequest *r) { + SfResponse *payload = sf_response_clone(r->payload, "unusable"); + if (!payload) return NULL; + ScrDyn *sources = scr_dyn_new_arr(); + scr_dyn_arr_push(sources, + scr_dyn_new_handle(r->signal, SCR_DYNH_ABORT_SIGNAL)); + ScrDyn *follower = scr_fetch_abort_any(sources); + scr_dyn_release(sources); + if (!follower) { + sf_response_release(payload); + return NULL; + } + SfRequest *c = calloc(1, sizeof *c); + if (!c) sf_oom(); + c->rc = 1; + c->payload = payload; + c->url = scr_str_retain(r->url); + c->method = scr_str_retain(r->method); + c->redirect = scr_str_retain(r->redirect); + c->signal = sf_signal_retain((SfSignal *)follower->v.handle.ptr); + scr_dyn_release(follower); + ScrDyn *out = scr_dyn_new_handle(c, SCR_DYNH_FETCH_REQUEST); + sf_request_release_v(c); + return out; +} static ScrDyn *sf_request_invoke(void *ptr, ScrDyn *self, const char *method, ScrDyn *const *args, size_t argc, const char *what) { SfRequest *r = ptr; + if (strcmp(method, "clone") == 0) return sf_request_clone(r); return sf_response_invoke(r->payload, self, method, args, argc, what); } @@ -5668,7 +5943,7 @@ static const ScrDynHandleMethod sf_reader_methods[] = { {NULL, 0, false}}; static const ScrDynHandleMethod sf_body_methods[] = { {"json", 0, true}, {"text", 0, true}, {"bytes", 0, true}, - {"arrayBuffer", 0, true}, {NULL, 0, false}}; + {"arrayBuffer", 0, true}, {"clone", 0, false}, {NULL, 0, false}}; static const ScrDynHandleMethod sf_headers_methods[] = { {"append", 2, false}, {"delete", 1, false}, {"entries", 0, false}, {"forEach", 1, false}, {"get", 1, false}, {"getSetCookie", 0, false}, diff --git a/tests/corpus/7742-response-clone-tee.ts b/tests/corpus/7742-response-clone-tee.ts new file mode 100644 index 000000000..9058c4d70 --- /dev/null +++ b/tests/corpus/7742-response-clone-tee.ts @@ -0,0 +1,154 @@ +// Response.clone and Request.clone on checked-dynamic handles: the body +// tees, so the original and the clone each read every chunk; a clone of a +// used or locked body throws; canceling both branches cancels the source +// with both reasons; the clone's headers are a copy. +const enc = new TextEncoder(); +const dec = new TextDecoder(); + +function describe(e: unknown): string { + const err = e as { name?: string; message?: string }; + return `${String(err.name)}: ${String(err.message)}`; +} + +async function readAll(stream: any): Promise { + const reader = stream.getReader(); + let out = ""; + for (;;) { + const { done, value } = await reader.read(); + if (done) return out; + out += dec.decode(value); + } +} + +async function main(): Promise { + const r: any = new Response("hello world", { status: 201, headers: { "x-a": "1" } }); + const c: any = r.clone(); + console.log(c.status, c.headers.get("x-a"), c.bodyUsed, r.bodyUsed, c.body === r.body); + console.log(await c.text(), "|", await r.text()); + console.log("used", c.bodyUsed, r.bodyUsed); + try { + r.clone(); + } catch (e) { + console.log("clone used:", describe(e), e instanceof TypeError); + } + + const locked: any = new Response("abc"); + locked.body.getReader(); + try { + locked.clone(); + } catch (e) { + console.log("clone locked:", describe(e)); + } + + const empty: any = new Response(null, { status: 204, statusText: "Nothing" }); + const emptyClone: any = empty.clone(); + console.log("null body", emptyClone.body, emptyClone.status, emptyClone.statusText, JSON.stringify(await emptyClone.text())); + + // Headers are copied: changing the clone's leaves the original's alone. + const h: any = new Response("x", { headers: { "x-b": "2" } }); + const hc: any = h.clone(); + hc.headers.set("x-b", "3"); + console.log("headers", h.headers.get("x-b"), hc.headers.get("x-b"), h.headers === hc.headers); + + // A pulled source: both branches see each chunk (the same object). + let n = 0; + const source: any = new ReadableStream({ + start(): void { + console.log("start"); + }, + pull(controller: any): void { + n++; + console.log("pull", n); + if (n <= 3) controller.enqueue(enc.encode(`chunk${n};`)); + else controller.close(); + }, + }); + const streamed: any = new Response(source); + const streamedClone: any = streamed.clone(); + console.log("cloned"); + const r1 = streamed.body.getReader(); + const r2 = streamedClone.body.getReader(); + const a = await r1.read(); + const b = await r2.read(); + console.log("same chunk", a.value === b.value, dec.decode(a.value)); + console.log("rest 1", await readAllFrom(r1)); + console.log("rest 2", await readAllFrom(r2)); + + // One branch read to the end first; the other buffers. + const big: any = new Response(new ReadableStream({ + start(controller: any): void { + controller.enqueue(enc.encode("a")); + controller.enqueue(enc.encode("b")); + controller.close(); + }, + })); + const bigClone: any = big.clone(); + console.log("clone first", await bigClone.text()); + console.log("then original", await readAll(big.body)); + + // Canceling both branches cancels the source with both reasons. + const cancelSource: any = new ReadableStream({ + pull(controller: any): void { + controller.enqueue(enc.encode("z")); + }, + cancel(reason: unknown): void { + console.log("source cancel", JSON.stringify(reason)); + }, + }); + const cr: any = new Response(cancelSource); + const crc: any = cr.clone(); + const p1 = cr.body.cancel("first"); + console.log("one canceled"); + const p2 = crc.body.cancel("second"); + await p1; + await p2; + console.log("both canceled", cr.bodyUsed, crc.bodyUsed); + + // An erroring source errors both branches. + const failing: any = new ReadableStream({ + pull(controller: any): void { + controller.error(new Error("source failed")); + }, + }); + const fr: any = new Response(failing); + const frc: any = fr.clone(); + for (const branch of [fr, frc]) { + try { + await branch.text(); + } catch (e) { + console.log("branch error:", describe(e)); + } + } + + // Request.clone: the body tees and the signal follows the original's. + const q: any = new Request("http://example.test/path", { method: "POST", body: "data" }); + const qc: any = q.clone(); + console.log("request", qc.method, qc.url, await qc.text(), await q.text(), qc.signal === q.signal); + try { + q.clone(); + } catch (e) { + console.log("request clone used:", describe(e)); + } + const ac = new AbortController(); + const q2: any = new Request("http://example.test/", { signal: ac.signal }); + const q2c: any = q2.clone(); + console.log("before abort", q2c.signal.aborted); + ac.abort("why"); + console.log("after abort", q2c.signal.aborted, q2c.signal.reason); + const q3: any = new Request("http://example.test/", { signal: AbortSignal.abort("early") }); + console.log("aborted clone", q3.clone().signal.aborted, q3.clone().signal.reason); + + // clone is a method value like the body readers. + console.log(typeof r.clone, "clone" in r, r.clone === c.clone); +} + +async function readAllFrom(reader: any): Promise { + let out = ""; + for (;;) { + const { done, value } = await reader.read(); + if (done) return out; + out += dec.decode(value); + } +} + +await main(); diff --git a/tests/harness/corpus-costs/7742-response-clone-tee.ts.json b/tests/harness/corpus-costs/7742-response-clone-tee.ts.json new file mode 100644 index 000000000..0c31a05f7 --- /dev/null +++ b/tests/harness/corpus-costs/7742-response-clone-tee.ts.json @@ -0,0 +1 @@ +{ "plain": 3000, "san": 4000 } From 3c2911ccad559e53e49339f0862068cb55ad3337 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Fri, 23 Oct 2026 07:11:39 -0700 Subject: [PATCH 37/53] Implement the identity TransformStream natively 'new TransformStream()' had no lowering, so hono's stream() and streamSSE() helpers failed to compile their checked-dynamic paths. The constructor without arguments now builds a native handle: its readable side is a native ReadableStream with a zero high-water mark, and its writable side runs the streams spec's WritableStream state machine (getWriter, abort, close, locked; the writer's write, close, abort, releaseLock, ready, closed and desiredSize). A write completes when the readable side reads its chunk, close/abort/cancel propagate across the pair, and deferred reactions take as many microtask turns as in Node. Web ReadableStreams can also be weak collection keys (hono keys a WeakMap by the response stream). A transformer, queuing strategies and the WritableStream constructor keep the SC2020 refusal. Co-Authored-By: Claude Opus 5.5 --- .../builtin-apis/streams-and-readline.mdx | 3 +- .../src/generated/node-v24-compatibility.json | 12 +- .../generated/node-v24-internal.json | 16 +- internal/compatibility/static-support.json | 4 +- .../ambient/scriptc-node-fallback.d.ts | 28 + .../compiler/src/backend/llvm/lib-shared.ts | 2 + packages/compiler/src/compat/fetch-profile.ts | 2 +- .../src/frontend/lowering/lower-stdlib.ts | 13 + .../compiler/src/frontend/lowering/lowerer.ts | 2 + packages/compiler/src/frontend/type-mapper.ts | 5 +- .../compiler/src/ir/builtin-signatures.ts | 1 + .../7743-transform-stream-identity.ts.json | 6 + packages/runtime/src/scr_dyn.h | 3 + packages/runtime/src/scr_fetch.c | 927 ++++++++++++++++++ packages/runtime/src/scr_generator.h | 1 + packages/runtime/src/scr_json.c | 1 + .../corpus/7743-transform-stream-identity.ts | 212 ++++ .../7743-transform-stream-identity.ts.json | 1 + 18 files changed, 1225 insertions(+), 14 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7743-transform-stream-identity.ts.json create mode 100644 tests/corpus/7743-transform-stream-identity.ts create mode 100644 tests/harness/corpus-costs/7743-transform-stream-identity.ts.json diff --git a/docs/content/limitations/builtin-apis/streams-and-readline.mdx b/docs/content/limitations/builtin-apis/streams-and-readline.mdx index fe6bb2dfb..273554a3b 100644 --- a/docs/content/limitations/builtin-apis/streams-and-readline.mdx +++ b/docs/content/limitations/builtin-apis/streams-and-readline.mdx @@ -27,7 +27,8 @@ A `Duplex.from` body function may take the `{ signal }` argument, which is aborted with the error the Duplex is destroyed with. `stream.compose` accepts streams and async generator functions in any position, an async function as the last stage, and an array, a string, or a generator as the first stage; one stage is `Duplex.from` of it. A web `ReadableStream` is read as an async iterable by `Readable.from` and as the first stage of `compose` or `pipeline`, and through its reader by `Readable.fromWeb` (with `objectMode`, `highWaterMark`, `encoding`, and `signal` in an options object literal) and `Duplex.from`. - A web stream in a later compose stage, `Readable.toWeb`, and web `WritableStream` and `TransformStream` interop are unsupported. + `new TransformStream()` without arguments builds the identity transform: its readable side is a web `ReadableStream`, its writable side supports `getWriter`, `abort`, `close`, and `locked`, and the writer supports `write`, `close`, `abort`, `releaseLock`, `ready`, `closed`, and `desiredSize`, with Node's backpressure (a write completes when the readable side reads its chunk). + A web stream in a later compose stage, `Readable.toWeb`, a `TransformStream` with a transformer or queuing strategies, the `WritableStream` constructor, `pipeTo`, and `pipeThrough` are unsupported. The `ERR_INVALID_ARG_VALUE` message for a stream that is no longer readable or writable names the stream's class instead of printing the stream. `readable.compose(stage, { signal })` is `stream.compose(readable, stage)` with the signal added; the options are an object literal whose `signal` is a variable or a property read. `pipeline` from `node:stream/promises` accepts a trailing `{ signal, end }` object, with or without function stages, and `finished` accepts `readable`, `writable`, `error`, `cleanup`, and `signal` options, with boolean literals for the flags. diff --git a/docs/src/generated/node-v24-compatibility.json b/docs/src/generated/node-v24-compatibility.json index c3f1835ab..fc5826666 100644 --- a/docs/src/generated/node-v24-compatibility.json +++ b/docs/src/generated/node-v24-compatibility.json @@ -35971,9 +35971,9 @@ "inherited": true }, "static": { - "status": "not-implemented", - "detail": "No static compiler lowering is registered for this API yet.", - "verification": "registry-gap" + "status": "partial", + "detail": "Implemented for the documented scriptc module-loader subset.", + "verification": "test-backed" } }, { @@ -81657,9 +81657,9 @@ "inherited": true }, "static": { - "status": "not-implemented", - "detail": "No static compiler lowering is registered for this API yet.", - "verification": "registry-gap" + "status": "partial", + "detail": "Implemented for the documented static async-iteration subset.", + "verification": "test-backed" } }, { diff --git a/internal/compatibility/generated/node-v24-internal.json b/internal/compatibility/generated/node-v24-internal.json index ba8692b49..6eab15d70 100644 --- a/internal/compatibility/generated/node-v24-internal.json +++ b/internal/compatibility/generated/node-v24-internal.json @@ -52194,6 +52194,7 @@ "tests/corpus/6294-webcrypto-js-values.mjs", "tests/corpus/6295-jose-hs256/main.mjs", "tests/corpus/6331-jose-root-entry/main.ts", + "tests/corpus/7743-transform-stream-identity.ts", "tests/corpus/text-codec-values/main.ts", "tests/corpus/webassembly-api.mjs", "tests/corpus/webassembly-dylink/main.mjs", @@ -53443,8 +53444,11 @@ "inherited": true }, "static": { - "status": "not-implemented", - "evidence": "compiler-unmatched:globals" + "status": "partial", + "evidence": "compiler-feature:globals.TransformStream", + "tests": [ + "tests/corpus/7743-transform-stream-identity.ts" + ] }, "anchorSource": "exact" }, @@ -111483,6 +111487,7 @@ "tests/corpus/2630-stream-consumers-errors.ts", "tests/corpus/2862-web-readable-async-iteration.ts", "tests/corpus/4875-stream-consumers-array-buffer.ts", + "tests/corpus/7743-transform-stream-identity.ts", "tests/corpus/stream-web-readable.ts", "tests/corpus/stream-web-require.cjs", "tests/harness/fetch-conformance.test.ts" @@ -112957,8 +112962,11 @@ "inherited": true }, "static": { - "status": "not-implemented", - "evidence": "compiler-unmatched:webstreams" + "status": "partial", + "evidence": "compiler-feature:webstreams.TransformStream", + "tests": [ + "tests/corpus/7743-transform-stream-identity.ts" + ] }, "anchorSource": "exact" }, diff --git a/internal/compatibility/static-support.json b/internal/compatibility/static-support.json index 100882c1a..4e5e29616 100644 --- a/internal/compatibility/static-support.json +++ b/internal/compatibility/static-support.json @@ -407,7 +407,9 @@ { "chapter": "zlib", "signatures": ["`zlib.constants`"], "status": "partial", "evidence": ["tests/corpus/5376-zlib-constants.cjs", "tests/corpus/5374-zlib-stream-flush-params.ts"] }, { "chapter": "dns", "symbols": ["dns.lookup", "dns.lookupService", "dns.getServers", "dns.getDefaultResultOrder", "dns.setDefaultResultOrder"], "status": "partial", "evidence": ["tests/corpus/5648-dns-lookup-forms.ts", "tests/corpus/5649-dns-service-servers.ts", "tests/corpus/5650-dns-lookup-errors-untyped.cjs"] }, { "chapter": "dns", "symbols": ["dnsPromises.lookup", "dnsPromises.lookupService", "dnsPromises.getDefaultResultOrder"], "status": "partial", "evidence": ["tests/corpus/dns-promises-lookup.ts", "tests/corpus/dns-promises-untyped.cjs"] }, - { "chapter": "webstreams", "symbols": ["ReadableStream"], "status": "partial", "evidence": ["tests/corpus/stream-web-readable.ts", "tests/corpus/stream-web-require.cjs", "tests/corpus/2862-web-readable-async-iteration.ts"] } + { "chapter": "webstreams", "symbols": ["ReadableStream"], "status": "partial", "evidence": ["tests/corpus/stream-web-readable.ts", "tests/corpus/stream-web-require.cjs", "tests/corpus/2862-web-readable-async-iteration.ts"] }, + { "chapter": "webstreams", "symbols": ["TransformStream"], "status": "partial", "evidence": ["tests/corpus/7743-transform-stream-identity.ts"] }, + { "chapter": "globals", "signatures": ["Class: `TransformStream`"], "status": "partial", "evidence": ["tests/corpus/7743-transform-stream-identity.ts"] } ], "dedicated": [ { "symbols": ["assert.doesNotThrow", "assert.doesNotReject"], "status": "partial", "evidence": ["tests/corpus/node24-no-error-assertions.ts", "tests/corpus/node24-no-error-assertions.cjs", "tests/corpus/node24-no-error-assertions-coercion.js", "tests/fixtures/node-types/no-error-assertions.ts", "tests/harness/project-config.test.ts"] }, diff --git a/packages/compiler/ambient/scriptc-node-fallback.d.ts b/packages/compiler/ambient/scriptc-node-fallback.d.ts index d7abe70df..09587aa72 100644 --- a/packages/compiler/ambient/scriptc-node-fallback.d.ts +++ b/packages/compiler/ambient/scriptc-node-fallback.d.ts @@ -1106,6 +1106,34 @@ declare var ReadableStream: { from(iterable: string): ReadableStream; readonly prototype: ReadableStream; }; +/* The writable side of a TransformStream and its writer. The + * WritableStream constructor has no native lowering; writable streams come + * from `new TransformStream()`. */ +interface WritableStreamDefaultWriter { + readonly closed: Promise; + readonly desiredSize: number | null; + readonly ready: Promise; + abort(reason?: unknown): Promise; + close(): Promise; + releaseLock(): void; + write(chunk?: W): Promise; +} +interface WritableStream { + readonly locked: boolean; + abort(reason?: unknown): Promise; + close(): Promise; + getWriter(): WritableStreamDefaultWriter; +} +/* The identity transform: chunks written to `writable` are read from + * `readable`, and a write completes when its chunk is read. */ +interface TransformStream { + readonly readable: ReadableStream; + readonly writable: WritableStream; +} +declare var TransformStream: { + new (): TransformStream; + readonly prototype: TransformStream; +}; /* fetch, typed as the native JSON + readable-stream slice: one URL, an * optional init with the members CLI requests carry, and a Response diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index 4f4a6c302..bc95ad98d 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -958,6 +958,7 @@ export const LIB_FN_SYMS: Record = { "fetch.streamIs": "scr_fetch_stream_is", "zlib.streamIs": "scr_zlib_stream_is", "fetch.streamNew": "scr_fetch_stream_new", + "fetch.transformNew": "scr_fetch_transform_new", "fetch.streamFrom": "scr_fetch_stream_from", "fetch.readerRead": "scr_fetch_reader_read", "json.parse": "scr_json_parse", @@ -1689,6 +1690,7 @@ export const USES_TIMERS_LIB_FNS = new Set([ "fetch.abortAny", "fetch.streamNew", "fetch.streamFrom", + "fetch.transformNew", "readable.new", // fs.ReadStream / fs.WriteStream open, read and write on the worker pool. "fs.createReadStream", diff --git a/packages/compiler/src/compat/fetch-profile.ts b/packages/compiler/src/compat/fetch-profile.ts index c4d1e3f50..ff7cedb22 100644 --- a/packages/compiler/src/compat/fetch-profile.ts +++ b/packages/compiler/src/compat/fetch-profile.ts @@ -900,7 +900,7 @@ export const NODE24_FETCH_COMPAT_PROFILE = { { name: "WritableStream/TransformStream", reason: - "native support currently covers readable fetch bodies; general Web Streams graphs are unsupported", + "native support covers readable fetch bodies and the identity `new TransformStream()`; the WritableStream constructor, transformers, and general Web Streams graphs are unsupported", }, { name: "Readable byte/BYOB streams", diff --git a/packages/compiler/src/frontend/lowering/lower-stdlib.ts b/packages/compiler/src/frontend/lowering/lower-stdlib.ts index e671a0a08..dc9d3def8 100644 --- a/packages/compiler/src/frontend/lowering/lower-stdlib.ts +++ b/packages/compiler/src/frontend/lowering/lower-stdlib.ts @@ -1945,6 +1945,19 @@ export function lowerStaticReadableStreamNew( }; } +/** `new TransformStream()`: the identity transform, a native handle whose + * readable side is a native ReadableStream and whose writable side is the + * runtime's WritableStream. A transformer or queuing strategies have no + * native lowering yet and keep the SC2020 refusal. */ +export function lowerStaticTransformStreamNew( + lowerer: Lowerer, + expr: ts.NewExpression, +): IrExpr | null { + if (stdlibGlobalNameOf(lowerer, expr.expression) !== "TransformStream") return null; + if ((expr.arguments?.nodes.length ?? 0) > 0) return null; + return { kind: "libCall", fn: "fetch.transformNew", args: [], type: DYN, loc: locOf(expr) }; +} + /** Native Request and Headers values share the fetch runtime's handles. */ export function lowerFetchWebNew(lowerer: Lowerer, expr: ts.NewExpression): IrExpr | null { const name = stdlibGlobalNameOf(lowerer, expr.expression); diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index e36ffd09f..80ff8b4ca 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -558,6 +558,7 @@ import { lowerStaticReadableStreamCancelCall, lowerStaticReadableStreamControllerCall, lowerStaticReadableStreamNew, + lowerStaticTransformStreamNew, lowerStaticReadableStreamReaderCall, lowerStaticResponseCall, lowerStdlibMethodCall, @@ -13423,6 +13424,7 @@ export class Lowerer { lowerBlobNew(this, expr) ?? lowerNetAddressNew(this, expr) ?? lowerStaticReadableStreamNew(this, expr) ?? + lowerStaticTransformStreamNew(this, expr) ?? lowerConsoleNew(this, expr); return ( builtin ?? this.withRuntimeOptionalClassValue(expr.expression, () => lowerNew(this, expr)) diff --git a/packages/compiler/src/frontend/type-mapper.ts b/packages/compiler/src/frontend/type-mapper.ts index f1e86067e..cbb468f98 100644 --- a/packages/compiler/src/frontend/type-mapper.ts +++ b/packages/compiler/src/frontend/type-mapper.ts @@ -213,6 +213,9 @@ export const FETCH_AMBIENT_TYPES = [ "ReadableStreamReadResult", "ReadableStreamReadValueResult", "ReadableStreamReadDoneResult", + "TransformStream", + "WritableStream", + "WritableStreamDefaultWriter", "Blob", "File", "BlobPropertyBag", @@ -3109,7 +3112,7 @@ function mapTypeInner(type: ts.Type, ctx: TypeMapperCtx): IrType | null { psym && (FETCH_AMBIENT_TYPES as readonly string[]).includes(ts.symbolName(psym)) && !( - ts.symbolName(psym) === "ReadableStream" && + (ts.symbolName(psym) === "ReadableStream" || ts.symbolName(psym) === "WritableStream") && psym.declarations.some( (d) => ts.isInterfaceDeclaration(d) && isDeclaredInAmbientNamespace(d, "NodeJS"), ) diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index 49d281d38..7c7df6adb 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -163,6 +163,7 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ "fetch.abortNow": { argTypes: [DYN], result: DYN }, "fetch.abortAny": { argTypes: [DYN], result: DYN }, "fetch.streamNew": { argTypes: [DYN], result: DYN }, + "fetch.transformNew": { argTypes: [], result: DYN }, // Program-dependent iterable: typed arrays/bytes/string stay intact so // the native stream can pull lazily; checked-dynamic values are the // fallback. The libCall validator in validate.ts checks the closed set. diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7743-transform-stream-identity.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7743-transform-stream-identity.ts.json new file mode 100644 index 000000000..0cfff542b --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7743-transform-stream-identity.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7743-transform-stream-identity.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_dyn.h b/packages/runtime/src/scr_dyn.h index 669f4aedb..d772f5414 100644 --- a/packages/runtime/src/scr_dyn.h +++ b/packages/runtime/src/scr_dyn.h @@ -212,6 +212,9 @@ typedef enum { X(WEB_CONTROLLER, 11) /* native ReadableStreamDefaultController */ \ X(WEB_READER, 10) /* native ReadableStreamDefaultReader */ \ X(WEB_STREAM, 9) /* native WHATWG ReadableStream */ \ + X(WEB_TRANSFORM, 73) /* native identity TransformStream (scr_fetch.c) */ \ + X(WEB_WRITABLE, 74) /* a TransformStream's WritableStream (scr_fetch.c) */ \ + X(WEB_WRITER, 75) /* WritableStreamDefaultWriter (scr_fetch.c) */ \ X(WORKER, 35) typedef enum { diff --git a/packages/runtime/src/scr_fetch.c b/packages/runtime/src/scr_fetch.c index 6049a4936..e92f488b2 100644 --- a/packages/runtime/src/scr_fetch.c +++ b/packages/runtime/src/scr_fetch.c @@ -147,6 +147,9 @@ struct SfStream { bool pull_again; bool discarded; bool request_discarding; + /* A zero high-water mark (a TransformStream's readable side): pull only + * while a read is pending. */ + bool hwm_zero; ScrDyn *error; SfReader *reader; /* owned while locked */ SfCollector *collector; /* owned while a body reader is active */ @@ -898,6 +901,7 @@ static void sf_stream_release(SfStream *s) { sf_response_discard_if_unobserved(s); return; } + scr_weak_dispose_inline(s); sf_stream_drop_chunks(s); if (s->reader) { s->reader->stream = NULL; @@ -1625,6 +1629,12 @@ static void sf_stream_pull(SfStream *s) { s->error) { return; } + /* ReadableStreamDefaultControllerShouldCallPull: with no room in the + * queue, only a pending read (or a body collector) asks for a chunk. */ + if (s->hwm_zero && !s->collector && + !(s->reader && s->reader->pending_head)) { + return; + } if (s->pulling) { s->pull_again = true; return; @@ -1852,6 +1862,7 @@ static ScrPromise *sf_stream_collect(SfStream *s, int mode) { c->mode = mode; s->collector = c; sf_stream_drain(s); + if (s->hwm_zero && !s->head) sf_stream_pull(s); return p; } @@ -2664,6 +2675,895 @@ static ScrDyn *sf_headers_get(void *ptr, const char *key, size_t len) { return NULL; } +/* ── TransformStream / WritableStream ────────────────────────────── */ + +/* + * The identity TransformStream (`new TransformStream()`): a native + * readable side (an SfStream with a zero high-water mark, fed by the + * transform) and a writable side implemented here with the streams spec's + * WritableStream state machine (default high-water mark 1, chunk size 1). + * The writable side has no other sink, so the TransformStream object, its + * WritableStream and the sink share one SfTransform; the handles differ + * only by tag. Promise reactions the spec chains on already-settled + * promises run after the same number of microtask turns as in Node's + * implementation (lib/internal/webstreams), so interleavings with user + * code match. + * + * Ownership: the transform owns its readable side and its writer; the + * readable side's pull/cancel callbacks own the transform until that side + * closes or errors (fetch teardown drops the callbacks of transforms + * nobody finished). A writer handle retains the transform while bound, + * like a reader handle retains its stream. + */ +enum { SFW_WRITABLE, SFW_ERRORING, SFW_ERRORED, SFW_CLOSED }; + +typedef struct SfTransform SfTransform; + +typedef struct SfWriteReq { + ScrDyn *chunk; + ScrPromise *promise; + struct SfWriteReq *next; +} SfWriteReq; + +typedef struct SfWriter { + size_t rc; + size_t handle_refs; + SfTransform *stream; /* weak; handle refs retain it while bound */ + ScrPromise *ready; + ScrPromise *closed; +} SfWriter; + +struct SfTransform { + size_t rc; + SfStream *readable; + /* WritableStream */ + bool started; /* the start algorithm's promise settled (one turn) */ + int state; + ScrDyn *stored_error; + SfWriter *writer; /* owned while locked */ + SfWriteReq *head; /* queued writes (chunk + request promise) */ + SfWriteReq *tail; + SfWriteReq *in_flight; + size_t queue_size; /* queueTotalSize: queued chunks + the in-flight one */ + bool close_queued; /* the close sentinel is in the queue */ + ScrPromise *close_request; + ScrPromise *in_flight_close; + bool has_pending_abort; + ScrPromise *pending_abort; + ScrDyn *pending_abort_reason; + bool pending_abort_was_erroring; + bool backpressure; + /* TransformStream */ + bool t_backpressure; + ScrPromise *bp_change; + bool write_waiting; /* the in-flight write waits for bp_change */ + ScrDyn *write_error; /* the in-flight write's failure, if any */ + ScrDyn *cancel_reason; /* the readable side's cancel reason, until it reacts */ + ScrPromise *finish; /* finishPromise of close/abort/cancel */ +}; + +static SfTransform *sf_transform_retain(SfTransform *t) { + t->rc++; + return t; +} + +static void sf_write_req_free(SfWriteReq *w) { + if (!w) return; + scr_dyn_release(w->chunk); + scr_promise_release(w->promise); + free(w); +} + +static void sf_writer_release(SfWriter *w) { + if (!w || --w->rc > 0) return; + scr_promise_release(w->ready); + scr_promise_release(w->closed); + free(w); +} + +static void sf_transform_release(SfTransform *t) { + if (!t || --t->rc > 0) return; + sf_stream_release(t->readable); + scr_dyn_release(t->stored_error); + if (t->writer) { + t->writer->stream = NULL; + sf_writer_release(t->writer); + } + while (t->head) { + SfWriteReq *w = t->head; + t->head = w->next; + sf_write_req_free(w); + } + sf_write_req_free(t->in_flight); + scr_promise_release(t->close_request); + scr_promise_release(t->in_flight_close); + scr_promise_release(t->pending_abort); + scr_dyn_release(t->pending_abort_reason); + scr_promise_release(t->bp_change); + scr_promise_release(t->finish); + scr_dyn_release(t->write_error); + scr_dyn_release(t->cancel_reason); + free(t); +} + +static void *sf_transform_retain_v(void *p) { + return sf_transform_retain((SfTransform *)p); +} +static void sf_transform_release_v(void *p) { + sf_transform_release((SfTransform *)p); +} + +static void *sf_writer_handle_retain_v(void *p) { + SfWriter *w = p; + w->rc++; + w->handle_refs++; + if (w->stream) sf_transform_retain(w->stream); + return w; +} + +static void sf_writer_handle_release_v(void *p) { + SfWriter *w = p; + SfTransform *t = w->stream; + if (w->handle_refs > 0) w->handle_refs--; + sf_writer_release(w); + sf_transform_release(t); +} + +static ScrBox *sf_transform_box(SfTransform *t) { + ScrBox *box = scr_box_new_obj(&sf_transform_retain_v, + &sf_transform_release_v, NULL); + scr_box_set_ref(box, sf_transform_retain(t)); + return box; +} + +static void sf_invalid_state(const char *message) { + char buf[96]; + int n = snprintf(buf, sizeof buf, "Invalid state: %s", message); + scr_throw_error_msg_code(SCR_ERR_TYPE, buf, (size_t)n, "ERR_INVALID_STATE"); +} + +static ScrPromise *sf_promise_invalid_state(const char *message) { + ScrPromise *p = scr_promise_new(); + sf_invalid_state(message); + scr_promise_reject_pending(p); + return p; +} + +static ScrPromise *sf_promise_rejected_with(ScrDyn *reason) { + ScrPromise *p = scr_promise_new(); + sf_reject_promise_reason(p, reason); + return p; +} + +static ScrPromise *sf_promise_resolved(void) { + ScrPromise *p = scr_promise_new(); + scr_promise_fulfill_void(p); + return p; +} + +/* Deferred steps: the reaction `op` after `ticks` microtask turns. */ +enum { + SFT_WRITE_DONE, + SFT_WRITE_CONTINUE, + SFT_CLOSE_DONE, + SFT_ABORT_DONE, + SFT_START, + SFT_CANCEL_DONE, +}; + +static void sf_transform_step(SfTransform *t, int op); + +typedef struct SfTransformTask { + size_t rc; + SfTransform *t; + int op; + int ticks; +} SfTransformTask; + +static void *sf_transform_task_retain_v(void *p) { + ((SfTransformTask *)p)->rc++; + return p; +} + +static void sf_transform_task_release_v(void *p) { + SfTransformTask *task = p; + if (--task->rc > 0) return; + sf_transform_release(task->t); + free(task); +} + +static void sf_transform_tick_task(ScrClosure *cb) { + SfTransformTask *task = (SfTransformTask *)scr_box_get_ref(cb->caps[0]); + if (--task->ticks > 0) { + ScrClosure *next = scr_closure_new((void *)&sf_transform_tick_task, 1); + next->caps[0] = scr_box_new_obj(&sf_transform_task_retain_v, + &sf_transform_task_release_v, NULL); + scr_box_set_ref(next->caps[0], sf_transform_task_retain_v(task)); + scr_queue_microtask(next); + } else { + sf_transform_step(task->t, task->op); + } + sf_transform_task_release_v(task); +} + +static void sf_transform_defer(SfTransform *t, int op, int ticks) { + SfTransformTask *task = calloc(1, sizeof *task); + if (!task) sf_oom(); + task->rc = 1; + task->t = sf_transform_retain(t); + task->op = op; + task->ticks = ticks; + ScrClosure *cb = scr_closure_new((void *)&sf_transform_tick_task, 1); + cb->caps[0] = scr_box_new_obj(&sf_transform_task_retain_v, + &sf_transform_task_release_v, NULL); + scr_box_set_ref(cb->caps[0], task); /* the box takes the reference */ + scr_queue_microtask(cb); +} + +/* ── WritableStream abstract operations ── */ + +static bool sfw_close_queued_or_in_flight(const SfTransform *t) { + return t->close_request || t->in_flight_close; +} + +static bool sfw_has_operation_in_flight(const SfTransform *t) { + return t->in_flight || t->in_flight_close; +} + +static void sfw_writer_ensure_ready_rejected(SfWriter *w, ScrDyn *reason) { + if (!scr_promise_is_pending(w->ready)) { + scr_promise_release(w->ready); + w->ready = scr_promise_new(); + } + sf_reject_promise_reason(w->ready, reason); + scr_promise_mark_handled(w->ready); +} + +static void sfw_writer_ensure_closed_rejected(SfWriter *w, ScrDyn *reason) { + if (!scr_promise_is_pending(w->closed)) { + scr_promise_release(w->closed); + w->closed = scr_promise_new(); + } + sf_reject_promise_reason(w->closed, reason); + scr_promise_mark_handled(w->closed); +} + +static void sfw_update_backpressure(SfTransform *t, bool backpressure) { + SfWriter *w = t->writer; + if (w && backpressure != t->backpressure) { + if (backpressure) { + scr_promise_release(w->ready); + w->ready = scr_promise_new(); + scr_promise_mark_handled(w->ready); + } else { + scr_promise_fulfill_void(w->ready); + } + } + t->backpressure = backpressure; +} + +static void sfw_reject_close_and_closed(SfTransform *t) { + if (t->close_request) { + sf_reject_promise_reason(t->close_request, t->stored_error); + scr_promise_release(t->close_request); + t->close_request = NULL; + } + if (t->writer) { + sf_reject_promise_reason(t->writer->closed, t->stored_error); + scr_promise_mark_handled(t->writer->closed); + } +} + +static void sft_abort_algorithm(SfTransform *t, ScrDyn *reason); + +static void sfw_finish_erroring(SfTransform *t) { + t->state = SFW_ERRORED; + /* The controller's queue resets; every queued write rejects. */ + while (t->head) { + SfWriteReq *w = t->head; + t->head = w->next; + sf_reject_promise_reason(w->promise, t->stored_error); + sf_write_req_free(w); + } + t->tail = NULL; + t->queue_size = 0; + t->close_queued = false; + if (!t->has_pending_abort) { + sfw_reject_close_and_closed(t); + return; + } + if (t->pending_abort_was_erroring) { + t->has_pending_abort = false; + sf_reject_promise_reason(t->pending_abort, t->stored_error); + scr_promise_release(t->pending_abort); + t->pending_abort = NULL; + scr_dyn_release(t->pending_abort_reason); + t->pending_abort_reason = NULL; + sfw_reject_close_and_closed(t); + return; + } + /* The sink's abort runs; its promise settles the abort request. */ + sft_abort_algorithm(t, t->pending_abort_reason); +} + +static void sfw_start_erroring(SfTransform *t, ScrDyn *reason) { + t->state = SFW_ERRORING; + scr_dyn_release(t->stored_error); + t->stored_error = scr_dyn_retain(reason); + if (t->writer) sfw_writer_ensure_ready_rejected(t->writer, reason); + if (!sfw_has_operation_in_flight(t) && t->started) sfw_finish_erroring(t); +} + +static void sfw_deal_with_rejection(SfTransform *t, ScrDyn *error) { + if (t->state == SFW_WRITABLE) { + sfw_start_erroring(t, error); + return; + } + sfw_finish_erroring(t); +} + +static void sfw_error_if_needed(SfTransform *t, ScrDyn *error) { + if (t->state == SFW_WRITABLE) sfw_start_erroring(t, error); +} + +static void sft_set_backpressure(SfTransform *t, bool backpressure); +static void sft_sink_write(SfTransform *t); +static void sft_sink_close(SfTransform *t); + +static void sfw_advance_queue(SfTransform *t) { + if (!t->started || t->in_flight || t->in_flight_close) return; + if (t->state == SFW_ERRORING) { + sfw_finish_erroring(t); + return; + } + if (t->head) { + SfWriteReq *w = t->head; + t->head = w->next; + if (!t->head) t->tail = NULL; + w->next = NULL; + t->in_flight = w; + sft_sink_write(t); + return; + } + if (t->close_queued) { + t->close_queued = false; + t->in_flight_close = t->close_request; + t->close_request = NULL; + sft_sink_close(t); + } +} + +static void sfw_finish_write(SfTransform *t, ScrDyn *error) { + SfWriteReq *w = t->in_flight; + t->in_flight = NULL; + if (error) { + sf_reject_promise_reason(w->promise, error); + sf_write_req_free(w); + sfw_deal_with_rejection(t, error); + return; + } + scr_promise_fulfill_void(w->promise); + sf_write_req_free(w); + if (t->queue_size) t->queue_size--; + if (!sfw_close_queued_or_in_flight(t) && t->state == SFW_WRITABLE) { + sfw_update_backpressure(t, t->queue_size >= 1); + } + sfw_advance_queue(t); +} + +static void sfw_finish_close(SfTransform *t, ScrDyn *error) { + ScrPromise *p = t->in_flight_close; + t->in_flight_close = NULL; + if (error) { + sf_reject_promise_reason(p, error); + scr_promise_release(p); + if (t->has_pending_abort) { + t->has_pending_abort = false; + sf_reject_promise_reason(t->pending_abort, error); + scr_promise_release(t->pending_abort); + t->pending_abort = NULL; + scr_dyn_release(t->pending_abort_reason); + t->pending_abort_reason = NULL; + } + sfw_deal_with_rejection(t, error); + return; + } + scr_promise_fulfill_void(p); + scr_promise_release(p); + if (t->state == SFW_ERRORING) { + scr_dyn_release(t->stored_error); + t->stored_error = NULL; + if (t->has_pending_abort) { + t->has_pending_abort = false; + scr_promise_fulfill_void(t->pending_abort); + scr_promise_release(t->pending_abort); + t->pending_abort = NULL; + scr_dyn_release(t->pending_abort_reason); + t->pending_abort_reason = NULL; + } + } + t->state = SFW_CLOSED; + if (t->writer) scr_promise_fulfill_void(t->writer->closed); +} + +/* WritableStreamAbort: +1 promise. */ +static ScrPromise *sfw_abort(SfTransform *t, ScrDyn *reason) { + if (t->state == SFW_CLOSED || t->state == SFW_ERRORED) { + return sf_promise_resolved(); + } + if (t->has_pending_abort) return scr_promise_retain(t->pending_abort); + bool was_erroring = t->state == SFW_ERRORING; + ScrPromise *p = scr_promise_new(); + t->has_pending_abort = true; + t->pending_abort = scr_promise_retain(p); + t->pending_abort_reason = + scr_dyn_retain(was_erroring ? scr_dyn_undefined() : reason); + t->pending_abort_was_erroring = was_erroring; + if (!was_erroring) sfw_start_erroring(t, reason); + return p; +} + +/* WritableStreamClose: +1 promise. */ +static ScrPromise *sfw_close(SfTransform *t) { + if (t->state == SFW_CLOSED || t->state == SFW_ERRORED) { + char message[64]; + snprintf(message, sizeof message, "WritableStream is %s", + t->state == SFW_CLOSED ? "closed" : "errored"); + return sf_promise_invalid_state(message); + } + ScrPromise *p = scr_promise_new(); + t->close_request = scr_promise_retain(p); + if (t->writer && t->backpressure && t->state == SFW_WRITABLE) { + scr_promise_fulfill_void(t->writer->ready); + } + t->close_queued = true; + sfw_advance_queue(t); + return p; +} + +/* WritableStreamDefaultWriterWrite: +1 promise. */ +static ScrPromise *sfw_writer_write(SfWriter *w, ScrDyn *chunk) { + SfTransform *t = w->stream; + if (!t) return sf_promise_invalid_state("Writer is not bound to a WritableStream"); + if (t->state == SFW_ERRORED || t->state == SFW_ERRORING) { + return sf_promise_rejected_with(t->stored_error); + } + if (sfw_close_queued_or_in_flight(t) || t->state == SFW_CLOSED) { + return sf_promise_invalid_state("WritableStream is closed"); + } + SfWriteReq *req = calloc(1, sizeof *req); + if (!req) sf_oom(); + req->chunk = scr_dyn_retain(chunk); + req->promise = scr_promise_new(); + ScrPromise *p = scr_promise_retain(req->promise); + if (t->tail) t->tail->next = req; + else t->head = req; + t->tail = req; + t->queue_size++; + sfw_update_backpressure(t, t->queue_size >= 1); + sfw_advance_queue(t); + return p; +} + +static SfWriter *sfw_get_writer(SfTransform *t) { + if (t->writer) { + sf_invalid_state("WritableStream is locked"); + return NULL; + } + SfWriter *w = calloc(1, sizeof *w); + if (!w) sf_oom(); + w->rc = 1; + w->stream = t; + w->ready = scr_promise_new(); + w->closed = scr_promise_new(); + scr_promise_mark_handled(w->ready); + scr_promise_mark_handled(w->closed); + switch (t->state) { + case SFW_WRITABLE: + if (sfw_close_queued_or_in_flight(t) || !t->backpressure) { + scr_promise_fulfill_void(w->ready); + } + break; + case SFW_ERRORING: + sf_reject_promise_reason(w->ready, t->stored_error); + break; + case SFW_CLOSED: + scr_promise_fulfill_void(w->ready); + scr_promise_fulfill_void(w->closed); + break; + default: + sf_reject_promise_reason(w->ready, t->stored_error); + sf_reject_promise_reason(w->closed, t->stored_error); + break; + } + t->writer = w; + w->rc++; /* the stream's ownership */ + return w; +} + +static void sfw_writer_release_lock(SfWriter *w) { + SfTransform *t = w->stream; + if (!t) return; + sf_invalid_state("Writer has been released"); + ScrCaught *caught = scr_exc_take(); + ScrDyn *released = scr_caught_to_dyn(caught); + scr_caught_release(caught); + sfw_writer_ensure_ready_rejected(w, released); + sfw_writer_ensure_closed_rejected(w, released); + scr_dyn_release(released); + size_t handle_refs = w->handle_refs; + t->writer = NULL; + w->stream = NULL; + sf_writer_release(w); /* the stream's ownership */ + while (handle_refs-- > 0) sf_transform_release(t); +} + +/* ── TransformStream algorithms (identity transformer) ── */ + +static void sft_set_backpressure(SfTransform *t, bool backpressure) { + if (t->bp_change) { + scr_promise_fulfill_void(t->bp_change); + scr_promise_release(t->bp_change); + } + t->bp_change = scr_promise_new(); + t->t_backpressure = backpressure; + if (t->write_waiting) { + t->write_waiting = false; + sf_transform_defer(t, SFT_WRITE_CONTINUE, 1); + } +} + +static bool sft_readable_wants_chunk(const SfStream *r) { + return r->collector || (r->reader && r->reader->pending_head); +} + +static bool sft_readable_can_enqueue(const SfStream *r) { + return !r->close_requested && !r->closed && !r->error; +} + +/* TransformStreamErrorWritableAndUnblockWrite */ +static void sft_error_writable(SfTransform *t, ScrDyn *error) { + sfw_error_if_needed(t, error); + if (t->t_backpressure) sft_set_backpressure(t, false); +} + +/* The identity transform's enqueue: NULL on success, else +1 error. */ +static ScrDyn *sft_perform_transform(SfTransform *t, ScrDyn *chunk) { + SfStream *r = t->readable; + if (!sft_readable_can_enqueue(r)) { + sf_invalid_state("Unable to enqueue"); + ScrCaught *caught = scr_exc_take(); + ScrDyn *error = scr_caught_to_dyn(caught); + scr_caught_release(caught); + sft_error_writable(t, error); + return error; + } + sf_stream_enqueue_value(r, chunk); + if (scr_exc_pending()) { + ScrCaught *caught = scr_exc_take(); + ScrDyn *error = scr_caught_to_dyn(caught); + scr_caught_release(caught); + sf_stream_error(r, error); + sft_error_writable(t, error); + return error; + } + bool backpressure = !sft_readable_wants_chunk(r); + if (backpressure != t->t_backpressure) sft_set_backpressure(t, true); + return NULL; +} + +static void sft_sink_write(SfTransform *t) { + if (t->t_backpressure) { + t->write_waiting = true; + return; + } + ScrDyn *error = sft_perform_transform(t, t->in_flight->chunk); + if (error) { + scr_dyn_release(t->write_error); + t->write_error = error; + } + sf_transform_defer(t, SFT_WRITE_DONE, 2); +} + +static void sft_sink_close(SfTransform *t) { + if (!t->finish) { + t->finish = scr_promise_new(); + scr_promise_mark_handled(t->finish); + sf_transform_defer(t, SFT_CLOSE_DONE, 1); + return; + } + /* A cancel or abort already finished the transform. */ + sf_transform_defer(t, SFT_CLOSE_DONE, 1); +} + +static void sft_abort_algorithm(SfTransform *t, ScrDyn *reason) { + (void)reason; + if (!t->finish) { + t->finish = scr_promise_new(); + scr_promise_mark_handled(t->finish); + } + sf_transform_defer(t, SFT_ABORT_DONE, 1); +} + +static void sf_transform_step(SfTransform *t, int op) { + switch (op) { + case SFT_WRITE_CONTINUE: { + if (!t->in_flight) return; + if (t->state == SFW_ERRORING) { + ScrDyn *error = scr_dyn_retain(t->stored_error); + scr_dyn_release(t->write_error); + t->write_error = error; + } else { + ScrDyn *error = sft_perform_transform(t, t->in_flight->chunk); + if (error) { + scr_dyn_release(t->write_error); + t->write_error = error; + } + } + sf_transform_defer(t, SFT_WRITE_DONE, 3); + return; + } + case SFT_WRITE_DONE: { + if (!t->in_flight) return; + ScrDyn *error = t->write_error; + t->write_error = NULL; + sfw_finish_write(t, error); + scr_dyn_release(error); + return; + } + case SFT_CLOSE_DONE: { + SfStream *r = t->readable; + ScrDyn *error = NULL; + if (r->error) { + error = scr_dyn_retain(r->error); + } else { + sf_stream_close(r); + } + if (error) sf_reject_promise_reason(t->finish, error); + else scr_promise_fulfill_void(t->finish); + if (t->in_flight_close) sfw_finish_close(t, error); + scr_dyn_release(error); + return; + } + case SFT_ABORT_DONE: { + SfStream *r = t->readable; + ScrDyn *reason = t->pending_abort_reason + ? scr_dyn_retain(t->pending_abort_reason) + : scr_dyn_retain(scr_dyn_undefined()); + bool readable_errored = r->error != NULL; + if (!readable_errored) sf_stream_error(r, reason); + if (t->has_pending_abort) { + t->has_pending_abort = false; + if (readable_errored) { + sf_reject_promise_reason(t->pending_abort, r->error); + } else { + scr_promise_fulfill_void(t->pending_abort); + } + scr_promise_release(t->pending_abort); + t->pending_abort = NULL; + scr_dyn_release(t->pending_abort_reason); + t->pending_abort_reason = NULL; + } + sfw_reject_close_and_closed(t); + scr_dyn_release(reason); + return; + } + case SFT_CANCEL_DONE: { + ScrDyn *reason = t->cancel_reason; + t->cancel_reason = NULL; + if (t->state == SFW_ERRORED) { + sf_reject_promise_reason(t->finish, t->stored_error); + } else { + sfw_error_if_needed(t, reason ? reason : scr_dyn_undefined()); + if (t->t_backpressure) sft_set_backpressure(t, false); + scr_promise_fulfill_void(t->finish); + } + scr_dyn_release(reason); + return; + } + case SFT_START: { + /* Both sides start one turn after construction, the writable side + * first (Node registers its start reaction first). */ + t->started = true; + sfw_advance_queue(t); + SfStream *r = t->readable; + r->started = true; + if (!r->close_requested && !r->closed && !r->error) { + sf_stream_drain(r); + sf_stream_pull(r); + } + return; + } + } +} + +/* The readable side's pull: release backpressure; the pull completes when + * backpressure changes again. */ +static ScrDyn *sft_pull_thunk(ScrClosure *cb, ScrDyn *const *args, + size_t argc) { + (void)args; + (void)argc; + SfTransform *t = (SfTransform *)scr_box_get_ref(cb->caps[0]); + sft_set_backpressure(t, false); + ScrDyn *out = scr_dyn_new_promise(t->bp_change); + sf_transform_release(t); + return out; +} + +/* The readable side's cancel (TransformStreamDefaultSourceCancelAlgorithm): + * errors the writable side with the reason and unblocks a waiting write. */ +static ScrDyn *sft_cancel_thunk(ScrClosure *cb, ScrDyn *const *args, + size_t argc) { + SfTransform *t = (SfTransform *)scr_box_get_ref(cb->caps[0]); + ScrDyn *reason = argc ? args[0] : scr_dyn_undefined(); + if (!t->finish) { + t->finish = scr_promise_new(); + scr_promise_mark_handled(t->finish); + scr_dyn_release(t->cancel_reason); + t->cancel_reason = scr_dyn_retain(reason); + /* The (identity) cancel algorithm's resolved promise reacts a turn + * later. */ + sf_transform_defer(t, SFT_CANCEL_DONE, 1); + } + ScrDyn *out = scr_dyn_new_promise(t->finish); + sf_transform_release(t); + return out; +} + +static ScrDyn *sft_func(SfTransform *t, ScrDynThunk thunk, const char *name) { + ScrClosure *cb = scr_closure_new((void *)thunk, 1); + cb->caps[0] = sf_transform_box(t); + return scr_dyn_new_func(cb, thunk, 1, "(controller)", name); +} + +ScrDyn *scr_fetch_transform_new(void) { + SfTransform *t = calloc(1, sizeof *t); + if (!t) sf_oom(); + t->rc = 1; + t->state = SFW_WRITABLE; + SfStream *r = sf_stream_new_native(); + r->started = false; + r->hwm_zero = true; + r->pull_cb = sft_func(t, &sft_pull_thunk, "pull"); + r->cancel_cb = sft_func(t, &sft_cancel_thunk, "cancel"); + r->source_this = scr_dyn_retain(scr_dyn_undefined()); + sf_stream_track_callbacks(r); + t->readable = r; + /* The writable side's high-water mark of 1 leaves room: no backpressure. + * The readable side's 0 does: writes wait for a read. */ + t->backpressure = false; + sft_set_backpressure(t, true); + sf_transform_defer(t, SFT_START, 1); + ScrDyn *out = scr_dyn_new_handle(t, SCR_DYNH_WEB_TRANSFORM); + sf_transform_release(t); + return out; +} + +static ScrDyn *sf_transform_get(void *ptr, const char *key, size_t len) { + SfTransform *t = ptr; + if (sf_name(key, len, "readable")) { + return scr_dyn_new_handle(t->readable, SCR_DYNH_WEB_STREAM); + } + if (sf_name(key, len, "writable")) { + return scr_dyn_new_handle(t, SCR_DYNH_WEB_WRITABLE); + } + return NULL; +} + +static ScrDyn *sf_transform_invoke(void *ptr, ScrDyn *self, + const char *method, ScrDyn *const *args, + size_t argc, const char *what) { + (void)ptr; + (void)self; + (void)args; + (void)argc; + (void)method; + sf_not_function(what); + return NULL; +} + +static ScrDyn *sf_promise_dyn(ScrPromise *p) { + ScrDyn *out = scr_dyn_new_promise(p); + scr_promise_release(p); + return out; +} + +static ScrDyn *sf_writable_invoke(void *ptr, ScrDyn *self, + const char *method, ScrDyn *const *args, + size_t argc, const char *what) { + SfTransform *t = ptr; + (void)self; + if (strcmp(method, "getWriter") == 0) { + SfWriter *w = sfw_get_writer(t); + if (!w) return NULL; + ScrDyn *out = scr_dyn_new_handle(w, SCR_DYNH_WEB_WRITER); + sf_writer_release(w); + return out; + } + if (strcmp(method, "abort") == 0) { + if (t->writer) { + return sf_promise_dyn( + sf_promise_invalid_state("WritableStream is locked")); + } + return sf_promise_dyn(sfw_abort(t, argc ? args[0] : scr_dyn_undefined())); + } + if (strcmp(method, "close") == 0) { + if (t->writer) { + return sf_promise_dyn( + sf_promise_invalid_state("WritableStream is locked")); + } + if (sfw_close_queued_or_in_flight(t)) { + return sf_promise_dyn( + sf_promise_invalid_state("WritableStream is closed")); + } + return sf_promise_dyn(sfw_close(t)); + } + sf_not_function(what); + return NULL; +} + +static ScrDyn *sf_writable_get(void *ptr, const char *key, size_t len) { + SfTransform *t = ptr; + if (sf_name(key, len, "locked")) return scr_dyn_new_bool(t->writer != NULL); + return NULL; +} + +static ScrDyn *sf_writer_invoke(void *ptr, ScrDyn *self, const char *method, + ScrDyn *const *args, size_t argc, + const char *what) { + SfWriter *w = ptr; + (void)self; + if (strcmp(method, "write") == 0) { + return sf_promise_dyn( + sfw_writer_write(w, argc ? args[0] : scr_dyn_undefined())); + } + if (strcmp(method, "close") == 0) { + SfTransform *t = w->stream; + if (!t) { + return sf_promise_dyn(sf_promise_invalid_state( + "Writer is not bound to a WritableStream")); + } + if (sfw_close_queued_or_in_flight(t)) { + return sf_promise_dyn( + sf_promise_invalid_state("WritableStream is closed")); + } + return sf_promise_dyn(sfw_close(t)); + } + if (strcmp(method, "abort") == 0) { + SfTransform *t = w->stream; + if (!t) { + return sf_promise_dyn(sf_promise_invalid_state( + "Writer is not bound to a WritableStream")); + } + return sf_promise_dyn(sfw_abort(t, argc ? args[0] : scr_dyn_undefined())); + } + if (strcmp(method, "releaseLock") == 0) { + sfw_writer_release_lock(w); + return scr_dyn_retain(scr_dyn_undefined()); + } + sf_not_function(what); + return NULL; +} + +static ScrDyn *sf_writer_get(void *ptr, const char *key, size_t len) { + SfWriter *w = ptr; + if (sf_name(key, len, "ready")) return scr_dyn_new_promise(w->ready); + if (sf_name(key, len, "closed")) return scr_dyn_new_promise(w->closed); + if (sf_name(key, len, "desiredSize")) { + SfTransform *t = w->stream; + if (!t) { + sf_invalid_state("Writer is not bound to a WritableStream"); + return NULL; + } + if (t->state == SFW_ERRORED || t->state == SFW_ERRORING) { + return scr_dyn_new_null(); + } + if (t->state == SFW_CLOSED) return scr_dyn_new_num(0); + return scr_dyn_new_num(1.0 - (double)t->queue_size); + } + return NULL; +} + /* ── Response ────────────────────────────────────────────────────── */ static SfResponse *sf_response_retain(SfResponse *r) { @@ -5675,6 +6575,13 @@ static const ScrDynHandleMethod sf_headers_methods[] = { {"has", 1, false}, {"keys", 0, false}, {"set", 2, false}, {"values", 0, false}, {NULL, 0, false}}; +static const ScrDynHandleMethod sf_writable_methods[] = { + {"abort", 0, true}, {"close", 0, true}, {"getWriter", 0, false}, + {NULL, 0, false}}; +static const ScrDynHandleMethod sf_writer_methods[] = { + {"abort", 0, true}, {"close", 0, true}, {"releaseLock", 0, false}, + {"write", 0, true}, {NULL, 0, false}}; + static ScrDyn *sf_rejected(void) { ScrPromise *p = scr_promise_settled_ref(NULL, &scr_dyn_retain_v, &scr_dyn_release_v, NULL); @@ -5721,6 +6628,23 @@ static const ScrDynHandleOps sf_headers_ops = { &sf_headers_invoke, &sf_headers_get, &sf_no_set, NULL, NULL, &sf_headers_step, .methods = sf_headers_methods}; +static const ScrDynHandleOps sf_transform_ops = { + "TransformStream", &sf_transform_retain_v, &sf_transform_release_v, + &sf_transform_invoke, &sf_transform_get, &sf_no_set, NULL}; +static const ScrDynHandleOps sf_writable_ops = { + "WritableStream", &sf_transform_retain_v, &sf_transform_release_v, + &sf_writable_invoke, &sf_writable_get, &sf_no_set, NULL, + .methods = sf_writable_methods, + .invalid_this = "Value of \"this\" must be of type WritableStream", + .rejected = &sf_rejected}; +static const ScrDynHandleOps sf_writer_ops = { + "WritableStreamDefaultWriter", &sf_writer_handle_retain_v, + &sf_writer_handle_release_v, &sf_writer_invoke, &sf_writer_get, + &sf_no_set, NULL, .methods = sf_writer_methods, + .invalid_this = + "Value of \"this\" must be of type WritableStreamDefaultWriter", + .rejected = &sf_rejected}; + static void sf_teardown(void) { while (sf_live) { SfTransfer *t = sf_live; @@ -5760,6 +6684,9 @@ void scr_fetch_install(void) { scr_dyn_handle_install(SCR_DYNH_FETCH_REQUEST, &sf_request_ops); scr_dyn_handle_install(SCR_DYNH_FETCH_RESPONSE, &sf_response_ops); scr_dyn_handle_install(SCR_DYNH_FETCH_HEADERS, &sf_headers_ops); + scr_dyn_handle_install(SCR_DYNH_WEB_TRANSFORM, &sf_transform_ops); + scr_dyn_handle_install(SCR_DYNH_WEB_WRITABLE, &sf_writable_ops); + scr_dyn_handle_install(SCR_DYNH_WEB_WRITER, &sf_writer_ops); scr_et_signal_core_fn = sf_signal_core_of; scr_atexit(sf_teardown); } diff --git a/packages/runtime/src/scr_generator.h b/packages/runtime/src/scr_generator.h index 828f339d5..c73ceb900 100644 --- a/packages/runtime/src/scr_generator.h +++ b/packages/runtime/src/scr_generator.h @@ -192,6 +192,7 @@ ScrDyn *scr_fetch_abort_timeout(ScrDyn *delay); ScrDyn *scr_fetch_abort_now(ScrDyn *reason); /* borrowed reason; +1 handle */ ScrDyn *scr_fetch_abort_any(ScrDyn *signals); /* borrowed array; +1 handle or NULL pending */ ScrDyn *scr_fetch_stream_new(ScrDyn *source); /* borrowed source; +1 handle or NULL pending */ +ScrDyn *scr_fetch_transform_new(void); /* +1 identity TransformStream handle */ ScrDyn *scr_fetch_stream_from(ScrDyn *iterable); /* borrowed iterable; +1 handle or NULL pending */ ScrDyn *scr_fetch_stream_from_array( ScrArr *iterable, diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index 14d0a901f..f23814526 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -15054,6 +15054,7 @@ static int scr_weak_key(const ScrDyn *key, void **ptr, unsigned *kind) { key->v.handle.tag != SCR_DYNH_WEAK_SET && key->v.handle.tag != SCR_DYNH_FINALIZATION_REGISTRY && key->v.handle.tag != SCR_DYNH_STDIO && key->v.handle.tag != SCR_DYNH_FETCH_REQUEST && key->v.handle.tag != SCR_DYNH_FETCH_RESPONSE && + key->v.handle.tag != SCR_DYNH_WEB_STREAM && key->v.handle.tag != SCR_DYNH_MAP && key->v.handle.tag != SCR_DYNH_SET && key->v.handle.tag != SCR_DYNH_NET_SOCKET && key->v.handle.tag != SCR_DYNH_NET_SERVER) return -1; *ptr = key->v.handle.ptr; *kind = 4 + key->v.handle.tag; return 1; diff --git a/tests/corpus/7743-transform-stream-identity.ts b/tests/corpus/7743-transform-stream-identity.ts new file mode 100644 index 000000000..d98303997 --- /dev/null +++ b/tests/corpus/7743-transform-stream-identity.ts @@ -0,0 +1,212 @@ +// The identity TransformStream: writes wait for reads (the readable side's +// high-water mark is 0), the writer's desiredSize/ready track the writable +// side's queue, close/abort/cancel propagate across the pair, and the +// readable side feeds a Response body (the StreamingApi pattern of Hono's +// stream helpers). +const enc = new TextEncoder(); +const dec = new TextDecoder(); + +function describe(e: unknown): string { + if (e instanceof Error) { + const code = (e as NodeJS.ErrnoException).code; + return `${e.name}: ${e.message}${code === undefined ? "" : ` [${code}]`}`; + } + return `value ${String(e)}`; +} + +async function basics(): Promise { + const ts = new TransformStream(); + const writer = ts.writable.getWriter(); + const reader = ts.readable.getReader(); + console.log("desired", writer.desiredSize, ts.writable.locked, ts.readable.locked); + const written = writer.write("a").then(() => console.log("write a done")); + console.log("desired after write", writer.desiredSize); + void writer.ready.then(() => console.log("ready")); + const first = await reader.read(); + console.log("read", first.done, first.value); + await written; + console.log("desired", writer.desiredSize); + const closing = writer.close().then(() => console.log("closed")); + const last = await reader.read(); + console.log("read", last.done, last.value); + await closing; + await writer.closed; + console.log("writer.closed resolved"); + try { + await writer.write("b"); + } catch (e) { + console.log("write after close:", describe(e)); + } + try { + ts.writable.getWriter(); + } catch (e) { + console.log("getWriter locked:", describe(e)); + } + writer.releaseLock(); + try { + await writer.write("c"); + } catch (e) { + console.log("released write:", describe(e)); + } + try { + await writer.closed; + } catch (e) { + console.log("released closed:", describe(e)); + } +} + +async function abortWriter(): Promise { + const ts = new TransformStream(); + const writer = ts.writable.getWriter(); + const reader = ts.readable.getReader(); + const pending = writer.write("z").catch((e: unknown) => console.log("pending write:", describe(e))); + await writer.abort("boom").then(() => console.log("abort resolved")); + await pending; + try { + await reader.read(); + } catch (e) { + console.log("read after abort:", describe(e)); + } + try { + await writer.closed; + } catch (e) { + console.log("closed after abort:", describe(e)); + } + try { + await writer.ready; + } catch (e) { + console.log("ready after abort:", describe(e)); + } + console.log("desired after abort", writer.desiredSize); +} + +async function cancelReadable(): Promise { + const ts = new TransformStream(); + const writer = ts.writable.getWriter(); + const pending = writer.write("q").catch((e: unknown) => console.log("write rejected by cancel:", describe(e))); + await ts.readable.cancel("nope").then(() => console.log("cancel resolved")); + await pending; + try { + await writer.closed; + } catch (e) { + console.log("writer.closed after cancel:", describe(e)); + } +} + +async function intoResponse(): Promise { + const ts = new TransformStream(); + const writer = ts.writable.getWriter(); + void (async (): Promise => { + for (const s of ["x", "y", "z"]) { + await writer.write(enc.encode(s)); + console.log("wrote", s); + } + await writer.close(); + console.log("writer closed"); + })(); + console.log("text", await new Response(ts.readable).text()); +} + +/** Hono's StreamingApi: a writer on the writable side and a pull-based + * ReadableStream over a reader of the readable side. */ +interface StreamingApi { + readonly responseReadable: ReadableStream; + write(input: string): Promise; + close(): Promise; + state(): string; +} + +function streamingApi(writable: WritableStream, readable: ReadableStream): StreamingApi { + const writer = writable.getWriter(); + const reader = readable.getReader(); + let closed = false; + let aborted = false; + const responseReadable = new ReadableStream({ + async pull(controller): Promise { + const { done, value } = await reader.read(); + if (done) controller.close(); + else controller.enqueue(value); + }, + cancel: (): void => { + if (!closed) aborted = true; + }, + }); + return { + responseReadable, + async write(input: string): Promise { + try { + await writer.write(enc.encode(input)); + } catch { + // a closed or aborted stream drops the write + } + }, + async close(): Promise { + closed = true; + try { + await writer.close(); + } catch { + // already closed + } + }, + state: (): string => `closed=${String(closed)} aborted=${String(aborted)}`, + }; +} + +async function streamedResponse(): Promise { + const { readable, writable } = new TransformStream(); + const api = streamingApi(writable, readable); + void (async (): Promise => { + try { + for (let i = 0; i < 3; i++) { + await api.write(`data: ${i}\n\n`); + } + } finally { + await api.close(); + } + })(); + const response = new Response(api.responseReadable, { headers: { "content-type": "text/event-stream" } }); + const reader = response.body!.getReader(); + let chunks = 0; + let text = ""; + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + chunks++; + text += dec.decode(value); + } + console.log("streamed", chunks, JSON.stringify(text), api.state()); +} + +async function writableMethods(): Promise { + const a = new TransformStream(); + await a.writable.abort("r").then(() => console.log("writable.abort ok")); + const b = new TransformStream(); + const writer = b.writable.getWriter(); + try { + await b.writable.close(); + } catch (e) { + console.log("close locked:", describe(e)); + } + try { + await b.writable.abort(); + } catch (e) { + console.log("abort locked:", describe(e)); + } + writer.releaseLock(); + const reader = b.readable.getReader(); + const closing = b.writable.close().then(() => console.log("writable.close ok")); + const end = await reader.read(); + console.log("readable done", end.done); + await closing; +} + +async function main(): Promise { + await basics(); + await abortWriter(); + await cancelReadable(); + await intoResponse(); + await streamedResponse(); + await writableMethods(); +} + +await main(); diff --git a/tests/harness/corpus-costs/7743-transform-stream-identity.ts.json b/tests/harness/corpus-costs/7743-transform-stream-identity.ts.json new file mode 100644 index 000000000..0c31a05f7 --- /dev/null +++ b/tests/harness/corpus-costs/7743-transform-stream-identity.ts.json @@ -0,0 +1 @@ +{ "plain": 3000, "san": 4000 } From fab4b403e9058092d9dac1b1fb28bd39bba9946d Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Fri, 23 Oct 2026 09:07:41 -0700 Subject: [PATCH 38/53] Give corpus scratch paths a per-process name The differential and LLVM differential suites can run the same corpus program's Node oracle at the same time in one shard. Programs whose scratch directory or file was derived only from argv[1] collided, and one oracle removed the paths under the other. Their names now include the pid, as 1403 and 5567 already do. Printed output never carries the name. Co-Authored-By: Claude Sonnet 5.5 --- tests/corpus/1006-json-fs-config.ts | 2 +- tests/corpus/1354-builtin-bare-specifiers.ts | 2 +- tests/corpus/1357-fsp-roundtrip.ts | 2 +- tests/corpus/2844-node-disposable-resources.ts | 2 +- tests/corpus/5566-fs-dirent-dyn-boundaries/main.mjs | 2 +- tests/corpus/957-builtins-namespace.ts | 2 +- tests/corpus/992-fs-roundtrip.ts | 2 +- tests/corpus/993-fs-readdir.ts | 2 +- tests/corpus/994-fs-errors.ts | 2 +- tests/corpus/996-fs-rc-stress.ts | 2 +- tests/corpus/997-fs-modules/store.ts | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/corpus/1006-json-fs-config.ts b/tests/corpus/1006-json-fs-config.ts index f2b1d48c0..1783202c9 100644 --- a/tests/corpus/1006-json-fs-config.ts +++ b/tests/corpus/1006-json-fs-config.ts @@ -11,7 +11,7 @@ function tail(path: string): string { } return path.slice(i + 1); } -const path = "tmp-1006-" + tail(process.argv[1]) + ".json"; +const path = "tmp-1006-" + tail(process.argv[1]) + "-" + process.pid + ".json"; type Config = { name: string; diff --git a/tests/corpus/1354-builtin-bare-specifiers.ts b/tests/corpus/1354-builtin-bare-specifiers.ts index cd0db6984..826c3e7a0 100644 --- a/tests/corpus/1354-builtin-bare-specifiers.ts +++ b/tests/corpus/1354-builtin-bare-specifiers.ts @@ -15,7 +15,7 @@ function tail(path: string): string { return path.slice(i + 1); } -const scratch = "tmp-1354-" + tail(process.argv[1]); +const scratch = "tmp-1354-" + tail(process.argv[1]) + "-" + process.pid; if (existsSync(scratch)) { if (existsSync(j(scratch, "f.txt"))) { rmSync(j(scratch, "f.txt")); diff --git a/tests/corpus/1357-fsp-roundtrip.ts b/tests/corpus/1357-fsp-roundtrip.ts index 4ccae7f3a..7368da360 100644 --- a/tests/corpus/1357-fsp-roundtrip.ts +++ b/tests/corpus/1357-fsp-roundtrip.ts @@ -15,7 +15,7 @@ function tail(path: string): string { } async function main(): Promise { - const scratch = "tmp-1357-" + tail(process.argv[1]); + const scratch = "tmp-1357-" + tail(process.argv[1]) + "-" + process.pid; if (existsSync(scratch)) { if (existsSync(scratch + "/a.txt")) { rmSync(scratch + "/a.txt"); diff --git a/tests/corpus/2844-node-disposable-resources.ts b/tests/corpus/2844-node-disposable-resources.ts index 042f617ce..ebeda87a2 100644 --- a/tests/corpus/2844-node-disposable-resources.ts +++ b/tests/corpus/2844-node-disposable-resources.ts @@ -14,7 +14,7 @@ async function main(): Promise { console.log(timeout.hasRef(), immediate.hasRef()); } - const path = "/tmp/scriptc-disposable-file.txt"; + const path = `/tmp/scriptc-disposable-file-${process.pid}.txt`; await writeFile(path, "resource"); { await using handle = await open(path, "r"); diff --git a/tests/corpus/5566-fs-dirent-dyn-boundaries/main.mjs b/tests/corpus/5566-fs-dirent-dyn-boundaries/main.mjs index 95bcbecee..77bdf8905 100644 --- a/tests/corpus/5566-fs-dirent-dyn-boundaries/main.mjs +++ b/tests/corpus/5566-fs-dirent-dyn-boundaries/main.mjs @@ -8,7 +8,7 @@ function tail(path) { while (i >= 0 && path.charAt(i) !== "/" && path.charAt(i) !== "\\") i = i - 1; return path.slice(i + 1); } -const dir = `/tmp/scr-dirent-dyn-${tail(process.argv[1] === undefined ? "x" : process.argv[1])}`; +const dir = `/tmp/scr-dirent-dyn-${tail(process.argv[1] === undefined ? "x" : process.argv[1])}-${process.pid}`; if (fs.existsSync(dir)) fs.rmSync(dir, { recursive: true, force: true }); fs.mkdirSync(`${dir}/sub`, { recursive: true }); fs.writeFileSync(`${dir}/a.txt`, "x"); diff --git a/tests/corpus/957-builtins-namespace.ts b/tests/corpus/957-builtins-namespace.ts index 4b27bfe79..f6980cda6 100644 --- a/tests/corpus/957-builtins-namespace.ts +++ b/tests/corpus/957-builtins-namespace.ts @@ -17,7 +17,7 @@ function tail(p: string): string { return p.slice(i + 1); } -const scratch = "tmp-957-" + tail(process.argv[1]); +const scratch = "tmp-957-" + tail(process.argv[1]) + "-" + process.pid; if (fs.existsSync(scratch)) { fs.rmSync(scratch + "/a.txt"); fs.rmdirSync(scratch); diff --git a/tests/corpus/992-fs-roundtrip.ts b/tests/corpus/992-fs-roundtrip.ts index a039c936f..92b527fb1 100644 --- a/tests/corpus/992-fs-roundtrip.ts +++ b/tests/corpus/992-fs-roundtrip.ts @@ -32,7 +32,7 @@ function freshDir(dir: string): void { } mkdirSync(dir); } -const scratch = "tmp-992-" + tail(process.argv[1]); +const scratch = "tmp-992-" + tail(process.argv[1]) + "-" + process.pid; freshDir(scratch); const file = scratch + "/notes.txt"; diff --git a/tests/corpus/993-fs-readdir.ts b/tests/corpus/993-fs-readdir.ts index e948d3878..b0409f2ef 100644 --- a/tests/corpus/993-fs-readdir.ts +++ b/tests/corpus/993-fs-readdir.ts @@ -26,7 +26,7 @@ function freshDir(dir: string): void { } mkdirSync(dir); } -const scratch = "tmp-993-" + tail(process.argv[1]); +const scratch = "tmp-993-" + tail(process.argv[1]) + "-" + process.pid; freshDir(scratch); console.log(readdirSync(scratch).length); // fresh dir is empty diff --git a/tests/corpus/994-fs-errors.ts b/tests/corpus/994-fs-errors.ts index 74a1ed74e..d317c1aba 100644 --- a/tests/corpus/994-fs-errors.ts +++ b/tests/corpus/994-fs-errors.ts @@ -30,7 +30,7 @@ function freshDir(dir: string): void { } mkdirSync(dir); } -const scratch = "tmp-994-" + tail(process.argv[1]); +const scratch = "tmp-994-" + tail(process.argv[1]) + "-" + process.pid; freshDir(scratch); let caught = 0; diff --git a/tests/corpus/996-fs-rc-stress.ts b/tests/corpus/996-fs-rc-stress.ts index d6a8f0ba3..063845309 100644 --- a/tests/corpus/996-fs-rc-stress.ts +++ b/tests/corpus/996-fs-rc-stress.ts @@ -28,7 +28,7 @@ function freshDir(dir: string): void { } mkdirSync(dir); } -const scratch = "tmp-996-" + tail(process.argv[1]); +const scratch = "tmp-996-" + tail(process.argv[1]) + "-" + process.pid; freshDir(scratch); // Contents into an array, back through map/filter/join. diff --git a/tests/corpus/997-fs-modules/store.ts b/tests/corpus/997-fs-modules/store.ts index 65e59df1b..1b2a0f1fe 100644 --- a/tests/corpus/997-fs-modules/store.ts +++ b/tests/corpus/997-fs-modules/store.ts @@ -17,7 +17,7 @@ export function scratchDir(tag: string): string { while (i >= 0 && script.charAt(i) !== "/" && script.charAt(i) !== "\\") { i = i - 1; } - const dir = tag + script.slice(i + 1); + const dir = tag + script.slice(i + 1) + "-" + process.pid; if (existsSync(dir)) { for (const name of readdirSync(dir)) { rmSync(dir + "/" + name); From 6828c23b5051ece0d4fa24e3f8a70e2ba8c4298c Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 11:20:05 -0500 Subject: [PATCH 39/53] Error tee branches when the source errors with no read pending The tee learned a source error only through a pending read's error steps, so a source that errored while both branches held buffered chunks and no read was pending left the branches readable: the next read returned a chunk where Node rejects. The tee now watches its source's error, as the streams spec reacts to the source reader's closed promise, and errors both branches in a microtask (resolving the cancel promise unless both branches were canceled). A read's error steps only end the read, and a clone of an already errored body errors both branches the same way. --- packages/runtime/src/scr_fetch.c | 50 +++++++++++++++++---- tests/corpus/7742-response-clone-tee.ts | 58 +++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 9 deletions(-) diff --git a/packages/runtime/src/scr_fetch.c b/packages/runtime/src/scr_fetch.c index 37de698ac..bd1b1ba08 100644 --- a/packages/runtime/src/scr_fetch.c +++ b/packages/runtime/src/scr_fetch.c @@ -167,6 +167,10 @@ struct SfStream { SfTransfer *response_owner; /* weak; transfer owns the stream */ bool callbacks_tracked; SfStream *callbacks_next; + /* Told when the stream errors (a tee watching its source's closed + * promise); ctx is borrowed and the watcher clears it before it goes. */ + void (*error_watch)(void *ctx, ScrDyn *reason); + void *error_watch_ctx; }; struct SfHeaders { @@ -1291,6 +1295,7 @@ static void sf_stream_error(SfStream *s, ScrDyn *reason) { if (s->reader) { sf_reject_promise_reason(s->reader->closed, reason); } + if (s->error_watch) s->error_watch(s->error_watch_ctx, reason); if (s->request_owner) { sf_transfer_stream_error(s->request_owner, reason); } @@ -1889,6 +1894,10 @@ static SfTee *sf_tee_retain(SfTee *t) { static void sf_tee_release(SfTee *t) { if (!t || --t->rc > 0) return; + if (t->source->error_watch_ctx == t) { + t->source->error_watch = NULL; + t->source->error_watch_ctx = NULL; + } sf_stream_release(t->branch[0]); sf_stream_release(t->branch[1]); sf_reader_release(t->reader); @@ -1951,18 +1960,37 @@ static void sf_tee_sink_close(void *ctx) { sf_tee_release(t); } +/* The source reader's closed promise rejected (spec: upon its rejection + * with r, error both branches, and resolve cancelPromise unless both + * branches were canceled), as a promise reaction: in a microtask. */ +static void sf_tee_source_errored_task(ScrClosure *cb) { + SfTee *t = (SfTee *)scr_box_get_ref(cb->caps[0]); + ScrDyn *reason = (ScrDyn *)scr_box_get_ref(cb->caps[1]); + sf_stream_error(t->branch[0], reason); + sf_stream_error(t->branch[1], reason); + if (!t->canceled[0] || !t->canceled[1]) scr_promise_fulfill_void(t->cancel_promise); + scr_dyn_release(reason); + sf_tee_release(t); +} + +/* The source errored, with or without a read of it pending. */ +static void sf_tee_source_errored(void *ctx, ScrDyn *reason) { + SfTee *t = ctx; + t->source->error_watch = NULL; + t->source->error_watch_ctx = NULL; + ScrClosure *cb = scr_closure_new((void *)&sf_tee_source_errored_task, 2); + cb->caps[0] = sf_tee_box(t); + cb->caps[1] = scr_box_new_obj(&scr_dyn_retain_v, &scr_dyn_release_v, NULL); + scr_box_set_ref(cb->caps[1], scr_dyn_retain(reason)); + scr_queue_microtask(cb); +} + +/* A read request's error steps only end the read: the source's error + * reaches the branches through sf_tee_source_errored. */ static void sf_tee_sink_error(void *ctx, ScrDyn *reason) { + (void)reason; SfTee *t = ctx; t->reading = false; - if (reason) { - /* The spec errors both branches when the source reader's closed - * promise rejects; a native read request learns it here. */ - sf_stream_error(t->branch[0], reason); - sf_stream_error(t->branch[1], reason); - if (!t->canceled[0] || !t->canceled[1]) { - scr_promise_fulfill_void(t->cancel_promise); - } - } sf_tee_release(t); } @@ -2041,6 +2069,8 @@ static void sf_stream_tee(SfStream *s, SfStream *out[2]) { t->rc = 1; t->source = sf_stream_retain(s); t->reader = sf_stream_get_reader(s); + s->error_watch = &sf_tee_source_errored; + s->error_watch_ctx = t; t->cancel_promise = scr_promise_new(); scr_promise_mark_handled(t->cancel_promise); for (int i = 0; i < 2; i++) { @@ -2053,6 +2083,8 @@ static void sf_stream_tee(SfStream *s, SfStream *out[2]) { t->branch[i] = sf_stream_retain(b); out[i] = b; } + /* An errored source's closed promise has already rejected. */ + if (s->error) sf_tee_source_errored(t, s->error); /* The branches start at once; a branch with room then pulls. */ sf_stream_schedule_initial_pull(out[0]); sf_stream_schedule_initial_pull(out[1]); diff --git a/tests/corpus/7742-response-clone-tee.ts b/tests/corpus/7742-response-clone-tee.ts index 9058c4d70..56c5e74e6 100644 --- a/tests/corpus/7742-response-clone-tee.ts +++ b/tests/corpus/7742-response-clone-tee.ts @@ -151,4 +151,62 @@ async function readAllFrom(reader: any): Promise { } } + +// Tee branches error when their source errors: after both buffered a +// chunk and with no source read pending, reads reject once the error's +// reaction ran (a read issued at once still takes the buffered chunk, as in +// Node); a clone of an errored body errors both branches. +async function readFirst(label: string, r: any): Promise { + try { + const chunk = await r.body.getReader().read(); + console.log(label, "read", JSON.stringify(chunk.value ? [...chunk.value] : null), chunk.done); + } catch (error) { + console.log(label, "rejects", (error as Error).message); + } +} + +function erroring(): { res: any; error: (reason: Error) => void } { + let ctrl: ReadableStreamDefaultController | undefined; + const source = new ReadableStream({ + start(c) { + ctrl = c; + c.enqueue(new Uint8Array([1, 2])); + }, + }); + return { res: new Response(source), error: (reason) => ctrl!.error(reason) }; +} + +const tick = (): Promise => new Promise((resolve) => setTimeout(resolve, 10)); + +async function sourceErrors(): Promise { + // Error, then read after the reaction: both branches reject. + { + const { res, error } = erroring(); + const clone = res.clone(); + await tick(); + error(new Error("late")); + await tick(); + await readFirst("later original", res); + await readFirst("later clone", clone); + } + // Error, then read at once: the first read takes the buffered chunk. + { + const { res, error } = erroring(); + const clone = res.clone(); + await tick(); + error(new Error("soon")); + await readFirst("at once original", res); + await readFirst("at once clone", clone); + } + // A body that errored before the clone. + { + const { res, error } = erroring(); + error(new Error("before")); + const clone = res.clone(); + await readFirst("errored original", res); + await readFirst("errored clone", clone); + } +} + await main(); +await sourceErrors(); From a77fe8c21c67372b3a1af4ad69fcad34a578ad59 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Fri, 23 Oct 2026 09:17:32 -0700 Subject: [PATCH 40/53] Decide an untyped property write with one own-property lookup scr_dyn_object_key_set, behind every untyped property write and every member of an untyped object literal, looked the key up as many as five times: a linear scan for an own accessor or read-only entry, a lookup before the prototype walk, one before the non-extensible check, one to know whether the key was created, and one more inside the store. The own property is now looked up once. An own data property is written in place (an own setter still runs; a read-only or getter-only property still refuses), and an absent key walks the prototype chain as before and is then added by scr_dyn_obj_set, which is unchanged. zod 3, z.array(Order).parse over 500 orders: 199.8M -> 194.1M instructions per round (-2.8%). Co-Authored-By: Claude Opus 5.5 --- .../corpus/7793-untyped-key-writes.js.json | 6 ++ packages/runtime/src/scr_json.c | 21 +++-- tests/corpus/7793-untyped-key-writes.js | 83 +++++++++++++++++++ .../7793-untyped-key-writes.js.json | 1 + 4 files changed, 104 insertions(+), 7 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7793-untyped-key-writes.js.json create mode 100644 tests/corpus/7793-untyped-key-writes.js create mode 100644 tests/harness/corpus-costs/7793-untyped-key-writes.js.json diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7793-untyped-key-writes.js.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7793-untyped-key-writes.js.json new file mode 100644 index 000000000..b887e0fdb --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7793-untyped-key-writes.js.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7793-untyped-key-writes.js" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index 14d0a901f..a1ab99a42 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -9369,10 +9369,12 @@ bool scr_dyn_in(const ScrDyn *v, const ScrStr *key) { return false; } +/* One lookup of the own property decides the assignment: an own property + * is written (or runs its setter, or refuses the write); an absent one + * consults the prototype chain and is then added. */ static void scr_dyn_object_key_set(ScrDyn *recv, ScrStr *key, ScrDyn *value, const ScrDyn *receiver) { - for (size_t i = 0; i < recv->v.obj.len; i++) { - ScrDynEntry *entry = &recv->v.obj.entries[i]; - if (entry->key_len != key->len || memcmp(entry->key, key->data, key->len) != 0) continue; + ScrDynEntry *entry = scr_dyn_entry(recv, key); + if (entry) { if (entry->accessor && entry->setter) { ScrDyn *args[] = {value}; ScrDyn *setter = scr_dyn_retain(entry->setter); @@ -9388,8 +9390,14 @@ static void scr_dyn_object_key_set(ScrDyn *recv, ScrStr *key, ScrDyn *value, con scr_throw_error_msg(SCR_ERR_TYPE, msg, sizeof msg - 1); return; } + scr_dyn_region_store(recv, value); + ScrDyn *old = entry->value; + entry->value = scr_dyn_retain(value); + scr_dyn_release(old); + scr_error_sync_cause(recv, key); + return; } - if (!scr_dyn_entry(recv, key)) { + { for (const ScrDyn *current = scr_dyn_proto_next(recv); current;) { ScrDyn *table = current->kind == SCR_DYN_FUNC ? scr_dyn_fn_properties(current) : NULL; const ScrDyn *object = table ? table : current; @@ -9421,7 +9429,7 @@ static void scr_dyn_object_key_set(ScrDyn *recv, ScrStr *key, ScrDyn *value, con break; } } - if (recv->non_extensible && !scr_dyn_entry(recv, key)) { + if (recv->non_extensible) { ScrJsonBuf buffer; scr_jb_init(&buffer); scr_jb_puts(&buffer, "Cannot add property "); @@ -9430,9 +9438,8 @@ static void scr_dyn_object_key_set(ScrDyn *recv, ScrStr *key, ScrDyn *value, con scr_throw_error(SCR_ERR_TYPE, scr_jb_finish(&buffer)); return; } - bool created = scr_dyn_entry(recv, key) == NULL; scr_dyn_obj_set(recv, key->data, key->len, scr_dyn_retain(value)); - if (created && key->len == 5 && memcmp(key->data, "stack", 5) == 0 && + if (key->len == 5 && memcmp(key->data, "stack", 5) == 0 && scr_errdyn_find(recv, false) != SIZE_MAX) { /* An error's own `stack` exists from construction as a non-enumerable * data property, so assigning it adds no enumerable key. */ diff --git a/tests/corpus/7793-untyped-key-writes.js b/tests/corpus/7793-untyped-key-writes.js new file mode 100644 index 000000000..93158145e --- /dev/null +++ b/tests/corpus/7793-untyped-key-writes.js @@ -0,0 +1,83 @@ +"use strict"; +// Untyped property writes through one site: new keys append in order, +// existing keys keep their position, own and inherited setters run, +// getter-only and read-only properties refuse the write, and +// non-extensible objects refuse new keys, as in Node. + +/** @param {any} target @param {string} key @param {unknown} value */ +function put(target, key, value) { + target[key] = value; +} + +/** @param {() => void} fn */ +function attempt(label, fn) { + try { + fn(); + console.log(label, "ok"); + } catch (e) { + console.log(label, e instanceof TypeError ? "TypeError" : String(e)); + } +} + +const order = {}; +for (const key of ["b", "a", "c", "a", "b", "d"]) put(order, key, key.charCodeAt(0)); +console.log(JSON.stringify(order), Object.keys(order).join("")); + +const log = []; +const withSetter = { + _v: 0, + get v() { + return this._v; + }, + set v(value) { + log.push(`own ${value}`); + this._v = value * 2; + }, + get onlyGetter() { + return "g"; + }, +}; +put(withSetter, "v", 4); +console.log(withSetter.v, JSON.stringify(log)); +attempt("getter-only", () => put(withSetter, "onlyGetter", 1)); + +const proto = { + /** @param {number} value */ + set inherited(value) { + log.push(`inherited ${value} on ${this === child}`); + }, +}; +Object.defineProperty(proto, "fixed", { value: 1, writable: false }); +const child = Object.create(proto); +put(child, "inherited", 7); +attempt("inherited read-only", () => put(child, "fixed", 2)); +put(child, "own", 3); +console.log(JSON.stringify(log), Object.keys(child).join(","), child.fixed); + +/** @type {any} */ +const sealed = { x: 1 }; +Object.seal(sealed); +put(sealed, "x", 2); +attempt("sealed add", () => put(sealed, "y", 3)); +/** @type {any} */ +const closed = { x: 1 }; +Object.preventExtensions(closed); +put(closed, "x", 5); +attempt("closed add", () => put(closed, "z", 1)); +const frozen = Object.freeze({ x: 1 }); +attempt("frozen write", () => put(frozen, "x", 9)); +console.log(JSON.stringify(sealed), JSON.stringify(closed), JSON.stringify(frozen)); + +const readOnly = {}; +Object.defineProperty(readOnly, "k", { value: "kept", writable: false, enumerable: true }); +attempt("own read-only", () => put(readOnly, "k", "changed")); +console.log(readOnly.k); + +const error = new Error("boom"); +put(error, "cause", "why"); +put(error, "code", "E_X"); +console.log(error.cause, error.code); + +const wide = {}; +for (let i = 0; i < 40; i++) put(wide, `k${i % 25}`, i); +console.log(Object.keys(wide).length, wide.k0, wide.k24, Object.keys(wide).slice(0, 3).join(",")); diff --git a/tests/harness/corpus-costs/7793-untyped-key-writes.js.json b/tests/harness/corpus-costs/7793-untyped-key-writes.js.json new file mode 100644 index 000000000..eb1b1f127 --- /dev/null +++ b/tests/harness/corpus-costs/7793-untyped-key-writes.js.json @@ -0,0 +1 @@ +{ "plain": 4300, "san": 4800 } From b92fb630435e4850e719729eda255164eb4d185c Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 11:49:45 -0500 Subject: [PATCH 41/53] Mark typed-array header loads invariant A typed array's length, data pointer and shared backing are fixed when the view is constructed. Each element access loads the data pointer beside the length, ahead of its bounds branch. LLVM then sinks the data pointer load into the in-bounds block, and LICM refuses to hoist a load that runs only on some iterations. A loop whose body branches or stores an element therefore reloaded the data pointer for every access. Header loads now carry `!invariant.load` next to their alias tag, so the data pointer and length leave such loops. Debug builds emit neither. --- packages/compiler/src/backend/llvm/bytes-alias.ts | 10 ++++++++-- .../src/backend/llvm/bytes-loop-versions.test.ts | 5 +++++ packages/compiler/src/backend/llvm/emit-statements.ts | 6 ++---- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/packages/compiler/src/backend/llvm/bytes-alias.ts b/packages/compiler/src/backend/llvm/bytes-alias.ts index 9aaeaa6c7..d43620fd9 100644 --- a/packages/compiler/src/backend/llvm/bytes-alias.ts +++ b/packages/compiler/src/backend/llvm/bytes-alias.ts @@ -7,9 +7,15 @@ import type { LlvmEmitterContext } from "./expr-context.js"; * buffer. Neither overlaps the other, a native field, array storage or a * module global's slot, so element stores leave header loads and global * receiver slots loop-invariant. Untagged accesses and calls still alias - * every category. */ + * every category. + * + * Header loads are also `!invariant.load`: no store or call between the + * construction of a view and its release changes them. Without it, a data + * pointer load ahead of a bounds branch is sunk into the in-bounds block, + * and LICM keeps a conditionally executed load inside the loop. */ export function bytesHeaderAlias(host: LlvmEmitterContext): string { - return host.memoryAliasAttachment("bytes:header"); + const tag = host.memoryAliasAttachment("bytes:header"); + return tag === "" ? "" : `${tag}, !invariant.load !{}`; } export function bytesElementAlias(host: LlvmEmitterContext): string { diff --git a/packages/compiler/src/backend/llvm/bytes-loop-versions.test.ts b/packages/compiler/src/backend/llvm/bytes-loop-versions.test.ts index cee91c66d..34b75861c 100644 --- a/packages/compiler/src/backend/llvm/bytes-loop-versions.test.ts +++ b/packages/compiler/src/backend/llvm/bytes-loop-versions.test.ts @@ -249,6 +249,11 @@ test("emitted loops version the body and tag typed-array memory", () => { const elements = /store double [^\n]*!tbaa !(\d+)/.exec(body)![1]; const header = /load ptr, ptr [^\n]*!tbaa !(\d+)/.exec(body)![1]; expect(elements).not.toBe(header); + // Header loads are invariant for the view's lifetime; element accesses + // are not. + expect(body).toMatch(/load ptr, ptr [^\n]*, !tbaa !\d+, !invariant\.load !\{\}$/m); + expect(body).toMatch(/load i64, ptr [^\n]*, !tbaa !\d+, !invariant\.load !\{\}$/m); + expect(body).not.toMatch(/(load double|store double)[^\n]*invariant\.load/); }); test("loops to a typed array's length version the other typed arrays they index", () => { diff --git a/packages/compiler/src/backend/llvm/emit-statements.ts b/packages/compiler/src/backend/llvm/emit-statements.ts index 48f7a1d5f..aef5ccf30 100644 --- a/packages/compiler/src/backend/llvm/emit-statements.ts +++ b/packages/compiler/src/backend/llvm/emit-statements.ts @@ -64,6 +64,7 @@ import { emitPairedLoopSafety, } from "./counted-loops.js"; import { emitBytesLoopVersionGuard, type BytesLoopVersion } from "./bytes-loop-versions.js"; +import { bytesHeaderAlias } from "./bytes-alias.js"; import { mangleLocal } from "../mangle.js"; import { LlvmUnsupportedError } from "./unsupported.js"; import type { LlValue } from "./expr-context.js"; @@ -1229,10 +1230,7 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { } B.line(`${inBounds} = icmp ult i32 ${narrow}, ${len}`); } else { - emitter.markMemoryPointer(lenPtr, "bytes:header"); - B.line( - `${len} = load ${emitter.sizeType}, ptr ${lenPtr}${emitter.fieldAliasAttachment(lenPtr)}`, - ); + B.line(`${len} = load ${emitter.sizeType}, ptr ${lenPtr}${bytesHeaderAlias(emitter)}`); B.line(`${inBounds} = icmp ult ${emitter.sizeType} ${index}, ${len}`); } B.condBr(inBounds, lb, exit); From d8816b47316dbfb0baefe5a0282f35c1a3823941 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 12:05:34 -0500 Subject: [PATCH 42/53] Evaluate an element assignment's receiver and index before its value An element assignment whose value is an optional element read or call, such as `out[n++] = src[n]!`, held the value in a temporary and then evaluated the receiver and index inside the store, after the value. So `n++` ran after `src[n]` was read, and `src[0]` was stored where Node stores `src[1]`. The receiver and index are now evaluated first whenever the value could change what they read: when it assigns a binding they read, or runs code that can assign their `let` and `var` bindings or fields. A literal, or a binding the value cannot change, is still read at the store. --- .../src/frontend/lowering/array-values.ts | 95 ++++++++++++++++++- .../7642-element-assignment-order.ts.json | 6 ++ tests/corpus/7642-element-assignment-order.ts | 37 ++++++++ .../7642-element-assignment-order.ts.json | 1 + 4 files changed, 137 insertions(+), 2 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7642-element-assignment-order.ts.json create mode 100644 tests/corpus/7642-element-assignment-order.ts create mode 100644 tests/harness/corpus-costs/7642-element-assignment-order.ts.json diff --git a/packages/compiler/src/frontend/lowering/array-values.ts b/packages/compiler/src/frontend/lowering/array-values.ts index 314cdfd24..9f23a860b 100644 --- a/packages/compiler/src/frontend/lowering/array-values.ts +++ b/packages/compiler/src/frontend/lowering/array-values.ts @@ -15,6 +15,7 @@ import { typeKey, unionContainerArmsOk, } from "../../ir/ir.js"; +import { everyExpr } from "../../ir/traverse.js"; import type { Lowerer } from "./lowerer.js"; import type * as ts from "../ts7/adapter.js"; @@ -193,6 +194,81 @@ export function unionArrayValueRead( return result; } +/** Expression kinds that evaluate without running other code or writing + * a binding. */ +const PURE_KINDS = new Set([ + "varRef", + "numLit", + "strLit", + "boolLit", + "unitLit", + "bin", + "unary", + "logical", + "ternary", + "toBool", + "fieldGet", + "recordGet", + "arrayGet", + "arrayState", + "arrayHas", + "unionIsTag", + "unionNarrow", + "unionWrap", +]); + +/** The locals evaluating `value` may assign, and whether it may run other + * code (anything but the pure kinds and the read-only element helpers), + * which can assign locals and fields through closures. */ +function valueEffects(lowerer: Lowerer, value: IrExpr): { writes: Set; opaque: boolean } { + const readers = new Set(); + for (const [key, name] of lowerer.arrHofHelpers) if (key.startsWith("idxOr:")) readers.add(name); + const writes = new Set(); + let opaque = false; + everyExpr(value, { + expr: (e) => { + if (e.kind === "assignExpr" || e.kind === "incDec") writes.add(e.localId); + else if (e.kind === "call" ? !readers.has(e.callee) : !PURE_KINDS.has(e.kind)) opaque = true; + return true; + }, + stmt: (s) => { + if (s.kind === "varDecl" || s.kind === "assign") writes.add(s.localId); + else opaque = true; + return true; + }, + }); + return { writes, opaque }; +} + +/** Whether evaluating `e` after `value` gives what it gives before it: a + * literal, or a local and field read `value` cannot change. */ +function unaffectedBy(e: IrExpr, effects: { writes: Set; opaque: boolean }): boolean { + switch (e.kind) { + case "numLit": + case "strLit": + return true; + case "varRef": + return !effects.writes.has(e.localId); + case "bin": + return unaffectedBy(e.left, effects) && unaffectedBy(e.right, effects); + case "fieldGet": + case "recordGet": + return !effects.opaque && unaffectedBy(e.obj, effects); + default: + return false; + } +} + +/** Whether `e` reads a `let` or `var` binding, which code the value runs + * can assign through a closure. */ +function readsMutableLocal(lowerer: Lowerer, e: IrExpr): boolean { + return !everyExpr(e, { + expr: (x) => + x.kind !== "varRef" || lowerer.ctx.locals.find((l) => l.id === x.localId)?.mutable === false, + stmt: () => true, + }); +} + /** Store a stabilized value without changing the array's element ABI. */ export function arrayValueStore( lowerer: Lowerer, @@ -210,10 +286,25 @@ export function arrayValueStore( // owned union temporary to release on every path. const unionId = value.type.unionId; let stable = value; - let prefix: IrStmt[] = []; + const prefix: IrStmt[] = []; + // The stores below evaluate the receiver and the index after the value. + // JavaScript evaluates them first (`out[n++] = src[n]` reads src[1]), so + // either may need its value held before the value runs. + if (value.kind !== "varRef") { + const effects = valueEffects(lowerer, value); + const hold = (e: IrExpr, name: string): IrExpr => { + if (unaffectedBy(e, effects) && !(effects.opaque && readsMutableLocal(lowerer, e))) + return e; + const temp = lowerer.declareHiddenLocal(name, e.type); + prefix.push({ kind: "varDecl", localId: temp.id, init: e, loc }); + return varRef(temp.id, temp.type, loc); + }; + arr = hold(arr, "%arrayTarget"); + index = hold(index, "%arrayIndex"); + } if (value.kind !== "varRef") { const temp = lowerer.declareHiddenLocal("%arrayValue", value.type); - prefix = [{ kind: "varDecl", localId: temp.id, init: value, loc }]; + prefix.push({ kind: "varDecl", localId: temp.id, init: value, loc }); stable = varRef(temp.id, temp.type, loc); } const write: IrStmt = { diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7642-element-assignment-order.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7642-element-assignment-order.ts.json new file mode 100644 index 000000000..17369f496 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7642-element-assignment-order.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7642-element-assignment-order.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7642-element-assignment-order.ts b/tests/corpus/7642-element-assignment-order.ts new file mode 100644 index 000000000..f2f2fae7b --- /dev/null +++ b/tests/corpus/7642-element-assignment-order.ts @@ -0,0 +1,37 @@ +// An element assignment evaluates its receiver and index before its value, +// also when the value is an optional element read or a call that assigns +// the bindings the receiver and index read. +const a = [10, 20, 30, 40]; +const b = [1, 2, 3, 4]; +let n = 0; +a[n++] = b[n]!; +console.log(a.join(), n); +a[n] = b[n++]; +console.log(a.join(), n); + +const optional: (number | undefined)[] = [undefined, 5, undefined]; +let k = 0; +optional[k++] = optional[k]; +console.log(optional, k); + +// A call in the value assigns the receiver and the index through closures. +let target: (number | undefined)[] = [0, 0, 0]; +const first = target; +let at = 0; +function swap(): number | undefined { + target = [9, 9, 9]; + at = 2; + return b[3]; +} +target[at] = swap(); +console.log(first, target, at); + +// The index and receiver are not evaluated twice. +let reads = 0; +function index(): number { + reads++; + return 1; +} +const c: (number | undefined)[] = [0, 0]; +c[index()] = optional[1]; +console.log(c, reads); diff --git a/tests/harness/corpus-costs/7642-element-assignment-order.ts.json b/tests/harness/corpus-costs/7642-element-assignment-order.ts.json new file mode 100644 index 000000000..9f67659a1 --- /dev/null +++ b/tests/harness/corpus-costs/7642-element-assignment-order.ts.json @@ -0,0 +1 @@ +{ "plain": 1500, "san": 2500 } From 849fad0ab6051f3914ba269ad068d07a80011213 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 12:08:08 -0500 Subject: [PATCH 43/53] Prepare v0.4.3 release --- CHANGELOG.md | 24 +++++++++++++++++-- packages/cli-darwin-arm64/package.json | 2 +- packages/cli-darwin-x64/package.json | 2 +- packages/cli-linux-arm64-gnu/package.json | 2 +- packages/cli-linux-arm64-musl/package.json | 2 +- packages/cli-linux-x64-gnu/package.json | 2 +- packages/cli-linux-x64-musl/package.json | 2 +- packages/cli-win32-x64-msvc/package.json | 2 +- packages/cli/package.json | 2 +- packages/compiler/package.json | 2 +- packages/compiler/surface-manifest.json | 2 +- packages/experimental/package.json | 2 +- packages/llvm-darwin-arm64/package.json | 2 +- packages/llvm-darwin-x64/package.json | 2 +- packages/llvm-linux-arm64-gnu/package.json | 2 +- packages/llvm-linux-arm64-musl/package.json | 2 +- packages/llvm-linux-x64-gnu/package.json | 2 +- packages/llvm-linux-x64-musl/package.json | 2 +- packages/llvm-win32-x64-msvc/package.json | 2 +- packages/runtime-android-arm64/package.json | 2 +- packages/runtime-darwin-arm64/package.json | 2 +- packages/runtime-darwin-x64/package.json | 2 +- packages/runtime-ios-arm64/package.json | 2 +- .../runtime-ios-simulator-arm64/package.json | 2 +- packages/runtime-linux-arm64-gnu/package.json | 2 +- .../runtime-linux-arm64-musl/package.json | 2 +- packages/runtime-linux-x64-gnu/package.json | 2 +- packages/runtime-linux-x64-musl/package.json | 2 +- packages/runtime-wasm32-wasi/package.json | 2 +- packages/runtime-win32-x64-msvc/package.json | 2 +- packages/runtime/package.json | 2 +- packages/threads/package.json | 2 +- packages/types/package.json | 2 +- 33 files changed, 54 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ec0b27dbf..96f13fa26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,28 @@ All notable changes to scriptc will be documented in this file. +## 0.4.3 + +### Highlights + +- **Faster checking and smaller builds with tsc-ts 0.0.9:** scriptc checks with tsc-ts 0.0.9, and builds reuse the bundled declaration files' diagnostics instead of checking them again. Small IR builds are 10.6 to 16.0% faster and small executable builds 5.5 to 7.6% faster, with identical output. A native build of tsc-ts with this release checks 0.4 to 6.6% faster than with 0.4.2 and peaks 4 to 16% lower. +- **Integer arrays and loops:** + - Number arrays that only hold int32 values use 4-byte slots in locals, across direct calls and in module globals, and loop-carried locals written from their elements stay int32. A workload that walks arrays of cursors runs 54.5% faster. + - Loops with two counters closing in on each other (`for (let i = lo, j = hi; i < j; i++, j--)`) and element copies through `a[i]!` run without per-element checks, and two-counter `while` loops run on integer counters. + - Loops over typed arrays and charAt scans are versioned on their bounds, and ASCII strings are scanned as bytes. +- **Less work for untyped and dynamic code:** + - String-keyed record reads under `??` and `!` no longer allocate: a dictionary-walking workload makes no union boxes instead of 363,359 and runs 12 to 15% faster. + - Checked-dynamic method calls dispatch on method ids, observed class methods call natively while their slot is unchanged, and calls on bag-tracked class instances skip building their views. +- **Node.js parity:** + - `crypto.getCipherInfo`, AES-CCM, AES key wrap, the legacy `createCipheriv` modes, `checkPrime`/`generatePrime`, and Node's argument checks for `generateKeyPair`. + - Streams: `construct`, `writev`, `decodeStrings` and `defaultEncoding`, including `writev` on `Transform` and `PassThrough`. + - `FinalizationRegistry`, `new.target`, `String.prototype.at` on untyped values, `Error.stackTraceLimit`, `spawnSync`'s `env` option, and class constructors called through `call`/`apply` throw Node's TypeError. + - Records enumerate in the key order their object literal wrote, and objects converting to a union with an empty-record arm keep every field. +- **npm packages:** zod 3 compiles natively and zod 4's core runs, minified packages such as marked compile, chalk 5 and hot-shots 8 build from built-in shims, and `@hono/node-server` 2.x is built in. User code types npm package declarations at a checked, identity-preserving boundary. +- **Builds:** the cached build is reused after whitespace and comment edits, function bodies are emitted on forked workers, and the LLVM emitter reads its analyses from one recorded walk of each module. + + + ## 0.4.2 ### Highlights @@ -159,8 +181,6 @@ All notable changes to scriptc will be documented in this file. - **The same program compiles to the same executable wherever it sits on disk** and whatever order its functions lower in, and edits outside a declaration do not rename the helpers and closures inside it. - **Libraries that use untyped byte arrays** link. - - ## 0.4.1 ### Highlights diff --git a/packages/cli-darwin-arm64/package.json b/packages/cli-darwin-arm64/package.json index 23584b60a..eee0ec3c8 100644 --- a/packages/cli-darwin-arm64/package.json +++ b/packages/cli-darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/cli-darwin-arm64", - "version": "0.4.2", + "version": "0.4.3", "description": "Native scriptc compiler for darwin-arm64", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/cli-darwin-x64/package.json b/packages/cli-darwin-x64/package.json index d9e6dafc6..b527a3cd7 100644 --- a/packages/cli-darwin-x64/package.json +++ b/packages/cli-darwin-x64/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/cli-darwin-x64", - "version": "0.4.2", + "version": "0.4.3", "description": "Native scriptc compiler for darwin-x64", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/cli-linux-arm64-gnu/package.json b/packages/cli-linux-arm64-gnu/package.json index 5b8cb4db2..512a7cd13 100644 --- a/packages/cli-linux-arm64-gnu/package.json +++ b/packages/cli-linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/cli-linux-arm64-gnu", - "version": "0.4.2", + "version": "0.4.3", "description": "Native scriptc compiler for linux-arm64-gnu", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/cli-linux-arm64-musl/package.json b/packages/cli-linux-arm64-musl/package.json index 0901602ec..8d0fdfb0f 100644 --- a/packages/cli-linux-arm64-musl/package.json +++ b/packages/cli-linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/cli-linux-arm64-musl", - "version": "0.4.2", + "version": "0.4.3", "description": "Native scriptc compiler for linux-arm64-musl", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/cli-linux-x64-gnu/package.json b/packages/cli-linux-x64-gnu/package.json index f01478aad..b71205614 100644 --- a/packages/cli-linux-x64-gnu/package.json +++ b/packages/cli-linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/cli-linux-x64-gnu", - "version": "0.4.2", + "version": "0.4.3", "description": "Native scriptc compiler for linux-x64-gnu", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/cli-linux-x64-musl/package.json b/packages/cli-linux-x64-musl/package.json index a6986a66a..eaeb6f15a 100644 --- a/packages/cli-linux-x64-musl/package.json +++ b/packages/cli-linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/cli-linux-x64-musl", - "version": "0.4.2", + "version": "0.4.3", "description": "Native scriptc compiler for linux-x64-musl", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/cli-win32-x64-msvc/package.json b/packages/cli-win32-x64-msvc/package.json index 6b03811b8..3e3b8259f 100644 --- a/packages/cli-win32-x64-msvc/package.json +++ b/packages/cli-win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/cli-win32-x64-msvc", - "version": "0.4.2", + "version": "0.4.3", "description": "Native scriptc compiler for win32-x64-msvc", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/cli/package.json b/packages/cli/package.json index 034747c29..264a7f5f1 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "scriptc", - "version": "0.4.2", + "version": "0.4.3", "description": "Compile ordinary TypeScript and JavaScript to small, fast native executables and WebAssembly modules that run without Node.js", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/compiler/package.json b/packages/compiler/package.json index a9e3d9616..60f6bd0f0 100644 --- a/packages/compiler/package.json +++ b/packages/compiler/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/compiler", - "version": "0.4.2", + "version": "0.4.3", "description": "The scriptc compiler — TypeScript/JavaScript frontend, typed IR, LLVM backend", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/compiler/surface-manifest.json b/packages/compiler/surface-manifest.json index e44f5d31e..3387217b5 100644 --- a/packages/compiler/surface-manifest.json +++ b/packages/compiler/surface-manifest.json @@ -1,6 +1,6 @@ { "schemaVersion": 2, - "compilerVersion": "0.4.2", + "compilerVersion": "0.4.3", "coverage": [ "Entries are projected mechanically from the compiler's own decision tables: the diagnostics registry, the unsupported-syntax dispatch tables, the stdlib and node-builtin lowering tables, and the supported-builtin-module list. Nothing is hand-maintained; the manifest regenerates byte-identically from the source tree at this version.", "Absence from this manifest means 'not projected', never 'unsupported'. Surfaces lowered through dedicated code paths rather than tables are not yet projected: console, JSON, Promise/async and the timer surface, the net/http/tls/https/dgram/dns/assert/test/stream/readline module member surfaces, template literals, the regex slice, global functions (parseInt, parseFloat, isNaN, isFinite), and the process surface outside its ambient slice.", diff --git a/packages/experimental/package.json b/packages/experimental/package.json index cfb4151a4..b3e8a6fb1 100644 --- a/packages/experimental/package.json +++ b/packages/experimental/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/experimental", - "version": "0.4.2", + "version": "0.4.3", "description": "Experimental scriptc APIs, with Node.js implementations: lifetime regions for ProgramLifetime objects, intern tables shared between threads, and per-thread slots on shared objects", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/llvm-darwin-arm64/package.json b/packages/llvm-darwin-arm64/package.json index 615fe9d81..ac99f1661 100644 --- a/packages/llvm-darwin-arm64/package.json +++ b/packages/llvm-darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/llvm-darwin-arm64", - "version": "0.4.2", + "version": "0.4.3", "description": "Pinned LLVM code-generation helper for scriptc on macOS arm64", "license": "Apache-2.0 WITH LLVM-exception", "homepage": "https://scriptc.dev", diff --git a/packages/llvm-darwin-x64/package.json b/packages/llvm-darwin-x64/package.json index fd6caf976..ca632dcd4 100644 --- a/packages/llvm-darwin-x64/package.json +++ b/packages/llvm-darwin-x64/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/llvm-darwin-x64", - "version": "0.4.2", + "version": "0.4.3", "description": "Pinned LLVM code-generation helper for scriptc on macOS x64", "license": "Apache-2.0 WITH LLVM-exception", "homepage": "https://scriptc.dev", diff --git a/packages/llvm-linux-arm64-gnu/package.json b/packages/llvm-linux-arm64-gnu/package.json index 7d66257f4..566b38e40 100644 --- a/packages/llvm-linux-arm64-gnu/package.json +++ b/packages/llvm-linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/llvm-linux-arm64-gnu", - "version": "0.4.2", + "version": "0.4.3", "description": "Pinned LLVM code-generation helper for scriptc on Linux arm64", "license": "Apache-2.0 WITH LLVM-exception", "homepage": "https://scriptc.dev", diff --git a/packages/llvm-linux-arm64-musl/package.json b/packages/llvm-linux-arm64-musl/package.json index b3a01089d..a9288c2ca 100644 --- a/packages/llvm-linux-arm64-musl/package.json +++ b/packages/llvm-linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/llvm-linux-arm64-musl", - "version": "0.4.2", + "version": "0.4.3", "description": "Statically linked LLVM code-generation helper for scriptc on Linux arm64 musl", "license": "Apache-2.0 WITH LLVM-exception", "homepage": "https://scriptc.dev", diff --git a/packages/llvm-linux-x64-gnu/package.json b/packages/llvm-linux-x64-gnu/package.json index 0b3264ec7..c55f51847 100644 --- a/packages/llvm-linux-x64-gnu/package.json +++ b/packages/llvm-linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/llvm-linux-x64-gnu", - "version": "0.4.2", + "version": "0.4.3", "description": "Pinned LLVM code-generation helper for scriptc on Linux x64", "license": "Apache-2.0 WITH LLVM-exception", "homepage": "https://scriptc.dev", diff --git a/packages/llvm-linux-x64-musl/package.json b/packages/llvm-linux-x64-musl/package.json index 6c83d1cb9..b257085bf 100644 --- a/packages/llvm-linux-x64-musl/package.json +++ b/packages/llvm-linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/llvm-linux-x64-musl", - "version": "0.4.2", + "version": "0.4.3", "description": "Statically linked LLVM code-generation helper for scriptc on Linux x64 musl", "license": "Apache-2.0 WITH LLVM-exception", "homepage": "https://scriptc.dev", diff --git a/packages/llvm-win32-x64-msvc/package.json b/packages/llvm-win32-x64-msvc/package.json index 9fb682f0f..0f3f99331 100644 --- a/packages/llvm-win32-x64-msvc/package.json +++ b/packages/llvm-win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/llvm-win32-x64-msvc", - "version": "0.4.2", + "version": "0.4.3", "description": "Pinned LLVM code-generation helper for scriptc on Windows x64", "license": "Apache-2.0 WITH LLVM-exception", "homepage": "https://scriptc.dev", diff --git a/packages/runtime-android-arm64/package.json b/packages/runtime-android-arm64/package.json index 362fbc572..392ef1717 100644 --- a/packages/runtime-android-arm64/package.json +++ b/packages/runtime-android-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/runtime-android-arm64", - "version": "0.4.2", + "version": "0.4.3", "description": "Precompiled scriptc library runtime pack for Android arm64", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/runtime-darwin-arm64/package.json b/packages/runtime-darwin-arm64/package.json index a4176ed60..354526024 100644 --- a/packages/runtime-darwin-arm64/package.json +++ b/packages/runtime-darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/runtime-darwin-arm64", - "version": "0.4.2", + "version": "0.4.3", "description": "Precompiled scriptc runtime pack for macOS arm64", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/runtime-darwin-x64/package.json b/packages/runtime-darwin-x64/package.json index 91bc4182b..58aa358b2 100644 --- a/packages/runtime-darwin-x64/package.json +++ b/packages/runtime-darwin-x64/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/runtime-darwin-x64", - "version": "0.4.2", + "version": "0.4.3", "description": "Precompiled scriptc runtime pack for macOS x64", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/runtime-ios-arm64/package.json b/packages/runtime-ios-arm64/package.json index 2fdd68897..6193c5258 100644 --- a/packages/runtime-ios-arm64/package.json +++ b/packages/runtime-ios-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/runtime-ios-arm64", - "version": "0.4.2", + "version": "0.4.3", "description": "Precompiled scriptc library runtime pack for iOS arm64", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/runtime-ios-simulator-arm64/package.json b/packages/runtime-ios-simulator-arm64/package.json index cbdf96d1d..7d01c45d3 100644 --- a/packages/runtime-ios-simulator-arm64/package.json +++ b/packages/runtime-ios-simulator-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/runtime-ios-simulator-arm64", - "version": "0.4.2", + "version": "0.4.3", "description": "Precompiled scriptc library runtime pack for iOS simulator arm64", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/runtime-linux-arm64-gnu/package.json b/packages/runtime-linux-arm64-gnu/package.json index e0d228925..419080c3a 100644 --- a/packages/runtime-linux-arm64-gnu/package.json +++ b/packages/runtime-linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/runtime-linux-arm64-gnu", - "version": "0.4.2", + "version": "0.4.3", "description": "Precompiled scriptc runtime pack for Linux arm64 glibc", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/runtime-linux-arm64-musl/package.json b/packages/runtime-linux-arm64-musl/package.json index 70eee223f..2bae09884 100644 --- a/packages/runtime-linux-arm64-musl/package.json +++ b/packages/runtime-linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/runtime-linux-arm64-musl", - "version": "0.4.2", + "version": "0.4.3", "description": "Precompiled scriptc runtime pack for Linux arm64 musl", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/runtime-linux-x64-gnu/package.json b/packages/runtime-linux-x64-gnu/package.json index 5822fd21d..6749ddeea 100644 --- a/packages/runtime-linux-x64-gnu/package.json +++ b/packages/runtime-linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/runtime-linux-x64-gnu", - "version": "0.4.2", + "version": "0.4.3", "description": "Precompiled scriptc runtime pack for Linux x64 glibc", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/runtime-linux-x64-musl/package.json b/packages/runtime-linux-x64-musl/package.json index 1db34d4a7..9b4803406 100644 --- a/packages/runtime-linux-x64-musl/package.json +++ b/packages/runtime-linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/runtime-linux-x64-musl", - "version": "0.4.2", + "version": "0.4.3", "description": "Precompiled scriptc runtime pack for Linux x64 musl", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/runtime-wasm32-wasi/package.json b/packages/runtime-wasm32-wasi/package.json index 5bbd23755..63a95601d 100644 --- a/packages/runtime-wasm32-wasi/package.json +++ b/packages/runtime-wasm32-wasi/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/runtime-wasm32-wasi", - "version": "0.4.2", + "version": "0.4.3", "description": "Precompiled scriptc runtime pack for WASI Preview 1", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/runtime-win32-x64-msvc/package.json b/packages/runtime-win32-x64-msvc/package.json index df4a9a109..01bec7c31 100644 --- a/packages/runtime-win32-x64-msvc/package.json +++ b/packages/runtime-win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/runtime-win32-x64-msvc", - "version": "0.4.2", + "version": "0.4.3", "description": "Precompiled scriptc runtime pack for Windows x64 MSVC", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/runtime/package.json b/packages/runtime/package.json index 9232266af..d367ad6bf 100644 --- a/packages/runtime/package.json +++ b/packages/runtime/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/runtime", - "version": "0.4.2", + "version": "0.4.3", "description": "scriptc native runtime sources and vendored dependencies", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/threads/package.json b/packages/threads/package.json index 11b8ef624..a37a3ba82 100644 --- a/packages/threads/package.json +++ b/packages/threads/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/threads", - "version": "0.4.2", + "version": "0.4.3", "description": "Zero-copy publication of immutable object graphs to worker threads (scriptc), with a deep-freezing Node.js implementation", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/types/package.json b/packages/types/package.json index 689c7b72f..906c68dec 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/types", - "version": "0.4.2", + "version": "0.4.3", "description": "Contract types for scriptc: type-only annotations (Int32, NoThrow, Immutable) that scriptc checks and optimizes, with no runtime semantics", "license": "Apache-2.0", "homepage": "https://scriptc.dev", From a36ef0b4ca50392320b92a6d4bb90e04566e359a Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 12:11:48 -0500 Subject: [PATCH 44/53] Settle aborts and pull TransformStream uploads as Node does --- packages/runtime/src/scr_fetch.c | 73 ++++++++++++++----- .../corpus/7743-transform-stream-identity.ts | 33 ++++++++- tests/fixtures/fetch/static-stream/main.mts | 16 ++++ 3 files changed, 102 insertions(+), 20 deletions(-) diff --git a/packages/runtime/src/scr_fetch.c b/packages/runtime/src/scr_fetch.c index e92f488b2..9792f8bed 100644 --- a/packages/runtime/src/scr_fetch.c +++ b/packages/runtime/src/scr_fetch.c @@ -1622,6 +1622,14 @@ static void sf_stream_schedule_pull_complete(SfStream *s) { scr_queue_microtask(cb); } +/* A consumer waiting for the stream's next chunk: a pending read, a body + * collector, or a fetch upload (fetch keeps reading the request body, + * including after an aborted transfer stops writing it). */ +static bool sf_stream_has_consumer(const SfStream *s) { + return s->collector || (s->reader && s->reader->pending_head) || s->request_owner || + s->request_discarding; +} + static void sf_stream_pull(SfStream *s) { if (!s->started || (!s->pull_cb && !sf_stream_has_from_source(s)) || @@ -1630,11 +1638,9 @@ static void sf_stream_pull(SfStream *s) { return; } /* ReadableStreamDefaultControllerShouldCallPull: with no room in the - * queue, only a pending read (or a body collector) asks for a chunk. */ - if (s->hwm_zero && !s->collector && - !(s->reader && s->reader->pending_head)) { - return; - } + * queue, only a consumer asks for a chunk: a pending read, a body + * collector, or a fetch upload, whose reader is always waiting. */ + if (s->hwm_zero && !sf_stream_has_consumer(s)) return; if (s->pulling) { s->pull_again = true; return; @@ -2730,6 +2736,12 @@ struct SfTransform { ScrPromise *in_flight_close; bool has_pending_abort; ScrPromise *pending_abort; + /* The sink abort found a finish promise from a cancel or close, which + * settles the abort instead (Node returns that promise). */ + bool abort_joins_finish; + /* The abort request's outcome between SFT_ABORT_DONE and + * SFT_ABORT_SETTLE: a rejection reason (+1), or NULL to fulfill. */ + ScrDyn *abort_settle_error; ScrDyn *pending_abort_reason; bool pending_abort_was_erroring; bool backpressure; @@ -2779,6 +2791,7 @@ static void sf_transform_release(SfTransform *t) { scr_promise_release(t->in_flight_close); scr_promise_release(t->pending_abort); scr_dyn_release(t->pending_abort_reason); + scr_dyn_release(t->abort_settle_error); scr_promise_release(t->bp_change); scr_promise_release(t->finish); scr_dyn_release(t->write_error); @@ -2849,6 +2862,7 @@ enum { SFT_ABORT_DONE, SFT_START, SFT_CANCEL_DONE, + SFT_ABORT_SETTLE, }; static void sf_transform_step(SfTransform *t, int op); @@ -3214,7 +3228,7 @@ static void sft_set_backpressure(SfTransform *t, bool backpressure) { } static bool sft_readable_wants_chunk(const SfStream *r) { - return r->collector || (r->reader && r->reader->pending_head); + return sf_stream_has_consumer(r); } static bool sft_readable_can_enqueue(const SfStream *r) { @@ -3278,6 +3292,7 @@ static void sft_sink_close(SfTransform *t) { static void sft_abort_algorithm(SfTransform *t, ScrDyn *reason) { (void)reason; + t->abort_joins_finish = t->finish != NULL; if (!t->finish) { t->finish = scr_promise_new(); scr_promise_mark_handled(t->finish); @@ -3327,25 +3342,49 @@ static void sf_transform_step(SfTransform *t, int op) { } case SFT_ABORT_DONE: { SfStream *r = t->readable; - ScrDyn *reason = t->pending_abort_reason - ? scr_dyn_retain(t->pending_abort_reason) - : scr_dyn_retain(scr_dyn_undefined()); - bool readable_errored = r->error != NULL; - if (!readable_errored) sf_stream_error(r, reason); + scr_dyn_release(t->abort_settle_error); + t->abort_settle_error = NULL; + if (t->abort_joins_finish && !scr_promise_is_pending(t->finish)) { + /* The cancel or close that created the finish promise settled + * it; the abort request takes its result. */ + if (scr_promise_is_rejected(t->finish)) + t->abort_settle_error = scr_promise_reason_dyn(t->finish); + } else { + ScrDyn *reason = t->pending_abort_reason + ? scr_dyn_retain(t->pending_abort_reason) + : scr_dyn_retain(scr_dyn_undefined()); + bool readable_errored = r->error != NULL; + if (!readable_errored) sf_stream_error(r, reason); + /* The sink abort algorithm's finish promise, which a + * readable.cancel() after the abort answers too: rejected with the + * readable side's stored error when it had already errored, else + * resolved. */ + if (t->finish && scr_promise_is_pending(t->finish)) { + if (readable_errored) sf_reject_promise_reason(t->finish, r->error); + else scr_promise_fulfill_void(t->finish); + } + if (readable_errored) t->abort_settle_error = scr_dyn_retain(r->error); + scr_dyn_release(reason); + } + /* WritableStreamFinishErroring settles the abort request in a + * reaction to the sink abort's promise, two turns after it. */ + sf_transform_defer(t, SFT_ABORT_SETTLE, 2); + return; + } + case SFT_ABORT_SETTLE: { + ScrDyn *error = t->abort_settle_error; + t->abort_settle_error = NULL; if (t->has_pending_abort) { t->has_pending_abort = false; - if (readable_errored) { - sf_reject_promise_reason(t->pending_abort, r->error); - } else { - scr_promise_fulfill_void(t->pending_abort); - } + if (error) sf_reject_promise_reason(t->pending_abort, error); + else scr_promise_fulfill_void(t->pending_abort); scr_promise_release(t->pending_abort); t->pending_abort = NULL; scr_dyn_release(t->pending_abort_reason); t->pending_abort_reason = NULL; } sfw_reject_close_and_closed(t); - scr_dyn_release(reason); + scr_dyn_release(error); return; } case SFT_CANCEL_DONE: { diff --git a/tests/corpus/7743-transform-stream-identity.ts b/tests/corpus/7743-transform-stream-identity.ts index d98303997..3664c8ff1 100644 --- a/tests/corpus/7743-transform-stream-identity.ts +++ b/tests/corpus/7743-transform-stream-identity.ts @@ -1,8 +1,8 @@ // The identity TransformStream: writes wait for reads (the readable side's // high-water mark is 0), the writer's desiredSize/ready track the writable -// side's queue, close/abort/cancel propagate across the pair, and the -// readable side feeds a Response body (the StreamingApi pattern of Hono's -// stream helpers). +// side's queue, close/abort/cancel propagate across the pair (in either +// order when they race), and the readable side feeds a Response body (the +// StreamingApi pattern of Hono's stream helpers). const enc = new TextEncoder(); const dec = new TextDecoder(); @@ -200,6 +200,32 @@ async function writableMethods(): Promise { await closing; } +// writable.abort() and readable.cancel() share the transform's finish +// promise: whichever runs second answers the first one's result. +function settled(label: string, p: Promise): Promise { + return p.then( + () => console.log(label, "fulfilled"), + (e: unknown) => console.log(label, "rejected", describe(e)), + ); +} + +async function abortCancelRaces(): Promise { + const a = new TransformStream(); + const aWriter = a.writable.getWriter(); + await new Promise((r) => setTimeout(r, 0)); + const aborted = settled("abort then cancel: abort", aWriter.abort("abort first")); + const cancelled = settled("abort then cancel: cancel", a.readable.cancel("cancel second")); + await aborted; + await cancelled; + const b = new TransformStream(); + const bWriter = b.writable.getWriter(); + await new Promise((r) => setTimeout(r, 0)); + const cancelledFirst = settled("cancel then abort: cancel", b.readable.cancel("cancel first")); + const abortedSecond = settled("cancel then abort: abort", bWriter.abort("abort second")); + await cancelledFirst; + await abortedSecond; +} + async function main(): Promise { await basics(); await abortWriter(); @@ -207,6 +233,7 @@ async function main(): Promise { await intoResponse(); await streamedResponse(); await writableMethods(); + await abortCancelRaces(); } await main(); diff --git a/tests/fixtures/fetch/static-stream/main.mts b/tests/fixtures/fetch/static-stream/main.mts index 9ff8c050b..0ae400bc3 100644 --- a/tests/fixtures/fetch/static-stream/main.mts +++ b/tests/fixtures/fetch/static-stream/main.mts @@ -259,6 +259,22 @@ try { console.log("consumed request reader:", (error as Error).name); } +// A TransformStream's readable side as the request body: fetch's upload +// reads it, so writes to the writable side do not wait on backpressure. +const uploadTransform = new TransformStream(); +const uploadWriter = uploadTransform.writable.getWriter(); +const transformPosted = fetch(`${process.argv[2]}/post-echo`, { + method: "POST", + body: uploadTransform.readable, + duplex: "half", +}); +for (const part of ["trans", "form ", "body"]) { + await uploadWriter.write(new TextEncoder().encode(part)); +} +await uploadWriter.close(); +const transformResult = (await (await transformPosted).json()) as { body: string }; +console.log("transform request body:", transformResult.body); + const prelockedRequestBody = ReadableStream.from([ Buffer.from("prelocked request"), ]); From efa797382ef4120f064e55f170788e096299cbce Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 12:20:28 -0500 Subject: [PATCH 45/53] Render the compatibility matrix's tooltips on demand The matrix mounted every badge's tooltip up front, hidden by CSS, and re-rendered every row on each keystroke. Tooltips now mount while their badge is hovered or has keyboard focus, are named by aria-describedby and close on Escape. Badges and rows are memoized, each row's search text is built once, the filter makes one pass for its rows and its count, and the table follows deferred search and status values, with expand all and collapse all as transitions, so input stays responsive. The tree and filter passes move to src/lib/compatibility-tree.ts, with unit tests that also render the badges on the server (pnpm test:unit, part of pnpm check). Closes #601 Co-authored-by: jonbaldie <8376953+jonbaldie@users.noreply.github.com> --- docs/package.json | 3 +- .../components/node-compatibility-matrix.tsx | 348 ++++++++---------- docs/src/lib/compatibility-tree.ts | 157 ++++++++ docs/test/compatibility-matrix.test.mjs | 115 ++++++ 4 files changed, 424 insertions(+), 199 deletions(-) create mode 100644 docs/src/lib/compatibility-tree.ts create mode 100644 docs/test/compatibility-matrix.test.mjs diff --git a/docs/package.json b/docs/package.json index 3c2a950fd..51d74d0a1 100644 --- a/docs/package.json +++ b/docs/package.json @@ -10,9 +10,10 @@ "build": "fumadocs-mdx && next build", "start": "next start", "typecheck": "fumadocs-mdx && next typegen && tsc --noEmit", + "test:unit": "node --test test/compatibility-matrix.test.mjs", "test:routes": "node scripts/test-routes.mjs", "compat:check": "pnpm --dir .. node-compat:check", - "check": "pnpm compat:check && pnpm typecheck && pnpm build && pnpm test:routes" + "check": "pnpm compat:check && pnpm typecheck && pnpm test:unit && pnpm build && pnpm test:routes" }, "dependencies": { "@vercel/geistdocs": "2.7.8", diff --git a/docs/src/components/node-compatibility-matrix.tsx b/docs/src/components/node-compatibility-matrix.tsx index 095bcdaf2..bcc5ece61 100644 --- a/docs/src/components/node-compatibility-matrix.tsx +++ b/docs/src/components/node-compatibility-matrix.tsx @@ -4,52 +4,28 @@ // empty after an edit. // @refresh reset -import { useEffect, useMemo, useState } from "react"; +import { memo, startTransition, useCallback, useDeferredValue, useEffect, useId, useMemo, useState, type FocusEvent, type KeyboardEvent, type ReactNode } from "react"; +import { + buildTree, + expandableIds, + filterTree, + flattenTree, + nodeCount, + rowMatcher, + searchTexts, + statusCounts, + statuses, + type NodeStability, + type Row, + type Snapshot, + type Status, + type Verification, +} from "@/lib/compatibility-tree"; // Set by next.config.mjs from the committed artifact. const artifactVersion = process.env.COMPATIBILITY_ARTIFACT_VERSION; const rowCount = Number(process.env.COMPATIBILITY_ROW_COUNT); -type Status = - | "supported" - | "partial" - | "refused" - | "not-implemented" - | "by-design" - | "unreviewed" - | "not-applicable"; -type VerificationBasis = - | "test-backed" - | "explicit-refusal" - | "verified-absence" - | "declared-gap" - | "registry-gap" - | "architectural-policy" - | "unreviewed" - | "not-applicable" - | "derived"; -type Verification = { label: string; detail: string }; -type Tier = { status: Status; detail: string; verification: VerificationBasis }; -type NodeStability = { index: string; level: number; text: string; inherited: boolean } | null; -type Row = { - id: string; - chapter: string; - kind: string; - name: string; - signature: string; - label: string; - apiSymbol: string; - depth: number; - scope: "api" | "documentation" | "metadata" | "configuration"; - anchor: string; - nodeStability: NodeStability; - static: Tier; -}; -type Chapter = { slug: string; title: string; entries: number; apiEntries: number }; -type Snapshot = { nodeVersion: string; verificationBases: Record; chapters: Chapter[]; rows: Row[] }; -type TreeNode = { row: Row; children: TreeNode[] }; -type VisibleNode = { node: TreeNode; level: number }; - const labels: Record = { supported: "Supported", partial: "Partial", @@ -70,7 +46,6 @@ const styles: Record = { "not-applicable": "border-gray-alpha-400 bg-background-200 text-gray-700", }; -const statuses = ["supported", "partial", "not-implemented", "refused", "by-design", "unreviewed", "not-applicable"] as const; const stabilityStyles: Record = { 0: "border-red-500/40 bg-red-100 text-red-900 dark:bg-red-200", 1: "border-amber-500/50 bg-amber-100 text-amber-900 dark:bg-amber-200", @@ -78,114 +53,76 @@ const stabilityStyles: Record = { 3: "border-gray-alpha-500 bg-gray-alpha-100 text-gray-900", }; -function StatusBadge({ status, detail, verification }: { status: Status; detail?: string; verification?: Verification }) { - const tooltip = verification ? `${detail ?? ""}${detail ? "\n\n" : ""}Verification: ${verification.label}. ${verification.detail}` : detail; +/** A badge's tooltip, mounted only while the badge is hovered or has + * keyboard focus (the :focus-visible rule the CSS-only tooltip used), and + * named by aria-describedby while open. Escape dismisses it. */ +function BadgeTooltip({ tooltipClass, children, badge }: { tooltipClass: string; children: ReactNode; badge: ReactNode }) { + const [open, setOpen] = useState(false); + const id = useId(); + const show = useCallback(() => setOpen(true), []); + const hide = useCallback(() => setOpen(false), []); + const focus = useCallback((event: FocusEvent) => { + if (event.currentTarget.matches(":focus-visible")) setOpen(true); + }, []); + const key = useCallback((event: KeyboardEvent) => { + if (event.key === "Escape") setOpen(false); + }, []); return ( - - - {labels[status]} - - {tooltip ? ( - - {detail ? {detail} : null} - {verification ? Verification: {verification.label}. {verification.detail} : null} + + {badge} + {open ? ( + + {children} ) : null} ); } -function StabilityBadge({ stability }: { stability: NodeStability }) { +export const StatusBadge = memo(function StatusBadge({ status, detail, verification }: { status: Status; detail?: string; verification?: Verification }) { + const tooltip = verification ? `${detail ?? ""}${detail ? "\n\n" : ""}Verification: ${verification.label}. ${verification.detail}` : detail; + const badge = ( + + {labels[status]} + + ); + if (!tooltip) return {badge}; + return ( + + {detail ? {detail} : null} + {verification ? Verification: {verification.label}. {verification.detail} : null} + + ); +}); + +export const StabilityBadge = memo(function StabilityBadge({ stability }: { stability: NodeStability }) { if (stability === null) return —; const detail = `Node.js Stability ${stability.index}: ${stability.text}${stability.inherited ? " (inherited from the nearest parent section)" : ""}`; return ( - - - {stability.index} - - - {detail} - - - ); -} - -function statusCounts(rows: Row[], tier: "static") { - return rows.reduce>( - (counts, row) => { - if (row.scope !== "api" || row.depth === 0) return counts; - counts[row[tier].status] += 1; - return counts; - }, - { - supported: 0, - partial: 0, - refused: 0, - "not-implemented": 0, - "by-design": 0, - unreviewed: 0, - "not-applicable": 0, - }, - ); -} - -/** Node's all.json is preorder with an explicit heading depth. */ -function buildTree(rows: Row[]): TreeNode[] { - const roots: TreeNode[] = []; - const stack: TreeNode[] = []; - for (const row of rows) { - const node: TreeNode = { row, children: [] }; - while (stack.length > row.depth) stack.pop(); - const parent = stack[row.depth - 1]; - if (row.depth === 0 || parent === undefined) roots.push(node); - else parent.children.push(node); - stack[row.depth] = node; - stack.length = row.depth + 1; - } - return roots; -} - -function filterTree(nodes: TreeNode[], matches: (row: Row) => boolean): TreeNode[] { - const filtered: TreeNode[] = []; - for (const node of nodes) { - const children = filterTree(node.children, matches); - if (matches(node.row) || children.length > 0) filtered.push({ row: node.row, children }); - } - return filtered; -} - -function flattenTree(nodes: TreeNode[], expanded: Set, forceExpanded: boolean): VisibleNode[] { - const visible: VisibleNode[] = []; - const visit = (items: TreeNode[], level: number) => { - for (const node of items) { - visible.push({ node, level }); - if (node.children.length > 0 && (forceExpanded || expanded.has(node.row.id))) { - visit(node.children, level + 1); + + {stability.index} + } - } - }; - visit(nodes, 1); - return visible; -} - -function nodeCount(nodes: TreeNode[]): number { - return nodes.reduce((count, node) => count + 1 + nodeCount(node.children), 0); -} - -function expandableIds(nodes: TreeNode[]): Set { - const ids = new Set(); - const visit = (items: TreeNode[]) => { - for (const node of items) { - if (node.children.length > 0) ids.add(node.row.id); - visit(node.children); - } - }; - visit(nodes); - return ids; -} + > + {detail} + + ); +}); function Chevron({ open }: { open: boolean }) { return ( @@ -195,6 +132,47 @@ function Chevron({ open }: { open: boolean }) { ); } +type MatrixRowProps = { + row: Row; + level: number; + hasChildren: boolean; + open: boolean; + filtering: boolean; + displayName: string; + href: string; + verification: Verification | undefined; + onToggle: (id: string) => void; +}; + +/** One table row. Memoized on its own values, so a filter or expansion + * change re-renders only the rows whose values changed. */ +const MatrixRow = memo(function MatrixRow({ row, level, hasChildren, open, filtering, displayName, href, verification, onToggle }: MatrixRowProps) { + const isRoot = level === 1; + return ( + + +
+ {hasChildren ? ( + + ) : } +
+ + {displayName} + +
+ {row.kind}{row.apiSymbol && row.apiSymbol !== row.signature && !isRoot ? ` · ${row.apiSymbol}` : ""} +
+
+
+ + + + + ); +}); + export function NodeCompatibilityMatrix() { const [snapshot, setSnapshot] = useState(null); const [loadError, setLoadError] = useState(false); @@ -202,15 +180,16 @@ export function NodeCompatibilityMatrix() { const [staticFilter, setStaticFilter] = useState<"all" | Status>("all"); const [expanded, setExpanded] = useState>(() => new Set()); - const rows = snapshot?.rows ?? []; - const chapters = snapshot?.chapters ?? []; + const rows = useMemo(() => snapshot?.rows ?? [], [snapshot]); + const chapters = useMemo(() => snapshot?.chapters ?? [], [snapshot]); const verificationBases = snapshot?.verificationBases; const apiRowCount = useMemo(() => rows.filter((row) => row.scope === "api").length, [rows]); const chapterTitles = useMemo(() => new Map(chapters.map((item) => [item.slug, item.title])), [chapters]); const base = `https://nodejs.org/docs/v${snapshot?.nodeVersion ?? "24.22.0"}/api/`; - const staticCounts = useMemo(() => statusCounts(rows, "static"), [rows]); + const staticCounts = useMemo(() => statusCounts(rows), [rows]); const visibleStatuses = useMemo(() => statuses.filter((status) => staticCounts[status] > 0), [staticCounts]); const tree = useMemo(() => buildTree(rows), [rows]); + const texts = useMemo(() => searchTexts(rows, chapterTitles), [rows, chapterTitles]); useEffect(() => { const controller = new AbortController(); @@ -231,36 +210,23 @@ export function NodeCompatibilityMatrix() { return () => controller.abort(); }, []); - const needle = query.trim().toLocaleLowerCase(); - const hasResultFilter = needle !== "" || staticFilter !== "all"; - const rowMatches = (row: Row) => { - if (staticFilter !== "all" && row.static.status !== staticFilter) return false; - if (!needle) return true; - return [chapterTitles.get(row.chapter), row.kind, row.name, row.signature, row.apiSymbol, row.static.detail] - .join(" ") - .toLocaleLowerCase() - .includes(needle); - }; + // The search field and the status select update at once; the table + // follows the deferred values, so input stays responsive while the rows + // re-render. + const deferredQuery = useDeferredValue(query); + const deferredStatic = useDeferredValue(staticFilter); + const needle = deferredQuery.trim().toLocaleLowerCase(); + const hasResultFilter = needle !== "" || deferredStatic !== "all"; - const resultTree = useMemo( - () => hasResultFilter ? filterTree(tree, rowMatches) : tree, - // rowMatches closes over these filter values. - // eslint-disable-next-line react-hooks/exhaustive-deps - [hasResultFilter, needle, staticFilter, tree], + const result = useMemo( + () => hasResultFilter ? filterTree(tree, rowMatcher(texts, needle, deferredStatic)) : { nodes: tree, matched: nodeCount(tree) }, + [hasResultFilter, needle, deferredStatic, texts, tree], ); const visible = useMemo( - () => flattenTree(resultTree, expanded, hasResultFilter), - [expanded, hasResultFilter, resultTree], - ); - const matchedCount = useMemo( - () => hasResultFilter ? tree.reduce((count, node) => { - const visit = (item: TreeNode): number => (rowMatches(item.row) ? 1 : 0) + item.children.reduce((sum, child) => sum + visit(child), 0); - return count + visit(node); - }, 0) : nodeCount(tree), - // rowMatches closes over these filter values. - // eslint-disable-next-line react-hooks/exhaustive-deps - [hasResultFilter, needle, staticFilter, tree], + () => flattenTree(result.nodes, expanded, hasResultFilter), + [expanded, hasResultFilter, result], ); + const matchedCount = result.matched; const changeQuery = (value: string) => setQuery(value); const resetFilters = () => { @@ -268,12 +234,12 @@ export function NodeCompatibilityMatrix() { setStaticFilter("all"); setExpanded(new Set()); }; - const toggle = (id: string) => setExpanded((current) => { + const toggle = useCallback((id: string) => setExpanded((current) => { const next = new Set(current); if (next.has(id)) next.delete(id); else next.add(id); return next; - }); + }), []); useEffect(() => { if (!snapshot) return; @@ -316,8 +282,8 @@ export function NodeCompatibilityMatrix() { {visible.length.toLocaleString()} visible · {matchedCount.toLocaleString()} {hasResultFilter ? "matching" : "API"} rows {!hasResultFilter ? ( - - + + ) : null} @@ -335,33 +301,19 @@ export function NodeCompatibilityMatrix() { {visible.map(({ node, level }) => { const row = node.row; - const hasChildren = node.children.length > 0; - const open = hasResultFilter || expanded.has(row.id); - const href = `${base}${row.chapter}.html${row.anchor ? `#${row.anchor}` : ""}`; - const isRoot = level === 1; - const displayName = isRoot ? chapterTitles.get(row.chapter) ?? row.label : row.label; return ( - - -
- {hasChildren ? ( - - ) : } -
- - {displayName} - -
- {row.kind}{row.apiSymbol && row.apiSymbol !== row.signature && !isRoot ? ` · ${row.apiSymbol}` : ""} -
-
-
- - - - + 0} + open={hasResultFilter || expanded.has(row.id)} + filtering={hasResultFilter} + displayName={level === 1 ? chapterTitles.get(row.chapter) ?? row.label : row.label} + href={`${base}${row.chapter}.html${row.anchor ? `#${row.anchor}` : ""}`} + verification={verificationBases?.[row.static.verification]} + onToggle={toggle} + /> ); })} diff --git a/docs/src/lib/compatibility-tree.ts b/docs/src/lib/compatibility-tree.ts new file mode 100644 index 000000000..4044bbc8d --- /dev/null +++ b/docs/src/lib/compatibility-tree.ts @@ -0,0 +1,157 @@ +// The compatibility matrix's data model: the ledger rows as a tree, and the +// search and filter passes over it. Kept free of React so the passes can be +// tested directly. + +export type Status = + | "supported" + | "partial" + | "refused" + | "not-implemented" + | "by-design" + | "unreviewed" + | "not-applicable"; +export type VerificationBasis = + | "test-backed" + | "explicit-refusal" + | "verified-absence" + | "declared-gap" + | "registry-gap" + | "architectural-policy" + | "unreviewed" + | "not-applicable" + | "derived"; +export type Verification = { label: string; detail: string }; +export type Tier = { status: Status; detail: string; verification: VerificationBasis }; +export type NodeStability = { index: string; level: number; text: string; inherited: boolean } | null; +export type Row = { + id: string; + chapter: string; + kind: string; + name: string; + signature: string; + label: string; + apiSymbol: string; + depth: number; + scope: "api" | "documentation" | "metadata" | "configuration"; + anchor: string; + nodeStability: NodeStability; + static: Tier; +}; +export type Chapter = { slug: string; title: string; entries: number; apiEntries: number }; +export type Snapshot = { + nodeVersion: string; + verificationBases: Record; + chapters: Chapter[]; + rows: Row[]; +}; +export type TreeNode = { row: Row; children: TreeNode[] }; +export type VisibleNode = { node: TreeNode; level: number }; + +export const statuses = ["supported", "partial", "not-implemented", "refused", "by-design", "unreviewed", "not-applicable"] as const; + +export function statusCounts(rows: Row[]): Record { + return rows.reduce>( + (counts, row) => { + if (row.scope !== "api" || row.depth === 0) return counts; + counts[row.static.status] += 1; + return counts; + }, + { + supported: 0, + partial: 0, + refused: 0, + "not-implemented": 0, + "by-design": 0, + unreviewed: 0, + "not-applicable": 0, + }, + ); +} + +/** Node's all.json is preorder with an explicit heading depth. */ +export function buildTree(rows: Row[]): TreeNode[] { + const roots: TreeNode[] = []; + const stack: TreeNode[] = []; + for (const row of rows) { + const node: TreeNode = { row, children: [] }; + while (stack.length > row.depth) stack.pop(); + const parent = stack[row.depth - 1]; + if (row.depth === 0 || parent === undefined) roots.push(node); + else parent.children.push(node); + stack[row.depth] = node; + stack.length = row.depth + 1; + } + return roots; +} + +/** The lowercase text a search matches against, built once per row: the + * chapter title, kind, name, signature, API symbol and support detail. */ +export function searchTexts(rows: Row[], chapterTitles: ReadonlyMap): Map { + return new Map( + rows.map((row) => [ + row.id, + [chapterTitles.get(row.chapter), row.kind, row.name, row.signature, row.apiSymbol, row.static.detail] + .join(" ") + .toLocaleLowerCase(), + ]), + ); +} + +/** The rows a status filter and a lowercase search needle keep. */ +export function rowMatcher( + texts: ReadonlyMap, + needle: string, + status: "all" | Status, +): (row: Row) => boolean { + return (row) => { + if (status !== "all" && row.static.status !== status) return false; + return needle === "" || (texts.get(row.id) ?? "").includes(needle); + }; +} + +/** The matching rows with their ancestors, and the number of matching + * rows, in one pass. */ +export function filterTree(nodes: TreeNode[], matches: (row: Row) => boolean): { nodes: TreeNode[]; matched: number } { + let matched = 0; + const visit = (items: TreeNode[]): TreeNode[] => { + const filtered: TreeNode[] = []; + for (const node of items) { + const children = visit(node.children); + const own = matches(node.row); + if (own) matched += 1; + if (own || children.length > 0) filtered.push({ row: node.row, children }); + } + return filtered; + }; + return { nodes: visit(nodes), matched }; +} + +export function flattenTree(nodes: TreeNode[], expanded: ReadonlySet, forceExpanded: boolean): VisibleNode[] { + const visible: VisibleNode[] = []; + const visit = (items: TreeNode[], level: number) => { + for (const node of items) { + visible.push({ node, level }); + if (node.children.length > 0 && (forceExpanded || expanded.has(node.row.id))) { + visit(node.children, level + 1); + } + } + }; + visit(nodes, 1); + return visible; +} + +export function nodeCount(nodes: TreeNode[]): number { + return nodes.reduce((count, node) => count + 1 + nodeCount(node.children), 0); +} + +export function expandableIds(nodes: TreeNode[]): Set { + const ids = new Set(); + const visit = (items: TreeNode[]) => { + for (const node of items) { + if (node.children.length > 0) ids.add(node.row.id); + visit(node.children); + } + }; + visit(nodes); + return ids; +} diff --git a/docs/test/compatibility-matrix.test.mjs b/docs/test/compatibility-matrix.test.mjs new file mode 100644 index 000000000..abd3df945 --- /dev/null +++ b/docs/test/compatibility-matrix.test.mjs @@ -0,0 +1,115 @@ +import assert from "node:assert/strict"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { test } from "node:test"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { createElement } from "react"; +import { renderToStaticMarkup } from "react-dom/server"; +import ts from "typescript"; +import { + buildTree, + expandableIds, + filterTree, + flattenTree, + nodeCount, + rowMatcher, + searchTexts, + statusCounts, +} from "../src/lib/compatibility-tree.ts"; + +const row = (id, depth, label, status, detail = "") => ({ + id, + chapter: "zlib", + kind: depth === 0 ? "module" : "method", + name: label, + signature: label, + label, + apiSymbol: label, + depth, + scope: "api", + anchor: "", + nodeStability: null, + static: { status, detail, verification: "test-backed" }, +}); +const rows = [ + row("root", 0, "zlib", "partial"), + row("a", 1, "zlib.gzipSync()", "supported", "Lowered with Node's options"), + row("b", 1, "zlib.Gzip", "partial"), + row("b1", 2, "gzip.flush()", "supported"), + row("c", 1, "zlib.zstdCompress()", "not-implemented"), +]; +const titles = new Map([["zlib", "Zlib"]]); + +test("the tree follows the rows' heading depths", () => { + const tree = buildTree(rows); + assert.equal(tree.length, 1); + assert.deepEqual(tree[0].children.map((node) => node.row.id), ["a", "b", "c"]); + assert.deepEqual(tree[0].children[1].children.map((node) => node.row.id), ["b1"]); + assert.equal(nodeCount(tree), 5); + assert.deepEqual([...expandableIds(tree)].sort(), ["b", "root"]); +}); + +test("filtering keeps matching rows with their ancestors and counts the matches once", () => { + const tree = buildTree(rows); + const texts = searchTexts(rows, titles); + const byText = filterTree(tree, rowMatcher(texts, "flush", "all")); + assert.equal(byText.matched, 1); + assert.deepEqual(flattenTree(byText.nodes, new Set(), true).map(({ node, level }) => `${node.row.id}@${level}`), ["root@1", "b@2", "b1@3"]); + const byDetail = filterTree(tree, rowMatcher(texts, "node's options", "all")); + assert.deepEqual(flattenTree(byDetail.nodes, new Set(), true).map(({ node }) => node.row.id), ["root", "a"]); + const byStatus = filterTree(tree, rowMatcher(texts, "", "supported")); + assert.equal(byStatus.matched, 2); + const both = filterTree(tree, rowMatcher(texts, "gzip", "partial")); + assert.equal(both.matched, 1); + assert.equal(filterTree(tree, rowMatcher(texts, "brotli", "all")).nodes.length, 0); +}); + +test("collapsed rows hide their children until expanded", () => { + const tree = buildTree(rows); + assert.deepEqual(flattenTree(tree, new Set(), false).map(({ node }) => node.row.id), ["root"]); + assert.deepEqual(flattenTree(tree, new Set(["root"]), false).map(({ node }) => node.row.id), ["root", "a", "b", "c"]); + assert.deepEqual(flattenTree(tree, new Set(["root", "b"]), false).map(({ node }) => node.row.id), ["root", "a", "b", "b1", "c"]); +}); + +test("status counts cover API rows below the chapters", () => { + const counts = statusCounts(rows); + assert.equal(counts.supported, 2); + assert.equal(counts.partial, 1); + assert.equal(counts["not-implemented"], 1); +}); + +/** The matrix component, compiled for Node from its TSX source. */ +async function loadMatrix() { + const docs = new URL("../", import.meta.url); + const source = await readFile(new URL("src/components/node-compatibility-matrix.tsx", docs), "utf8"); + const { outputText } = ts.transpileModule(source, { + compilerOptions: { jsx: ts.JsxEmit.ReactJSX, module: ts.ModuleKind.ESNext, target: ts.ScriptTarget.ES2022 }, + }); + const lib = pathToFileURL(fileURLToPath(new URL("src/lib/compatibility-tree.ts", docs))).href; + const dir = new URL("node_modules/.cache/compatibility-matrix-test/", docs); + await mkdir(dir, { recursive: true }); + const file = new URL("node-compatibility-matrix.mjs", dir); + await writeFile(file, outputText.replace('"@/lib/compatibility-tree"', JSON.stringify(lib))); + return import(file.href); +} + +test("badges render without their tooltips until hovered or focused", async () => { + const { StatusBadge, StabilityBadge } = await loadMatrix(); + const status = renderToStaticMarkup( + createElement(StatusBadge, { + status: "partial", + detail: "Lowered with Node's options", + verification: { label: "Test-backed", detail: "Corpus programs cover it." }, + }), + ); + assert.ok(status.includes('tabindex="0"'), status); + assert.ok(status.includes('title="Lowered with Node's options'), status); + assert.ok(!status.includes('role="tooltip"'), status); + assert.ok(!status.includes("aria-describedby"), status); + const stability = renderToStaticMarkup( + createElement(StabilityBadge, { stability: { index: "2", level: 2, text: "Stable", inherited: false } }), + ); + assert.ok(stability.includes('tabindex="0"'), stability); + assert.ok(!stability.includes('role="tooltip"'), stability); + const legend = renderToStaticMarkup(createElement(StatusBadge, { status: "supported" })); + assert.ok(!legend.includes("tabindex"), legend); +}); From 4d706302292d78adc1f31be3e39f9434f6e53ecb Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 12:30:31 -0500 Subject: [PATCH 46/53] Keep a keyboard-focused badge's tooltip open when the pointer leaves Hover and keyboard focus now open the tooltip separately, as the CSS-only tooltip's hover and focus-visible rules did, so moving the pointer off a focused badge no longer unmounts its tooltip or drops aria-describedby. Escape still dismisses it. --- .../components/node-compatibility-matrix.tsx | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/docs/src/components/node-compatibility-matrix.tsx b/docs/src/components/node-compatibility-matrix.tsx index bcc5ece61..af2435668 100644 --- a/docs/src/components/node-compatibility-matrix.tsx +++ b/docs/src/components/node-compatibility-matrix.tsx @@ -55,27 +55,34 @@ const stabilityStyles: Record = { /** A badge's tooltip, mounted only while the badge is hovered or has * keyboard focus (the :focus-visible rule the CSS-only tooltip used), and - * named by aria-describedby while open. Escape dismisses it. */ + * named by aria-describedby while open. Hover and focus are tracked apart, + * so moving the pointer away keeps a keyboard-focused badge's tooltip. + * Escape dismisses it until the next hover or focus. */ function BadgeTooltip({ tooltipClass, children, badge }: { tooltipClass: string; children: ReactNode; badge: ReactNode }) { - const [open, setOpen] = useState(false); + const [hovered, setHovered] = useState(false); + const [focused, setFocused] = useState(false); + const open = hovered || focused; const id = useId(); - const show = useCallback(() => setOpen(true), []); - const hide = useCallback(() => setOpen(false), []); + const enter = useCallback(() => setHovered(true), []); + const leave = useCallback(() => setHovered(false), []); const focus = useCallback((event: FocusEvent) => { - if (event.currentTarget.matches(":focus-visible")) setOpen(true); + if (event.currentTarget.matches(":focus-visible")) setFocused(true); }, []); + const blur = useCallback(() => setFocused(false), []); const key = useCallback((event: KeyboardEvent) => { - if (event.key === "Escape") setOpen(false); + if (event.key !== "Escape") return; + setHovered(false); + setFocused(false); }, []); return ( {badge} From cda584260f660821d30912ea7ff911de8dda65de Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Fri, 23 Oct 2026 09:52:26 -0700 Subject: [PATCH 47/53] Run typed arrays that the checker types any on their native bytes A JavaScript binding referenced by its own initializer's inference collapses to `any` in the checker while it still lowers to a Uint8Array: hono's etag digest declares `const nextChunk = new Uint8Array(new ArrayBuffer(f(chunk)))` beside `chunk = nextChunk`. The typed-array lowerings key on the checker type and declined such a receiver, and the dynamic paths required a dyn value, so `nextChunk.set(...)` and `nextChunk.byteLength` threw SC1090/SC2020 at run time and multi-chunk etag digests failed. Method calls and property reads on such a receiver now box it by reference (the box shares the native storage, so writes stay visible) and dispatch on the runtime kind. Co-Authored-By: Claude Opus 5.5 --- .../lowering/expressions/property-access.ts | 19 ++++++ .../src/frontend/lowering/lower-calls.ts | 8 +++ .../7748-npm-any-bytes-receiver/main.ts.json | 6 ++ .../7748-npm-any-bytes-receiver/main.ts | 33 ++++++++++ .../node_modules/chunk-buffers/index.d.ts | 1 + .../node_modules/chunk-buffers/index.js | 18 ++++++ .../node_modules/chunk-buffers/package.json | 1 + .../node_modules/etag-digest/index.d.ts | 1 + .../node_modules/etag-digest/index.js | 60 +++++++++++++++++++ .../node_modules/etag-digest/package.json | 1 + .../7748-npm-any-bytes-receiver/main.ts.json | 1 + 11 files changed, 149 insertions(+) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7748-npm-any-bytes-receiver/main.ts.json create mode 100644 tests/corpus/7748-npm-any-bytes-receiver/main.ts create mode 100644 tests/corpus/7748-npm-any-bytes-receiver/node_modules/chunk-buffers/index.d.ts create mode 100644 tests/corpus/7748-npm-any-bytes-receiver/node_modules/chunk-buffers/index.js create mode 100644 tests/corpus/7748-npm-any-bytes-receiver/node_modules/chunk-buffers/package.json create mode 100644 tests/corpus/7748-npm-any-bytes-receiver/node_modules/etag-digest/index.d.ts create mode 100644 tests/corpus/7748-npm-any-bytes-receiver/node_modules/etag-digest/index.js create mode 100644 tests/corpus/7748-npm-any-bytes-receiver/node_modules/etag-digest/package.json create mode 100644 tests/harness/corpus-costs/7748-npm-any-bytes-receiver/main.ts.json diff --git a/packages/compiler/src/frontend/lowering/expressions/property-access.ts b/packages/compiler/src/frontend/lowering/expressions/property-access.ts index 7b12297a8..b0c7a66bb 100644 --- a/packages/compiler/src/frontend/lowering/expressions/property-access.ts +++ b/packages/compiler/src/frontend/lowering/expressions/property-access.ts @@ -1499,6 +1499,25 @@ export function lowerPropertyAccessExpr( ) { return afterReceiver(recvLowered, dynUndefinedExpr(loc), loc); } + // A typed array under a checker `any` (a JS binding its own initializer's + // inference refers to: hono's etag digest `nextChunk`) reads through its + // box, as the method-call twin dispatches. The box shares the native + // storage, so it needs no fresh-result restriction. + if ( + recvLowered.type.kind === "bytes" && + (lowerer.typeOf(expr.expression).flags & ts.TypeFlags.Any) !== 0 + ) { + return lowerer.maybeNarrow( + { + kind: "dynKeyGet", + value: lowerer.coerceToExpected(recvLowered, DYN), + key: { kind: "strLit", value: expr.name.text, type: STRING, loc }, + type: DYN, + loc, + }, + expr, + ); + } lowerer.unsupported( "SC1090", expr, diff --git a/packages/compiler/src/frontend/lowering/lower-calls.ts b/packages/compiler/src/frontend/lowering/lower-calls.ts index 672801e9e..6779cde82 100644 --- a/packages/compiler/src/frontend/lowering/lower-calls.ts +++ b/packages/compiler/src/frontend/lowering/lower-calls.ts @@ -3755,6 +3755,14 @@ function lowerDynReceiverMethodCall( // 'unknown' slot does. if (checkerUntyped && DYN_HANDLE_KINDS.has(recv.type.kind)) recv = lowerer.coerceToExpected(recv, DYN); + // A typed array whose checker type collapsed to `any` (a JS binding + // referenced in its own initializer's inference — hono's etag digest + // `const nextChunk = new Uint8Array(new ArrayBuffer(f(chunk)))` beside + // `chunk = nextChunk`) still lowered to its native bytes. The typed + // lowerings key on the checker type and decline it, so it boxes by + // reference (the box shares the native storage, so writes through the + // call stay visible) and the call dispatches on the runtime kind. + if (checkerUntyped && recv.type.kind === "bytes") recv = lowerer.coerceToExpected(recv, DYN); // The last-resort retry (boxTypedTemporary — every other lowering // already declined): a checker-`any` receiver whose value is a FRESH // typed result of an earlier step in the chain (`out.split("\n")` diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7748-npm-any-bytes-receiver/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7748-npm-any-bytes-receiver/main.ts.json new file mode 100644 index 000000000..89db88211 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7748-npm-any-bytes-receiver/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7748-npm-any-bytes-receiver/main.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7748-npm-any-bytes-receiver/main.ts b/tests/corpus/7748-npm-any-bytes-receiver/main.ts new file mode 100644 index 000000000..58301765a --- /dev/null +++ b/tests/corpus/7748-npm-any-bytes-receiver/main.ts @@ -0,0 +1,33 @@ +// Typed arrays the checker types `any` in package JavaScript: hono's etag +// digest (hono/dist/middleware/etag/digest.js) merges multi-chunk bodies +// through `nextChunk.set(...)`, where `nextChunk`'s initializer depends on +// `chunk`, which is assigned `nextChunk`. Method calls and property reads on +// such values run on the native bytes, as in Node. +import { generateDigest } from "etag-digest"; +import { grow } from "chunk-buffers"; + +const enc = new TextEncoder(); +const sha1 = (body: Uint8Array): Promise => crypto.subtle.digest({ name: "SHA-1" }, body); + +function chunked(parts: string[]): ReadableStream { + let i = 0; + return new ReadableStream({ + pull(controller): void { + if (i < parts.length) controller.enqueue(enc.encode(parts[i++]!)); + else controller.close(); + }, + }); +} + +async function main(): Promise { + console.log("single", await generateDigest(chunked([JSON.stringify({ books: 400 })]), sha1)); + console.log("chunked", await generateDigest(chunked(["alpha\n", "beta\n", "gamma\n", "delta\n"]), sha1)); + const large = ["x".repeat(200000), "y".repeat(100000), "z".repeat(5)]; + console.log("large", await generateDigest(chunked(large), sha1)); + console.log("exact", await generateDigest(chunked(["a".repeat(262144), "b"]), sha1)); + console.log("empty", await generateDigest(chunked([]), sha1)); + console.log("null", await generateDigest(null, sha1)); + console.log(grow([new Uint8Array([1, 2, 3]), new Uint8Array([4]), new Uint8Array([5])])); +} + +await main(); diff --git a/tests/corpus/7748-npm-any-bytes-receiver/node_modules/chunk-buffers/index.d.ts b/tests/corpus/7748-npm-any-bytes-receiver/node_modules/chunk-buffers/index.d.ts new file mode 100644 index 000000000..26fe364a0 --- /dev/null +++ b/tests/corpus/7748-npm-any-bytes-receiver/node_modules/chunk-buffers/index.d.ts @@ -0,0 +1 @@ +export declare function grow(parts: Uint8Array[]): Uint8Array | undefined; diff --git a/tests/corpus/7748-npm-any-bytes-receiver/node_modules/chunk-buffers/index.js b/tests/corpus/7748-npm-any-bytes-receiver/node_modules/chunk-buffers/index.js new file mode 100644 index 000000000..a0a1fea79 --- /dev/null +++ b/tests/corpus/7748-npm-any-bytes-receiver/node_modules/chunk-buffers/index.js @@ -0,0 +1,18 @@ +// `next` is referenced by its own initializer's inference (through +// `chunk = next`), so the checker types it `any` while it is a Uint8Array. +export function grow(parts) { + let chunk; + for (const value of parts) { + if (!chunk) chunk = value.slice(0); + else { + const next = new Uint8Array(new ArrayBuffer(Math.max(8, chunk.byteLength * 2))); + next.set(chunk.subarray(0, 2)); + next.set([9], 2); + next[3] = next.byteLength; + console.log(next.length, next[0], next.subarray(1, 3), next.at(-1), next.byteOffset); + console.log(next.slice(2, 4), next.buffer.byteLength, next.BYTES_PER_ELEMENT); + chunk = next; + } + } + return chunk; +} diff --git a/tests/corpus/7748-npm-any-bytes-receiver/node_modules/chunk-buffers/package.json b/tests/corpus/7748-npm-any-bytes-receiver/node_modules/chunk-buffers/package.json new file mode 100644 index 000000000..74e354d34 --- /dev/null +++ b/tests/corpus/7748-npm-any-bytes-receiver/node_modules/chunk-buffers/package.json @@ -0,0 +1 @@ +{ "name": "chunk-buffers", "version": "1.0.0", "type": "module", "main": "index.js", "types": "index.d.ts" } diff --git a/tests/corpus/7748-npm-any-bytes-receiver/node_modules/etag-digest/index.d.ts b/tests/corpus/7748-npm-any-bytes-receiver/node_modules/etag-digest/index.d.ts new file mode 100644 index 000000000..79d9dc396 --- /dev/null +++ b/tests/corpus/7748-npm-any-bytes-receiver/node_modules/etag-digest/index.d.ts @@ -0,0 +1 @@ +export declare const generateDigest: (stream: ReadableStream | null, generator: (body: Uint8Array) => ArrayBuffer | Promise) => Promise; diff --git a/tests/corpus/7748-npm-any-bytes-receiver/node_modules/etag-digest/index.js b/tests/corpus/7748-npm-any-bytes-receiver/node_modules/etag-digest/index.js new file mode 100644 index 000000000..bf438a5ea --- /dev/null +++ b/tests/corpus/7748-npm-any-bytes-receiver/node_modules/etag-digest/index.js @@ -0,0 +1,60 @@ +// hono/dist/middleware/etag/digest.js (hono 4.13.13) +const mergeBuffers = (buffer1, buffer2) => { + if (!buffer1) return buffer2; + const merged = new Uint8Array(new ArrayBuffer(buffer1.byteLength + buffer2.byteLength)); + merged.set(new Uint8Array(buffer1), 0); + merged.set(buffer2, buffer1.byteLength); + return merged; +}; +const CHUNK_SIZE = 262144; +const generateDigest = async (stream, generator) => { + if (!stream) return null; + let result = void 0; + let chunk; + let chunkLength = 0; + const digest = async (body) => { + result = await generator(mergeBuffers(result, body)); + }; + const reader = stream.getReader(); + for (;;) { + const { value, done } = await reader.read(); + if (done) break; + let offset = 0; + while (offset < value.byteLength) { + const remaining = value.byteLength - offset; + if (chunkLength === 0 && remaining >= CHUNK_SIZE) { + await digest(value.subarray(offset, offset + CHUNK_SIZE)); + offset += CHUNK_SIZE; + continue; + } + const requiredLength = chunkLength + remaining; + if (requiredLength < CHUNK_SIZE) { + if (!chunk) chunk = value.slice(offset); + else { + if (chunk.byteLength < requiredLength) { + const nextChunk = new Uint8Array(new ArrayBuffer(Math.min(CHUNK_SIZE, Math.max(requiredLength, chunk.byteLength * 2)))); + nextChunk.set(chunk.subarray(0, chunkLength)); + chunk = nextChunk; + } + chunk.set(value.subarray(offset), chunkLength); + } + chunkLength = requiredLength; + break; + } + const length = CHUNK_SIZE - chunkLength; + if (chunk?.byteLength !== CHUNK_SIZE) { + const nextChunk = new Uint8Array(/* @__PURE__ */ new ArrayBuffer(CHUNK_SIZE)); + if (chunk) nextChunk.set(chunk.subarray(0, chunkLength)); + chunk = nextChunk; + } + chunk.set(value.subarray(offset, offset + length), chunkLength); + await digest(chunk); + chunkLength = 0; + offset += length; + } + } + if (chunk && chunkLength > 0) await digest(chunk.subarray(0, chunkLength)); + if (!result) return null; + return Array.prototype.map.call(new Uint8Array(result), (x) => x.toString(16).padStart(2, "0")).join(""); +}; +export { generateDigest }; diff --git a/tests/corpus/7748-npm-any-bytes-receiver/node_modules/etag-digest/package.json b/tests/corpus/7748-npm-any-bytes-receiver/node_modules/etag-digest/package.json new file mode 100644 index 000000000..f15495eed --- /dev/null +++ b/tests/corpus/7748-npm-any-bytes-receiver/node_modules/etag-digest/package.json @@ -0,0 +1 @@ +{ "name": "etag-digest", "version": "1.0.0", "type": "module", "main": "index.js", "types": "index.d.ts" } diff --git a/tests/harness/corpus-costs/7748-npm-any-bytes-receiver/main.ts.json b/tests/harness/corpus-costs/7748-npm-any-bytes-receiver/main.ts.json new file mode 100644 index 000000000..197263cad --- /dev/null +++ b/tests/harness/corpus-costs/7748-npm-any-bytes-receiver/main.ts.json @@ -0,0 +1 @@ +{ "plain": 4600, "san": 7300 } From c0e24cdb3f6df5a56cf84a2331ab83492f4b1892 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 12:52:57 -0500 Subject: [PATCH 48/53] Embed island modules, bound V8's stack scan in-thread, and keep Node's environment out --- .github/workflows/ci.yml | 6 +- packages/cli/src/bootstrap.ts | 5 +- .../compiler/src/backend/llvm/emit-module.ts | 38 +- packages/compiler/src/backend/llvm/emitter.ts | 5 + .../compiler/src/diagnostics/diagnostic.ts | 15 +- .../src/executable/executable-cache.ts | 17 +- .../compiler/src/frontend/input-tracker.ts | 27 + .../frontend/lowering/lower-node-island.ts | 39 +- .../compiler/src/frontend/lowering/lowerer.ts | 17 +- packages/compiler/src/index.ts | 57 +- .../src/ir/function-references.test.ts | 6 + packages/compiler/src/ir/ir.ts | 20 +- packages/compiler/src/node-islands/libnode.ts | 14 +- .../compiler/src/node-islands/modules.test.ts | 188 ++++++ packages/compiler/src/node-islands/modules.ts | 627 ++++++++++++++++++ packages/compiler/src/startup-cache.ts | 1 + .../7661-node-call-modules/main.ts.json | 6 + .../7662-node-call-exit-events/main.ts.json | 6 + .../libnode-common/bridge/scr_node_bridge.cc | 71 +- packages/libnode-common/test/bridge.test.ts | 2 +- packages/node/README.md | 2 + packages/node/index.d.ts | 2 + packages/runtime/src/scr_node_bridge.h | 12 +- packages/runtime/src/scr_node_island.c | 621 ++++++++++++++++- packages/runtime/src/scr_node_island.h | 9 + packages/runtime/src/scr_node_island_host.cjs | 54 +- packages/runtime/src/scr_node_island_host.h | 54 +- .../7661-node-call-modules/lib/entry.cjs | 14 + .../corpus/7661-node-call-modules/lib/esm.mjs | 5 + .../7661-node-call-modules/lib/factor.mjs | 1 + .../corpus/7661-node-call-modules/lib/util.js | 5 + tests/corpus/7661-node-call-modules/main.ts | 12 + .../node_modules/greeter/data/message.txt | 1 + .../node_modules/greeter/index.js | 8 + .../node_modules/greeter/package.json | 1 + .../node_modules/shout/index.js | 2 + .../node_modules/shout/package.json | 1 + .../7661-node-call-modules/settings.json | 1 + .../7662-node-call-exit-events/listeners.cjs | 13 + .../corpus/7662-node-call-exit-events/main.ts | 8 + tests/fixtures/node-island/gc-stress.ts | 24 + tests/fixtures/node-island/gc.cjs | 41 ++ tests/fixtures/node-island/helper.cjs | 10 + tests/fixtures/node-island/lifecycle.ts | 1 + tests/harness/ci-stack-scope.test.ts | 16 + .../7661-node-call-modules/main.ts.json | 1 + .../7662-node-call-exit-events/main.ts.json | 1 + tests/harness/node-island.test.ts | 124 +++- 48 files changed, 2138 insertions(+), 73 deletions(-) create mode 100644 packages/compiler/src/node-islands/modules.test.ts create mode 100644 packages/compiler/src/node-islands/modules.ts create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7661-node-call-modules/main.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7662-node-call-exit-events/main.ts.json create mode 100644 tests/corpus/7661-node-call-modules/lib/entry.cjs create mode 100644 tests/corpus/7661-node-call-modules/lib/esm.mjs create mode 100644 tests/corpus/7661-node-call-modules/lib/factor.mjs create mode 100644 tests/corpus/7661-node-call-modules/lib/util.js create mode 100644 tests/corpus/7661-node-call-modules/main.ts create mode 100644 tests/corpus/7661-node-call-modules/node_modules/greeter/data/message.txt create mode 100644 tests/corpus/7661-node-call-modules/node_modules/greeter/index.js create mode 100644 tests/corpus/7661-node-call-modules/node_modules/greeter/package.json create mode 100644 tests/corpus/7661-node-call-modules/node_modules/shout/index.js create mode 100644 tests/corpus/7661-node-call-modules/node_modules/shout/package.json create mode 100644 tests/corpus/7661-node-call-modules/settings.json create mode 100644 tests/corpus/7662-node-call-exit-events/listeners.cjs create mode 100644 tests/corpus/7662-node-call-exit-events/main.ts create mode 100644 tests/fixtures/node-island/gc-stress.ts create mode 100644 tests/fixtures/node-island/gc.cjs create mode 100644 tests/harness/corpus-costs/7661-node-call-modules/main.ts.json create mode 100644 tests/harness/corpus-costs/7662-node-call-exit-events/main.ts.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fd2e7aec5..098af5326 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,9 +52,13 @@ jobs: const stackBase = pr?.stack?.base?.ref; const prBase = pr?.base?.ref; // Position 1 can already be merged; compare the current base branches. + // A PR into the default branch is the lowest unmerged layer of any + // stack, whatever the stack metadata names as its base. + const defaultBranch = event.repository?.default_branch; const upperLayer = process.env.GITHUB_EVENT_NAME === 'pull_request' && sameRepository && trustedAuthor && typeof stackBase === 'string' && stackBase.length > 0 && - typeof prBase === 'string' && prBase.length > 0 && stackBase !== prBase; + typeof prBase === 'string' && prBase.length > 0 && stackBase !== prBase && + prBase !== defaultBranch; const full = !upperLayer; appendFileSync(process.env.GITHUB_OUTPUT, `full=${full}\n`); console.log(full ? 'Run full CI' : 'Defer full CI until this PR becomes the lowest unmerged layer'); diff --git a/packages/cli/src/bootstrap.ts b/packages/cli/src/bootstrap.ts index 9eb49f823..dc5a155d5 100644 --- a/packages/cli/src/bootstrap.ts +++ b/packages/cli/src/bootstrap.ts @@ -70,9 +70,6 @@ async function tryFastPath(): Promise { values["profile-generate"] !== undefined || values["profile-use"] !== undefined || values.sparse || - // The full compiler keys these builds by the recorded libnode pack. - values.libnode !== undefined || - process.env["SCRIPTC_LIBNODE"] !== undefined || (values["external-types"] ?? []).length > 0 ) return null; @@ -159,6 +156,8 @@ async function tryFastPath(): Promise { ) : startup.executableNativeEnvironmentFingerprint(), nodeVersion: process.version, + // An entry of a program that calls into Node.js serves only the same pack. + libnodeIdentity: () => startup.libnodeCacheKey(values.libnode), }); if (hit === null) return null; if (!values["keep-llvm"]) await rm(hit.llvmPath, { force: true }); diff --git a/packages/compiler/src/backend/llvm/emit-module.ts b/packages/compiler/src/backend/llvm/emit-module.ts index c16834910..f4f3bd0f1 100644 --- a/packages/compiler/src/backend/llvm/emit-module.ts +++ b/packages/compiler/src/backend/llvm/emit-module.ts @@ -17,6 +17,7 @@ import { } from "../mangle.js"; import { type IrGlobal, + type IrNodeIslandModules, type IrType, RUNTIME_ERROR_CLASSES, RUNTIME_EVENT_CLASS, @@ -57,7 +58,8 @@ import { ffiNativeTypeLl, } from "./common.js"; import { literalCapWord, literalCapWord32 } from "./string-key-hash.js"; -import { hasLoneSurrogate, wtf8Bytes } from "../literals.js"; +import { hasLoneSurrogate, llvmBytes, wtf8Bytes } from "../literals.js"; +import { nodeIslandManifest } from "../../node-islands/modules.js"; import { undefinedArmTag } from "../../ir/analysis.js"; import { NULLABLE_ABSENT, NULLABLE_NULL } from "./nullable-unions.js"; import { literalEqualityDefinition } from "./string-literal-equality.js"; @@ -381,6 +383,11 @@ export function emitModuleParts(emitter: LlEmitter): string[] { emitter.cstr(emitter.mod.nodeIsland?.nodeVersion ?? ""), ] : null; + // The island module tree: its manifest, and its files unless they ship + // beside the executable (node-islands/modules.ts). + const nodeIslandModules = usesNodeIsland ? emitter.mod.nodeIsland?.modules : undefined; + const nodeIslandModulesCall = + nodeIslandModules === undefined ? null : nodeIslandModulesInstall(emitter, nodeIslandModules); if (emitter.ffiHasForeignCallback) emitter.declare(`declare void @scr_ffi_install()`); if (usesStream) emitter.declare(`declare void @scr_stream_install()`); if (usesNetLookup) emitter.declare(`declare void @scr_net_lookup_install()`); @@ -732,6 +739,7 @@ export function emitModuleParts(emitter: LlEmitter): string[] { for (let site = 0; site < emitter.jsIcSiteCount; site++) out.push(`@sc_js_ic_${site} = internal ${tl}global [${JS_IC_WORDS} x i64] zeroinitializer`); } + if (nodeIslandModulesCall !== null) out.push(...nodeIslandModulesCall.globals); for (const table of emitter.constantNumericTables.values()) { const n = table.values.length; let tableTmp = 0; @@ -1027,6 +1035,7 @@ export function emitModuleParts(emitter: LlEmitter): string[] { ` call void @scr_node_island_install(ptr ${nodeIslandConfig[0]}, ptr ${nodeIslandConfig[1]})`, ] : []), + ...(nodeIslandModulesCall === null ? [] : [nodeIslandModulesCall.call]), ...(emitter.ffiHasForeignCallback ? [` call void @scr_ffi_install()`] : []), ...(snapshotsTlsCa ? [` call void @scr_tls_ca_install()`] : []), ...(usesFetch ? [` call void @scr_fetch_install()`] : []), @@ -1171,6 +1180,33 @@ export function emitModuleParts(emitter: LlEmitter): string[] { /** The shared abort helpers (emitted only when referenced): the OOM * abort of untraced shape allocation and the invalid-union-tag abort — * both print the runtime ABI’s exact message on fd 2 and abort. */ +/** The island module tree's install call before %main, and the constant + * that carries its files when the executable ships them itself + * (scr_node_island.c: scr_node_island_install_modules). */ +function nodeIslandModulesInstall( + emitter: LlEmitter, + modules: IrNodeIslandModules, +): { call: string; globals: string[] } { + emitter.declare( + `declare void @scr_node_island_install_modules(ptr, ${emitter.sizeType}, ptr, ${emitter.sizeType}, ptr)`, + ); + const manifest = nodeIslandManifest(modules); + const manifestPtr = emitter.cstr(manifest); + const location = emitter.cstr(modules.location ?? ""); + const data = modules.location === undefined ? emitter.nodeIslandModuleData : null; + const globals = + data === null || data.length === 0 + ? [] + : [ + `@sc_node_island_modules = private unnamed_addr constant [${data.length} x i8] c"${llvmBytes(data, false)}", align 16`, + ]; + const st = emitter.sizeType; + return { + call: ` call void @scr_node_island_install_modules(ptr ${manifestPtr}, ${st} ${Buffer.byteLength(manifest, "utf8")}, ptr ${globals.length > 0 ? "@sc_node_island_modules" : "null"}, ${st} ${globals.length > 0 ? data!.length : 0}, ptr ${location})`, + globals, + }; +} + export function helperDefs(emitter: LlEmitter): string[] { const defs: string[] = []; const msgHelper = (fnName: string, msgSym: string, msg: string): void => { diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 050c3fb90..545d146a4 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -473,6 +473,9 @@ export interface LlvmTargetOptions { * teardown after `process.exit` to a helper process (`--detached-exit`). * Executables only; the runtime's weak default keeps it off. */ detachedExit?: boolean; + /** The files of the program's Node island module tree, in manifest order + * (node-islands/modules.ts), when the executable carries the tree. */ + nodeIslandModuleData?: Uint8Array; /** Translated WebAssembly memories check bounds explicitly instead of * relying on guard pages (wasm-modules.ts wasmBoundsChecks). */ wasmBoundsChecks?: boolean; @@ -1036,6 +1039,7 @@ export class LlEmitter { readonly emitLibraryIdentity: boolean; readonly runtimeAbiMarker: boolean; readonly detachedExit: boolean; + readonly nodeIslandModuleData: Uint8Array | null; readonly wasmBoundsChecks: boolean; /** Interned string literals: UTF-8 text → { symbol, byte length } — * first-use order, the runtime ABI’s determinism discipline. */ @@ -1618,6 +1622,7 @@ export class LlEmitter { this.emitLibraryIdentity = options.emitLibraryIdentity !== false; this.runtimeAbiMarker = options.runtimeAbiMarker === true; this.detachedExit = options.detachedExit === true; + this.nodeIslandModuleData = options.nodeIslandModuleData ?? null; this.wasmBoundsChecks = options.wasmBoundsChecks === true || wasmBoundsChecks({ diff --git a/packages/compiler/src/diagnostics/diagnostic.ts b/packages/compiler/src/diagnostics/diagnostic.ts index 6d6e81516..96b639b96 100644 --- a/packages/compiler/src/diagnostics/diagnostic.ts +++ b/packages/compiler/src/diagnostics/diagnostic.ts @@ -66,7 +66,8 @@ * likely-hot functions that use untyped values (SC6101, * diagnostics/dynamic-tier.ts) * SC8xxx Node islands: the program calls into Node.js, and the build - * finds no usable libnode pack (SC8005) + * finds no usable libnode pack (SC8005), or cannot collect the + * modules a call loads (SC8006) * SC9xxx internal compiler errors (still source-anchored) */ import type { SrcLoc } from "../ir/ir.js"; @@ -133,6 +134,18 @@ export function nodeIslandNodeDiag(message: string, file: string): ScrDiagnostic }; } +/** SC8006 — a Node island call whose modules the build cannot collect into + * the executable: a computed specifier, or a module of the program that + * loads others through computed specifiers. */ +export function nodeIslandModuleDiag(message: string, loc: SrcLoc): ScrDiagnostic { + return { + code: "SC8006", + message, + loc, + hint: "name the module with a string literal (a relative path or a package name); the build ships the modules it can see with the executable", + }; +} + /* ── SC6xxx: divergences from Node (warnings) ─────────────────────────── */ /** SC6001 — a value passed to (or declared as) a narrower record type is diff --git a/packages/compiler/src/executable/executable-cache.ts b/packages/compiler/src/executable/executable-cache.ts index b9be0724d..664b23f65 100644 --- a/packages/compiler/src/executable/executable-cache.ts +++ b/packages/compiler/src/executable/executable-cache.ts @@ -121,6 +121,9 @@ interface EarlyExecutableCacheStamp { native: EarlyExecutableNativeFeatures; /** The build's performance warnings (SC61xx), which a cache hit repeats. */ performance?: ScrDiagnostic[]; + /** Programs that call into Node.js: the libnode pack identity the build + * recorded (EarlyExecutableCacheOptions.libnodeIdentity). */ + libnode?: string; integrity: string; } @@ -139,9 +142,12 @@ export interface EarlyExecutableCacheOptions { windowsSubsystem?: "gui"; /** Omitted unless --detached-exit is set. */ detachedExit?: true; - /** Omitted unless --libnode or SCRIPTC_LIBNODE is set (calls into Node.js - * record the pack). */ - libnode?: string; + /** The identity of the libnode pack a program that calls into Node.js + * records (node-islands/libnode.ts libnodeCacheKey), or undefined when the + * build names none. Not part of the key: an entry of a program that calls + * into Node records it, and a hit requires the same identity, so the pack + * settings never change the key of a program that does not. */ + libnodeIdentity?: () => string | undefined; /** Omitted without --profile-generate/--profile-use; otherwise the * pattern, or the digest of the profile's bytes. */ profile?: string; @@ -283,7 +289,6 @@ function cacheKey(options: EarlyExecutableCacheOptions): string { ...(options.strip ? ["strip"] : []), ...(options.windowsSubsystem === "gui" ? ["windows-subsystem-gui"] : []), ...(options.detachedExit ? ["detached-exit"] : []), - ...(options.libnode !== undefined ? [`libnode:${options.libnode}`] : []), ...(options.profile !== undefined ? [options.profile] : []), ...(options.checker !== undefined ? [`checker:${options.checker}`] : []), ...(options.youngNursery ? ["young-nursery"] : []), @@ -326,7 +331,6 @@ function routeKey(options: Omit { const value = lowerer.lowerExpr(arg); @@ -109,15 +127,18 @@ export function lowerNodeApiCall( lowerer.noLowering(`'${lowerer.fmt(bad)}' results of ${surface}`, call, JSON_HINT); } lowerer.usesNodeIsland = true; + // The caller's tree path is known once every call is collected (Lowerer). + const callerLiteral: IrExpr & { kind: "strLit" } = { + kind: "strLit", + value: caller, + type: STRING, + loc, + }; + lowerer.nodeIslandCallers.push({ literal: callerLiteral, caller }); const raw: IrExpr = { kind: "libCall", fn: member === "call" ? "nodeIsland.call" : "nodeIsland.callSync", - args: [ - { kind: "strLit", value: callerPath(call), type: STRING, loc }, - specifier, - name, - { kind: "dynArrLit", elems: values, type: DYN, loc }, - ], + args: [callerLiteral, specifier, name, { kind: "dynArrLit", elems: values, type: DYN, loc }], type: member === "call" ? { kind: "promise", inner: DYN } : DYN, loc, }; diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index 43297e77c..4b6b0acdf 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -675,6 +675,7 @@ import { maybePromiseUnionOf, promiseArmedUnionOf, } from "./maybe-promise.js"; +import { NodeIslandModuleCollector } from "../../node-islands/modules.js"; /** Entry function name. '%' cannot appear in a TS identifier, so a user * function can never collide with it (mangling is injective per prefix). */ @@ -3140,6 +3141,10 @@ export class Lowerer { /** The program calls into Node.js (@scriptc/node): the module carries * the nodeIsland record, which the executable build fills in. */ usesNodeIsland = false; + /** The modules Node island calls load (lower-node-island.ts). */ + readonly nodeIslandModules = new NodeIslandModuleCollector(); + /** Each call's caller literal, set to its tree path when lowering ends. */ + readonly nodeIslandCallers: { literal: IrExpr & { kind: "strLit" }; caller: string }[] = []; /** Per-directory answers of experimental-import.ts's declaration-file test. */ readonly experimentalFiles = new Map(); /** Interned LifetimeRegion.run helpers, by callback type key. */ @@ -7805,7 +7810,7 @@ export class Lowerer { entry: ENTRY_NAME, ...(this.usesWorkers ? { workers: true } : {}), ...(this.usesLifetimeRegions ? { lifetimeRegions: true as const } : {}), - ...(this.usesNodeIsland ? { nodeIsland: { libnode: "", nodeVersion: "" } } : {}), + ...(this.usesNodeIsland ? { nodeIsland: this.nodeIslandRecord() } : {}), ...(this.declBoundary ? { declBoundary: true as const } : {}), ...(this.threadSlotKeys.size > 0 ? { threadSlotKeys: [...this.threadSlotKeys].sort() } @@ -8644,6 +8649,16 @@ export class Lowerer { this.diags.push(d); } + /** The module's Node island record: the collected module tree, with each + * call's caller literal set to its path in that tree. The executable + * build fills in the libnode pack (index.ts). */ + private nodeIslandRecord(): NonNullable { + const modules = this.nodeIslandModules.tree(); + for (const { literal, caller } of this.nodeIslandCallers) + literal.value = NodeIslandModuleCollector.callerPath(modules, caller); + return { libnode: "", nodeVersion: "", ...(modules === null ? {} : { modules }) }; + } + unsupported( code: keyof typeof UNSUPPORTED & `SC${number}`, node: ts.Node, diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index bcaf5a74f..415e12531 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -111,9 +111,11 @@ import { iceDiag, nativeCodegenDiag, nodeIslandNodeDiag, + nodeIslandModuleDiag, type ScrDiagnostic, } from "./diagnostics/diagnostic.js"; import { libnodeCacheKey, libnodeRecord } from "./node-islands/libnode.js"; +import { placeNodeIslandModules, readNodeIslandModules } from "./node-islands/modules.js"; import { loadLibraryProfile, type LibraryProfile } from "./library/library-profile.js"; import { clearFenceEvalCaches, decorateLibraryRefusals } from "./library/fence-eval.js"; import { @@ -785,7 +787,32 @@ async function prepareExecutableInput( const recorded = libnodeRecord(opts.libnode, opts.outPath); if (typeof recorded === "string") return { ok: false, diagnostics: [nodeIslandNodeDiag(recorded, entryPath)], sourceTexts }; - mod.nodeIsland = recorded; + mod.nodeIsland = { ...mod.nodeIsland, ...recorded }; + } + } + // The island module tree ships with the program: beside the executable + // with --libnode=beside, else inside it (node-islands/modules.ts). + let nodeIslandModuleData: Uint8Array | undefined; + const islandModules = mod.nodeIsland?.modules; + if (islandModules !== undefined && outputKind !== "ir") { + if (outputKind === "exe" && opts.libnode === "beside") { + const problem = placeNodeIslandModules(islandModules, `${opts.outPath}.libnode/modules`); + if (problem !== null) + return { + ok: false, + diagnostics: [nodeIslandModuleDiag(problem, { file: entryPath, start: 0, end: 0 })], + sourceTexts, + }; + islandModules.location = `$ORIGIN/${basename(opts.outPath)}.libnode/modules`; + } else { + const data = readNodeIslandModules(islandModules); + if (typeof data === "string") + return { + ok: false, + diagnostics: [nodeIslandModuleDiag(data, { file: entryPath, start: 0, end: 0 })], + sourceTexts, + }; + nodeIslandModuleData = data; } } if (warningSink !== undefined && prepared.warnings.length > 0) { @@ -820,6 +847,7 @@ async function prepareExecutableInput( lowRegion: lowRegionTarget(buildPlatform, opts.sanitize === true), ...(opts.sanitize === true ? { wasmBoundsChecks: true } : {}), ...inlineRcOption(opts.optimization, opts.sanitize === true), + ...(nodeIslandModuleData === undefined ? {} : { nodeIslandModuleData }), }); } catch (err) { if (!(err instanceof LlvmUnsupportedError)) throw err; @@ -890,6 +918,7 @@ async function prepareExecutableInput( ...(opts.sanitize === true ? { wasmBoundsChecks: true } : {}), ...inlineRcOption(opts.optimization, opts.sanitize === true), ...(opts.detachedExit === true ? { detachedExit: true } : {}), + ...(nodeIslandModuleData === undefined ? {} : { nodeIslandModuleData }), }); } catch (err) { if (!(err instanceof LlvmUnsupportedError)) throw err; @@ -1469,8 +1498,6 @@ async function compileTracked( if (outputKind === "exe") { const helperObjectRoute = opts.nativeProgramObject === true || usesPrecompiledRuntimePack(opts, "llvm"); - // Calls into Node record the libnode pack, keyed by its manifest (node-islands/libnode.ts). - const libnodeKey = libnodeCacheKey(opts.libnode); const routeOptions: Omit< EarlyExecutableCacheOptions, "implementation" | "implementationDependencies" | "nativeEnvironment" @@ -1485,7 +1512,9 @@ async function compileTracked( ...(opts.strip ? { strip: true as const } : {}), ...(opts.windowsSubsystem === "gui" ? { windowsSubsystem: "gui" as const } : {}), ...(opts.detachedExit === true ? { detachedExit: true as const } : {}), - ...(libnodeKey === undefined ? {} : { libnode: libnodeKey }), + // Calls into Node record the libnode pack; their entries record its + // identity (node-islands/libnode.ts) and a hit requires it unchanged. + libnodeIdentity: () => libnodeCacheKey(opts.libnode), ...(profile === undefined ? {} : { profile: nativeProfileKey(profile) }), ...checkerField(), ...(youngNurseryRequested() ? { youngNursery: true as const } : {}), @@ -1531,26 +1560,22 @@ async function compileTracked( implementationDependencies: implementation.dependencies, }; } - const earlyHit = + const cachedHit = earlyCacheOptions === null ? null : await readEarlyExecutableCache(cacheRoot, earlyCacheOptions); + // A --libnode=beside build that calls into Node places its island modules + // next to the executable from the lowered module, which a cache hit does + // not have: such builds take the full path. + const earlyHit = + cachedHit !== null && cachedHit.native.nodeIsland === true && opts.libnode === "beside" + ? null + : cachedHit; if (earlyHit !== null) { timing("executable-cache-hit"); if (earlyCacheOptions === null) { throw new InternalCompilerError("executable cache hit without executable cache options"); } - // The pack a --libnode=beside build places next to the executable is an output of the - // build, like the executable: a restored executable that calls into Node gets it again. - if (earlyHit.native.nodeIsland === true && opts.libnode === "beside") { - const recorded = libnodeRecord(opts.libnode, opts.outPath); - if (typeof recorded === "string") - return { - ok: false, - diagnostics: [nodeIslandNodeDiag(recorded, entryPath)], - sourceTexts: new Map(), - }; - } // A cached build repeats its performance warning (SC6101): the code it // names is just as untyped as when it was built. if (warningSink !== undefined && earlyHit.performance !== undefined) { diff --git a/packages/compiler/src/ir/function-references.test.ts b/packages/compiler/src/ir/function-references.test.ts index 0e09a7567..9541cb723 100644 --- a/packages/compiler/src/ir/function-references.test.ts +++ b/packages/compiler/src/ir/function-references.test.ts @@ -221,6 +221,12 @@ const MODULE_FIELDS: Record>, Role> = { "threadSlotKeys[]": "other", "nodeIsland.libnode": "other", "nodeIsland.nodeVersion": "other", + "nodeIsland.modules.root": "other", + "nodeIsland.modules.location": "other", + "nodeIsland.modules.files[].path": "other", + "nodeIsland.modules.files[].sha256": "other", + "nodeIsland.modules.links[].path": "other", + "nodeIsland.modules.links[].target": "other", }; const loc = { file: "references.ts", start: 0, end: 0 }; diff --git a/packages/compiler/src/ir/ir.ts b/packages/compiler/src/ir/ir.ts index 6ed304d73..e458c9045 100644 --- a/packages/compiler/src/ir/ir.ts +++ b/packages/compiler/src/ir/ir.ts @@ -1009,6 +1009,18 @@ export function isRefCounted(t: IrType): boolean { /* ── module ────────────────────────────────────────────────────────────── */ +/** The island module tree of a program (node-islands/modules.ts): paths + * relative to `root`, the directory on the build machine that holds them. + * Only the relative paths enter an executable. */ +export interface IrNodeIslandModules { + root: string; + files: { path: string; size: number; sha256: string }[]; + links: { path: string; target: string }[]; + /** Where the executable finds the tree: absent when it carries the tree + * itself, else a `$ORIGIN/` path (`--libnode=beside`). */ + location?: string; +} + export interface IrModule { /** Bumped on any breaking IR change; serialize.ts refuses mismatches. */ irVersion: 17; @@ -1067,7 +1079,13 @@ export interface IrModule { * executable) and the exact Node version its bridge links. Set by the * executable build (node-islands/libnode.ts); the emitted main records * both before %main. */ - nodeIsland?: { libnode: string; nodeVersion: string }; + nodeIsland?: { + libnode: string; + nodeVersion: string; + /** The modules the calls load (node-islands/modules.ts), when any call + * loads a file. */ + modules?: IrNodeIslandModules; + }; /** Outbound native FFI declarations used by `ffiCall` expressions. * These are link-time C ABI imports, not runtime dynamic-library handles: * executable builds resolve their symbols from the manifest's archive diff --git a/packages/compiler/src/node-islands/libnode.ts b/packages/compiler/src/node-islands/libnode.ts index 8bf85a548..a0ae29d2f 100644 --- a/packages/compiler/src/node-islands/libnode.ts +++ b/packages/compiler/src/node-islands/libnode.ts @@ -30,7 +30,7 @@ import { basename, isAbsolute, join, resolve, sep } from "node:path"; export const NODE_ISLAND_MIN_NODE = "24.13.1"; /** The bridge ABI this compiler's runtime speaks (scr_node_bridge.h). */ -export const NODE_BRIDGE_ABI = 2; +export const NODE_BRIDGE_ABI = 3; /** The prefix of a recorded path relative to the executable's directory. */ export const ORIGIN = "$ORIGIN/"; @@ -142,11 +142,13 @@ export function libnodeRecord( return { libnode: `${ORIGIN}${name}`, nodeVersion: manifest.nodeVersion }; } -/** The part of an executable's cache key that names its libnode pack, or - * undefined when neither --libnode nor SCRIPTC_LIBNODE is set. It holds the - * option, the pack directory and a digest of the pack's manifest.json, so - * that a pack upgraded in place (another Node version, bridge ABI or file - * list) gives cached executables a new key instead of their old record. */ +/** The identity of the libnode pack a build that calls into Node.js + * records, which the program's executable cache entry keeps + * (EarlyExecutableCacheOptions.libnodeIdentity), or undefined when neither + * --libnode nor SCRIPTC_LIBNODE is set. It holds the option, the pack + * directory and a digest of the pack's manifest.json, so that a pack + * upgraded in place (another Node version, bridge ABI or file list) + * invalidates the entry instead of restoring its old record. */ export function libnodeCacheKey( option: string | undefined, env: NodeJS.ProcessEnv = process.env, diff --git a/packages/compiler/src/node-islands/modules.test.ts b/packages/compiler/src/node-islands/modules.test.ts new file mode 100644 index 000000000..6badf376a --- /dev/null +++ b/packages/compiler/src/node-islands/modules.test.ts @@ -0,0 +1,188 @@ +import { mkdirSync, mkdtempSync, realpathSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { describe, expect, test } from "vitest"; +import { + NodeIslandModuleCollector, + nodeIslandManifest, + readNodeIslandModules, + scanModuleSpecifiers, +} from "./modules.js"; + +function project(files: Record): string { + const root = realpathSync(mkdtempSync(join(tmpdir(), "scriptc-island-modules-"))); + for (const [path, text] of Object.entries(files)) { + mkdirSync(dirname(join(root, path)), { recursive: true }); + writeFileSync(join(root, path), text); + } + return root; +} + +describe("module specifiers in JavaScript", () => { + test("string-literal loads in every form", () => { + const scan = scanModuleSpecifiers(` + const a = require("./a"); + const b = require.resolve('./b.json'); + import c from "c"; + import * as d from "d/sub"; + import { e } from "./e.mjs"; + import "./side"; + export * from "./f"; + export { g } from "./g"; + const h = await import("./h.js"); + import.meta.url; + obj.require("x"); + const t = \`\${require("./t")}\`; + `); + expect(scan.specifiers).toEqual([ + "./a", + "./b.json", + "c", + "d/sub", + "./e.mjs", + "./side", + "./f", + "./g", + "./h.js", + "./t", + ]); + expect(scan.computed).toEqual([]); + }); + + test("comments, strings and regular expressions hide nothing and invent nothing", () => { + const scan = scanModuleSpecifiers(` + // require("./commented") + /* import x from "./block" */ + const s = "require('./in-string')"; + const r = /require\\("\\.\\/in-regex"\\)/g; + const q = a / b / c; + const real = require("./real"); + `); + expect(scan.specifiers).toEqual(["./real"]); + expect(scan.computed).toEqual([]); + }); + + test("computed loads are reported", () => { + const forms = (text: string) => scanModuleSpecifiers(text).computed.map((c) => c.form); + expect(forms("require(name)")).toEqual(["require() with a computed specifier"]); + expect(forms("require('./a' + b)")).toEqual(["require() with a computed specifier"]); + expect(forms("import(`./x/${y}`)")).toEqual(["import() with a computed specifier"]); + expect(forms("const r = require; r('./a')")).toEqual(["require as a value"]); + expect(forms("createRequire(import.meta.url)('./a')")).toEqual([ + "a require() from createRequire()", + ]); + expect(forms("function require(x) {} typeof require")).toEqual([]); + }); +}); + +describe("the collected module tree", () => { + test("a relative module and what it loads; missing modules are not shipped", () => { + const root = project({ + "src/main.ts": "", + "src/math.cjs": + "module.exports = require('./util');\ntry { require('./optional'); } catch {}", + "src/util.js": "exports.x = require('../data.json');", + "data.json": "{}", + "src/unused.js": "", + }); + const collector = new NodeIslandModuleCollector(); + expect(collector.add(join(root, "src/main.ts"), "./math.cjs")).toBeNull(); + expect(collector.add(join(root, "src/main.ts"), "./does-not-exist.cjs")).toBeNull(); + expect(collector.add(join(root, "src/main.ts"), "node:fs")).toBeNull(); + const tree = collector.tree()!; + expect(tree.root).toBe(root); + expect(tree.files.map((f) => f.path)).toEqual(["data.json", "src/math.cjs", "src/util.js"]); + expect(NodeIslandModuleCollector.callerPath(tree, join(root, "src/main.ts"))).toBe( + "src/main.ts", + ); + }); + + test("a module of the program ships with the package.json that scopes it", () => { + const root = project({ + "package.json": JSON.stringify({ type: "module" }), + "src/main.ts": "", + "src/esm.js": "export const x = 1;", + }); + const collector = new NodeIslandModuleCollector(); + expect(collector.add(join(root, "src/main.ts"), "./esm.js")).toBeNull(); + expect(collector.tree()!.files.map((f) => f.path)).toEqual(["package.json", "src/esm.js"]); + }); + + test("a computed load in a module of the program is refused", () => { + const root = project({ "main.ts": "", "dyn.cjs": "module.exports = require(process.env.X);" }); + const problem = new NodeIslandModuleCollector().add(join(root, "main.ts"), "./dyn.cjs"); + expect(problem).toBe( + `${join(root, "dyn.cjs")}:1 loads a module through require() with a computed specifier; a module that runs in a Node island loads modules by string-literal specifiers only`, + ); + }); + + test("a package ships whole, with its dependencies, but not its nested node_modules", () => { + const root = project({ + "main.ts": "", + "node_modules/pkg/package.json": JSON.stringify({ dependencies: { dep: "1", absent: "1" } }), + "node_modules/pkg/index.js": "module.exports = require(process.env.ANYTHING_GOES);", + "node_modules/pkg/lib/extra.json": "{}", + "node_modules/pkg/node_modules/.bin/x": "", + "node_modules/dep/package.json": "{}", + "node_modules/dep/index.js": "", + "node_modules/other/index.js": "", + }); + const collector = new NodeIslandModuleCollector(); + expect(collector.add(join(root, "main.ts"), "pkg/lib/extra.json")).toBeNull(); + expect(collector.tree()!.files.map((f) => f.path)).toEqual([ + "node_modules/dep/index.js", + "node_modules/dep/package.json", + "node_modules/pkg/index.js", + "node_modules/pkg/lib/extra.json", + "node_modules/pkg/package.json", + ]); + }); + + test("pnpm's links between package directories stay links inside the tree", () => { + const root = project({ + "app/main.ts": "", + "app/node_modules/.pnpm/a@1/node_modules/a/package.json": JSON.stringify({ + dependencies: { b: "1" }, + }), + "app/node_modules/.pnpm/a@1/node_modules/a/index.js": "", + "app/node_modules/.pnpm/b@1/node_modules/b/index.js": "", + }); + symlinkSync(".pnpm/a@1/node_modules/a", join(root, "app/node_modules/a")); + symlinkSync( + "../../b@1/node_modules/b", + join(root, "app/node_modules/.pnpm/a@1/node_modules/b"), + ); + const collector = new NodeIslandModuleCollector(); + expect(collector.add(join(root, "app/main.ts"), "a")).toBeNull(); + const tree = collector.tree()!; + expect(tree.root).toBe(join(root, "app")); + expect(tree.links).toEqual([ + { path: "node_modules/.pnpm/a@1/node_modules/b", target: "../../b@1/node_modules/b" }, + { path: "node_modules/a", target: ".pnpm/a@1/node_modules/a" }, + ]); + expect(tree.files.map((f) => f.path)).toEqual([ + "node_modules/.pnpm/a@1/node_modules/a/index.js", + "node_modules/.pnpm/a@1/node_modules/a/package.json", + "node_modules/.pnpm/b@1/node_modules/b/index.js", + ]); + }); + + test("built-in modules alone ship no tree", () => { + const root = project({ "main.ts": "" }); + const collector = new NodeIslandModuleCollector(); + expect(collector.add(join(root, "main.ts"), "os")).toBeNull(); + expect(collector.tree()).toBeNull(); + expect(NodeIslandModuleCollector.callerPath(null, join(root, "main.ts"))).toBe("main.ts"); + }); + + test("the manifest names relative paths only, and the data is checked against it", () => { + const root = project({ "main.ts": "", "m.cjs": "exports.f = () => 1;" }); + const collector = new NodeIslandModuleCollector(); + collector.add(join(root, "main.ts"), "./m.cjs"); + const tree = collector.tree()!; + expect(nodeIslandManifest(tree)).not.toContain(root); + expect(readNodeIslandModules(tree)).toEqual(new TextEncoder().encode("exports.f = () => 1;")); + writeFileSync(join(root, "m.cjs"), "exports.f = () => 2;"); + expect(readNodeIslandModules(tree)).toBe(`${join(root, "m.cjs")} changed during the build`); + }); +}); diff --git a/packages/compiler/src/node-islands/modules.ts b/packages/compiler/src/node-islands/modules.ts new file mode 100644 index 000000000..0d63dc333 --- /dev/null +++ b/packages/compiler/src/node-islands/modules.ts @@ -0,0 +1,627 @@ +/* The island module tree: the JavaScript a program's Node island calls load, + * collected at build time and shipped with the executable, so a deployed + * binary never reads modules from the machine that built it. + * + * Each `call`/`callSync` names its module with a string literal. The build + * resolves it from the calling module as Node's require() does: + * - a relative or absolute specifier names a module of the program: the + * module and every module its string-literal `require()`, `import` and + * `export ... from` reach are collected; a computed specifier in such a + * module is refused, since the build cannot know what it loads; + * - a package specifier names a package: its whole directory is collected + * (computed requires inside packages keep working), and so are its + * `dependencies`, `optionalDependencies` and `peerDependencies`, found as + * Node finds them; + * - a Node built-in module needs nothing, and neither does a module the + * build cannot find: loading it throws MODULE_NOT_FOUND when the call + * runs, as under Node. + * Paths keep their layout relative to one root directory (the deepest + * directory that holds every collected path), with symbolic links between + * package directories (pnpm's layout) kept as links. Only these relative + * paths enter the executable. Every read goes through the frontend input + * tracker, so an edit to an island module invalidates cached builds. */ +import { createHash } from "node:crypto"; +import { mkdirSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { builtinModules } from "node:module"; +import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; +import { + trackedAccessibleEntries, + trackedDirectoryExists, + trackedFileExists, + trackedReadBytes, + trackedReadFile, + trackedRealpath, +} from "../frontend/input-tracker.js"; +import type { IrNodeIslandModules } from "../ir/ir.js"; + +export type NodeIslandModuleTree = IrNodeIslandModules; +type NodeIslandModuleFile = IrNodeIslandModules["files"][number]; +type NodeIslandModuleLink = IrNodeIslandModules["links"][number]; + +const BUILTINS: ReadonlySet = new Set(builtinModules); + +/** Extensions require() tries for an extensionless file, in its order. */ +const FILE_EXTENSIONS = [".js", ".json", ".node"]; + +/** Module files whose imports are traced. */ +const TRACED = /\.(?:[cm]?js|[cm]?ts)$/; + +function isBuiltin(specifier: string): boolean { + return specifier.startsWith("node:") || BUILTINS.has(specifier); +} + +function isPathSpecifier(specifier: string): boolean { + return ( + specifier === "." || + specifier === ".." || + specifier.startsWith("./") || + specifier.startsWith("../") || + isAbsolute(specifier) + ); +} + +function packageName(specifier: string): string { + const parts = specifier.split("/"); + return specifier.startsWith("@") ? parts.slice(0, 2).join("/") : parts[0]!; +} + +/** The file require() loads for `base` (a file, an extension away, or a + * directory with a package.json main or an index file), or null. */ +function resolveFile(base: string): string | null { + if (trackedFileExists(base)) return base; + for (const extension of FILE_EXTENSIONS) + if (trackedFileExists(base + extension)) return base + extension; + if (!trackedDirectoryExists(base)) return null; + const manifest = trackedReadFile(join(base, "package.json")); + if (manifest !== null) { + let main: unknown; + try { + main = (JSON.parse(manifest) as { main?: unknown }).main; + } catch { + main = undefined; + } + if (typeof main === "string" && main !== "") { + const target = join(base, main); + const found = resolveFile(target) ?? resolveFile(join(target, "index")); + if (found !== null) return found; + } + } + for (const extension of FILE_EXTENSIONS) { + const index = join(base, `index${extension}`); + if (trackedFileExists(index)) return index; + } + return null; +} + +/** The directory of package `name` as require() finds it from `fromDir`: + * `node_modules/` in `fromDir` and each ancestor not itself named + * node_modules. */ +function packageDirectory(fromDir: string, name: string): string | null { + for (let dir = fromDir; ;) { + if (basename(dir) !== "node_modules") { + const candidate = join(dir, "node_modules", name); + if (trackedDirectoryExists(candidate)) return candidate; + } + const parent = dirname(dir); + if (parent === dir) return null; + dir = parent; + } +} + +/** The package.json nearest to `dir`, as Node finds a module's package + * scope, or null. */ +function packageScope(dir: string): string | null { + for (let at = dir; ;) { + if (basename(at) === "node_modules") return null; + const candidate = join(at, "package.json"); + if (trackedFileExists(candidate)) return candidate; + const parent = dirname(at); + if (parent === at) return null; + at = parent; + } +} + +function sha256(bytes: Uint8Array): string { + return createHash("sha256").update(bytes).digest("hex"); +} + +/** The deepest directory that contains every path. */ +function commonDirectory(paths: readonly string[]): string { + let common = paths[0]!.split(sep); + for (const path of paths.slice(1)) { + const parts = path.split(sep); + let n = 0; + while (n < common.length && n < parts.length && common[n] === parts[n]) n++; + common = common.slice(0, n); + } + const joined = common.join(sep); + return joined === "" ? sep : joined; +} + +function portable(path: string): string { + return sep === "/" ? path : path.split(sep).join("/"); +} + +export class NodeIslandModuleCollector { + /** Real paths of collected files. */ + private readonly files = new Set(); + /** Link path (its directory real) → real target. */ + private readonly links = new Map(); + private readonly packages = new Set(); + private readonly traced = new Set(); + /** Real paths of the calling modules. */ + private readonly callers = new Set(); + + /** Collects the module `specifier` names from `caller` (a real path). + * Returns null, or the reason the build cannot ship it. */ + add(caller: string, specifier: string): string | null { + this.callers.add(caller); + return this.resolveFrom(caller, specifier); + } + + /** The collected tree, or null when no call loads a file. */ + tree(): NodeIslandModuleTree | null { + if (this.files.size === 0 && this.links.size === 0) return null; + const anchors = [ + ...this.files, + ...this.links.keys(), + ...this.links.values(), + ...[...this.callers].map((caller) => dirname(caller)), + ]; + const root = commonDirectory(anchors); + const files: NodeIslandModuleFile[] = []; + for (const file of [...this.files].sort()) { + const bytes = trackedReadBytes(file); + if (bytes === null) continue; + files.push({ + path: portable(relative(root, file)), + size: bytes.length, + sha256: sha256(bytes), + }); + } + const links: NodeIslandModuleLink[] = [...this.links] + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + .map(([path, target]) => ({ + path: portable(relative(root, path)), + target: portable(relative(dirname(path), target)), + })); + return { root, files, links }; + } + + /** The tree path a caller's module has: relative to the tree root. */ + static callerPath(tree: NodeIslandModuleTree | null, caller: string): string { + return tree === null ? basename(caller) : portable(relative(tree.root, caller)); + } + + private resolveFrom(fromFile: string, specifier: string): string | null { + if (isBuiltin(specifier)) return null; + if (specifier.startsWith("#")) + return `'${specifier}' is a package import ("imports" in package.json), which Node island modules of the program cannot use`; + if (specifier.startsWith("file:") || /^[a-z][a-z0-9+.-]*:/i.test(specifier)) + return `'${specifier}' is a URL; name the module by its relative path or package name`; + // A module the build cannot find is not shipped: loading it throws + // MODULE_NOT_FOUND when the call runs, as it does under Node (an + // optional require in a try block keeps working). + if (isPathSpecifier(specifier)) { + const target = resolveFile(resolve(dirname(fromFile), specifier)); + return target === null ? null : this.addModule(this.placed(target)); + } + const dir = packageDirectory(dirname(fromFile), packageName(specifier)); + if (dir !== null) this.addPackage(dir); + return null; + } + + /** The real path of `path`, recording a link when `path` itself (not an + * ancestor) is a symbolic link. */ + private placed(path: string): string { + const target = trackedRealpath(path) ?? path; + const parent = trackedRealpath(dirname(path)) ?? dirname(path); + const link = join(parent, basename(path)); + if (link !== target) this.links.set(link, target); + return target; + } + + /** A module of the program: collected with the package.json that scopes + * it (Node reads its "type"), and its imports traced. */ + private addModule(file: string): string | null { + if (this.traced.has(file)) return null; + this.traced.add(file); + this.files.add(file); + const scope = packageScope(dirname(file)); + if (scope !== null) this.files.add(scope); + if (!TRACED.test(file)) return null; + const text = trackedReadFile(file); + if (text === null) return `cannot read ${file}`; + const scan = scanModuleSpecifiers(text); + if (scan.computed.length > 0) { + const first = scan.computed[0]!; + const line = text.slice(0, first.at).split("\n").length; + return `${file}:${line} loads a module through ${first.form}; a module that runs in a Node island loads modules by string-literal specifiers only`; + } + for (const specifier of scan.specifiers) { + const problem = this.resolveFrom(file, specifier); + if (problem !== null) return problem; + } + return null; + } + + /** A package: its whole directory, then its dependencies. */ + private addPackage(found: string): void { + const dir = this.placed(found); + if (this.packages.has(dir)) return; + this.packages.add(dir); + this.addDirectory(dir, new Set()); + const manifest = trackedReadFile(join(dir, "package.json")); + if (manifest === null) return; + let parsed: Record; + try { + parsed = JSON.parse(manifest) as Record; + } catch { + return; + } + const names = new Set(); + for (const field of ["dependencies", "optionalDependencies", "peerDependencies"]) { + const deps = parsed[field]; + if (deps !== null && typeof deps === "object") + for (const name of Object.keys(deps)) names.add(name); + } + for (const name of [...names].sort()) { + // A dependency that is not installed (optional, or a peer the program + // does not use) is not loaded either. + const depDir = packageDirectory(dir, name); + if (depDir !== null) this.addPackage(depDir); + } + } + + /** Every file below `dir`, except nested node_modules (dependencies come + * through package.json) and .git. */ + private addDirectory(dir: string, visited: Set): void { + const real = trackedRealpath(dir) ?? dir; + if (visited.has(real)) return; + visited.add(real); + const entries = trackedAccessibleEntries(dir); + if (entries === null) return; + for (const file of entries.files) this.files.add(join(dir, file)); + for (const sub of entries.directories) { + if (sub === "node_modules" || sub === ".git") continue; + this.addDirectory(join(dir, sub), visited); + } + } +} + +/* ── module specifiers in JavaScript source ───────────────────────────── */ + +interface Token { + kind: "name" | "string" | "punct" | "other"; + value: string; + at: number; +} + +/** Keywords after which a `/` starts a regular expression. */ +const REGEX_AFTER = new Set([ + "return", + "typeof", + "instanceof", + "in", + "of", + "new", + "delete", + "void", + "throw", + "case", + "do", + "else", + "yield", + "await", +]); + +function isNameStart(c: string): boolean { + return /[A-Za-z_$\u0080-￿]/.test(c); +} + +function isNamePart(c: string): boolean { + return /[A-Za-z0-9_$\u0080-￿]/.test(c); +} + +/** The significant tokens of a JavaScript or TypeScript source: names, + * string literals (cooked) and punctuators, without comments, template + * text or regular expressions. */ +export function moduleTokens(text: string): Token[] { + const tokens: Token[] = []; + /** Brace depths at which a template literal's `${` was opened. */ + const templates: number[] = []; + let depth = 0; + let i = 0; + const n = text.length; + const regexAllowed = (): boolean => { + const last = tokens[tokens.length - 1]; + if (last === undefined) return true; + if (last.kind === "name") return REGEX_AFTER.has(last.value); + if (last.kind === "punct") + return last.value !== ")" && last.value !== "]" && last.value !== "}"; + return false; + }; + const template = (start: number): void => { + // From just after a backtick or a closing `}` to the end of the + // template or the next `${`. + while (i < n) { + const c = text[i]!; + if (c === "\\") { + i += 2; + continue; + } + if (c === "`") { + i++; + tokens.push({ kind: "other", value: "`", at: start }); + return; + } + if (c === "$" && text[i + 1] === "{") { + i += 2; + templates.push(depth); + depth++; + tokens.push({ kind: "punct", value: "${", at: start }); + return; + } + i++; + } + }; + while (i < n) { + const c = text[i]!; + if (c === " " || c === "\t" || c === "\n" || c === "\r" || c === "\f" || c === "\v") { + i++; + continue; + } + if (c === "/" && text[i + 1] === "/") { + while (i < n && text[i] !== "\n") i++; + continue; + } + if (c === "/" && text[i + 1] === "*") { + const end = text.indexOf("*/", i + 2); + i = end < 0 ? n : end + 2; + continue; + } + if (c === "#" && i === 0 && text[1] === "!") { + while (i < n && text[i] !== "\n") i++; + continue; + } + if (c === "'" || c === '"') { + const start = i; + let value = ""; + i++; + while (i < n && text[i] !== c && text[i] !== "\n") { + if (text[i] === "\\") { + const e = text[i + 1] ?? ""; + value += e === "n" ? "\n" : e === "t" ? "\t" : e === "r" ? "\r" : e === "\n" ? "" : e; + i += 2; + continue; + } + value += text[i]; + i++; + } + i++; + tokens.push({ kind: "string", value, at: start }); + continue; + } + if (c === "`") { + const start = i; + i++; + template(start); + continue; + } + if (c === "/" && regexAllowed()) { + const start = i; + i++; + let inClass = false; + while (i < n && text[i] !== "\n") { + const r = text[i]!; + if (r === "\\") { + i += 2; + continue; + } + if (r === "[") inClass = true; + else if (r === "]") inClass = false; + else if (r === "/" && !inClass) break; + i++; + } + i++; + while (i < n && isNamePart(text[i]!)) i++; + tokens.push({ kind: "other", value: "/re/", at: start }); + continue; + } + if (isNameStart(c)) { + const start = i; + while (i < n && isNamePart(text[i]!)) i++; + tokens.push({ kind: "name", value: text.slice(start, i), at: start }); + continue; + } + if (/[0-9]/.test(c) || (c === "." && /[0-9]/.test(text[i + 1] ?? ""))) { + const start = i; + while (i < n && /[0-9A-Za-z_.]/.test(text[i]!)) i++; + tokens.push({ kind: "other", value: text.slice(start, i), at: start }); + continue; + } + if (c === "{") depth++; + if (c === "}") { + if (templates.length > 0 && templates[templates.length - 1] === depth - 1) { + templates.pop(); + depth--; + const start = i; + i++; + tokens.push({ kind: "punct", value: "}", at: start }); + template(start); + continue; + } + depth--; + } + if (c === "." && text[i + 1] === "." && text[i + 2] === ".") { + tokens.push({ kind: "punct", value: "...", at: i }); + i += 3; + continue; + } + if (c === "?" && text[i + 1] === ".") { + tokens.push({ kind: "punct", value: "?.", at: i }); + i += 2; + continue; + } + tokens.push({ kind: "punct", value: c, at: i }); + i++; + } + return tokens; +} + +export interface ModuleSpecifierScan { + /** String-literal specifiers of require(), require.resolve(), import() + * and import/export declarations, in source order. */ + specifiers: string[]; + /** Loads whose module the source does not name with a string literal. */ + computed: { at: number; form: string }[]; +} + +export function scanModuleSpecifiers(text: string): ModuleSpecifierScan { + const tokens = moduleTokens(text); + const specifiers: string[] = []; + const computed: { at: number; form: string }[] = []; + const at = (k: number): Token | undefined => tokens[k]; + const is = (k: number, kind: Token["kind"], value?: string): boolean => { + const t = tokens[k]; + return t !== undefined && t.kind === kind && (value === undefined || t.value === value); + }; + /** A call `(` at k whose single argument is a string literal. */ + const literalCall = (k: number): string | null => + is(k, "punct", "(") && + is(k + 1, "string") && + (is(k + 2, "punct", ")") || is(k + 2, "punct", ",")) + ? at(k + 1)!.value + : null; + for (let k = 0; k < tokens.length; k++) { + const t = tokens[k]!; + if (t.kind !== "name") continue; + const member = is(k - 1, "punct", ".") || is(k - 1, "punct", "?."); + if (t.value === "require" && !member && !is(k + 1, "punct", ":")) { + if (is(k - 1, "name", "function") || is(k - 1, "name", "typeof")) continue; + let call = k + 1; + let form = "require()"; + if (is(k + 1, "punct", ".") && is(k + 2, "name", "resolve")) { + call = k + 3; + form = "require.resolve()"; + } else if (is(k + 1, "punct", ".")) { + continue; + } + if (!is(call, "punct", "(")) { + computed.push({ at: t.at, form: "require as a value" }); + continue; + } + const literal = literalCall(call); + if (literal === null) computed.push({ at: t.at, form: `${form} with a computed specifier` }); + else specifiers.push(literal); + continue; + } + if (t.value === "createRequire" && !member && is(k + 1, "punct", "(")) { + computed.push({ at: t.at, form: "a require() from createRequire()" }); + continue; + } + if (t.value === "import" && !member) { + if (is(k + 1, "punct", ".")) continue; // import.meta + if (is(k + 1, "punct", "(")) { + const literal = literalCall(k + 1); + if (literal === null) + computed.push({ at: t.at, form: "import() with a computed specifier" }); + else specifiers.push(literal); + continue; + } + if (is(k + 1, "string")) { + specifiers.push(at(k + 1)!.value); + continue; + } + for (let j = k + 1; j < tokens.length && j < k + 400; j++) { + if (is(j, "punct", ";")) break; + if (is(j, "name", "from") && is(j + 1, "string")) { + specifiers.push(at(j + 1)!.value); + break; + } + } + continue; + } + if (t.value === "export" && !member) { + let j = k + 1; + if (is(j, "name", "type")) j++; + if (is(j, "punct", "*")) { + for (; j < tokens.length && j < k + 400; j++) { + if (is(j, "punct", ";")) break; + if (is(j, "name", "from") && is(j + 1, "string")) { + specifiers.push(at(j + 1)!.value); + break; + } + } + } else if (is(j, "punct", "{")) { + while (j < tokens.length && !is(j, "punct", "}")) j++; + if (is(j + 1, "name", "from") && is(j + 2, "string")) specifiers.push(at(j + 2)!.value); + } + } + } + return { specifiers, computed }; +} + +/* ── shipping the tree ────────────────────────────────────────────────── */ + +/** The manifest the executable records (scr_node_island.c reads it): + * `{"files":[[path,size,sha256],...],"links":[[path,target],...]}`. Its + * SHA-256 names the tree. */ +export function nodeIslandManifest(modules: IrNodeIslandModules): string { + return JSON.stringify({ + files: modules.files.map((file) => [file.path, file.size, file.sha256]), + links: modules.links.map((link) => [link.path, link.target]), + }); +} + +/** The tree's files concatenated in manifest order, each checked against + * the digest the build recorded, or the reason one no longer matches. */ +export function readNodeIslandModules(modules: IrNodeIslandModules): Uint8Array | string { + const parts: Uint8Array[] = []; + let total = 0; + for (const file of modules.files) { + const path = join(modules.root, ...file.path.split("/")); + let bytes: Uint8Array; + try { + bytes = new Uint8Array(readFileSync(path)); + } catch { + return `${path} was removed during the build`; + } + if (bytes.length !== file.size || sha256(bytes) !== file.sha256) + return `${path} changed during the build`; + parts.push(bytes); + total += bytes.length; + } + const data = new Uint8Array(total); + let offset = 0; + for (const part of parts) { + data.set(part, offset); + offset += part.length; + } + return data; +} + +/** Writes the tree to `target` (replacing it), for `--libnode=beside`, or + * returns the reason it cannot. */ +export function placeNodeIslandModules( + modules: IrNodeIslandModules, + target: string, +): string | null { + const data = readNodeIslandModules(modules); + if (typeof data === "string") return data; + rmSync(target, { recursive: true, force: true }); + mkdirSync(target, { recursive: true }); + let offset = 0; + for (const file of modules.files) { + const path = join(target, ...file.path.split("/")); + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, data.subarray(offset, offset + file.size)); + offset += file.size; + } + for (const link of modules.links) { + const path = join(target, ...link.path.split("/")); + mkdirSync(dirname(path), { recursive: true }); + symlinkSync(link.target, path); + } + return null; +} diff --git a/packages/compiler/src/startup-cache.ts b/packages/compiler/src/startup-cache.ts index 1bc74bef0..950a96d08 100644 --- a/packages/compiler/src/startup-cache.ts +++ b/packages/compiler/src/startup-cache.ts @@ -2,6 +2,7 @@ * full lowering/emitter graph; exact builds need only these cache/native * identity primitives before deciding whether that graph is necessary. */ export { readRoutedExecutableCache } from "./executable/executable-cache.js"; +export { libnodeCacheKey } from "./node-islands/libnode.js"; export type { EarlyExecutableRouteOptions } from "./executable/executable-cache.js"; export { executableNativeEnvironmentFingerprint, diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7661-node-call-modules/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7661-node-call-modules/main.ts.json new file mode 100644 index 000000000..85440ddc4 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7661-node-call-modules/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7661-node-call-modules/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7662-node-call-exit-events/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7662-node-call-exit-events/main.ts.json new file mode 100644 index 000000000..455735463 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7662-node-call-exit-events/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7662-node-call-exit-events/main.ts" + ], + "diags": [] +} diff --git a/packages/libnode-common/bridge/scr_node_bridge.cc b/packages/libnode-common/bridge/scr_node_bridge.cc index e770d140f..ef60851d6 100644 --- a/packages/libnode-common/bridge/scr_node_bridge.cc +++ b/packages/libnode-common/bridge/scr_node_bridge.cc @@ -49,6 +49,7 @@ node::EmbedderSnapshotData::Pointer g_snapshot; std::unique_ptr g_setup; Locker *g_locker = nullptr; Global g_call; +Global g_finish; char *Copy(const std::string &text, size_t *len) { char *out = static_cast(std::malloc(text.size() + 1)); @@ -59,6 +60,12 @@ char *Copy(const std::string &text, size_t *len) { return out; } +// The start of the runtime's stack when the caller runs on it (see the +// v8::base::Stack definitions at the end of this file), else 0. +uintptr_t HostStackStart() { + return g_host != nullptr && g_host->stack_start != nullptr ? g_host->stack_start() : 0; +} + Local Utf8(Isolate *isolate, const char *text, size_t len) { return String::NewFromUtf8(isolate, text, NewStringType::kNormal, static_cast(len)) .ToLocalChecked(); @@ -117,12 +124,18 @@ char *Start(const ScrNodeBridgeHost *host, const char *bootstrap, const char *ar if (dladdr(reinterpret_cast(&napi_create_function), &info) && info.dli_fname) dlopen(info.dli_fname, RTLD_NOW | RTLD_NOLOAD | RTLD_GLOBAL); std::vector args = {argv0}; + // The program is not a node process: NODE_OPTIONS from its environment + // does not apply, and Node leaves the process's resource limits and code + // pages as they are. auto result = node::InitializeOncePerProcess( args, {node::ProcessInitializationFlags::kNoInitializeV8, node::ProcessInitializationFlags::kNoInitializeNodeV8Platform, node::ProcessInitializationFlags::kNoStdioInitialization, node::ProcessInitializationFlags::kNoDefaultSignalHandling, - node::ProcessInitializationFlags::kNoPrintHelpOrVersionOutput}); + node::ProcessInitializationFlags::kNoPrintHelpOrVersionOutput, + node::ProcessInitializationFlags::kDisableNodeOptionsEnv, + node::ProcessInitializationFlags::kNoAdjustResourceLimits, + node::ProcessInitializationFlags::kNoUseLargePages}); if (result->early_return() || result->exit_code() != 0) { std::string message = "Node initialization failed"; for (const std::string &error : result->errors()) message += ": " + error; @@ -132,9 +145,12 @@ char *Start(const ScrNodeBridgeHost *host, const char *bootstrap, const char *ar v8::V8::InitializePlatform(g_platform.get()); v8::V8::Initialize(); g_snapshot = node::EmbedderSnapshotData::BuiltinSnapshotData(); + // Modules resolve inside the program's own module tree, never from + // NODE_PATH or the user's global module directories. auto flags = static_cast( node::EnvironmentFlags::kDefaultFlags | node::EnvironmentFlags::kNoCreateInspector | - node::EnvironmentFlags::kNoStartDebugSignalHandler); + node::EnvironmentFlags::kNoStartDebugSignalHandler | + node::EnvironmentFlags::kNoGlobalSearchPaths); std::vector errors; g_setup = g_snapshot ? node::CommonEnvironmentSetup::CreateFromSnapshot( g_platform.get(), &errors, g_snapshot.get(), result->args(), @@ -182,12 +198,16 @@ char *Start(const ScrNodeBridgeHost *host, const char *bootstrap, const char *ar } Local dispatcher; Local call; + Local finish; if (!global->Get(context, Utf8(isolate, "__scriptcNodeIsland")).ToLocal(&dispatcher) || !dispatcher->IsObject() || !dispatcher.As()->Get(context, Utf8(isolate, "call")).ToLocal(&call) || - !call->IsFunction()) + !call->IsFunction() || + !dispatcher.As()->Get(context, Utf8(isolate, "finish")).ToLocal(&finish) || + !finish->IsFunction()) return Copy("the Node island bootstrap left no dispatcher", nullptr); g_call.Reset(isolate, call.As()); + g_finish.Reset(isolate, finish.As()); global->Delete(context, Utf8(isolate, "__scriptcNodeIsland")).FromMaybe(false); global->Delete(context, Utf8(isolate, "__scriptcNodeIslandHost")).FromMaybe(false); return nullptr; @@ -254,11 +274,29 @@ void Pump() { g_platform->DrainTasks(isolate); } +// The program ends with `code`: Node's process ends with it too. Node's +// 'beforeExit' runs (and the work it schedules), then 'exit'; the result +// is the exit code Node's process ends with (process.exitCode). +int Finish(int code) { + if (!g_setup) return code; + Isolate *isolate = g_setup->isolate(); + { + HandleScope handles(isolate); + Local context = g_setup->context(); + Context::Scope context_scope(context); + TryCatch try_catch(isolate); + Local argv[] = {Integer::New(isolate, code)}; + (void)g_finish.Get(isolate)->Call(context, v8::Undefined(isolate), 1, argv); + } + return node::SpinEventLoop(g_setup->env()).FromMaybe(code); +} + void Stop() { if (!g_setup) return; Isolate *isolate = g_setup->isolate(); node::Stop(g_setup->env()); g_call.Reset(); + g_finish.Reset(); isolate->Exit(); delete g_locker; g_locker = nullptr; @@ -271,10 +309,37 @@ void Stop() { const ScrNodeBridge kBridge = { SCR_NODE_BRIDGE_ABI, NODE_VERSION_STRING, NODE_MODULE_VERSION, Start, Call, Release, BackendFd, BackendTimeout, Alive, Pump, Stop, + Finish, }; } // namespace +#if defined(__linux__) +// V8 asks v8::base::Stack for the start of the current thread's stack when +// it records the bound of its conservative stack scan (isolate creation, +// Locker). libnode calls these exported functions through its PLT, and this +// library precedes libnode in the lookup scope, so the definitions below +// answer for it: the runtime's stack while the caller runs on it, else +// libnode's own answer (RTLD_NEXT). Without them a garbage collection that +// scans the stack while JavaScript runs on the runtime's stack walks from +// that stack up to the main thread's and faults in the unmapped gap. +// (macOS binds libnode's calls inside libnode; its packs need another way.) +#define SCR_V8_STACK_START(fn, symbol) \ + extern "C" __attribute__((visibility("default"))) void *fn() __asm__(symbol); \ + extern "C" void *fn() { \ + static void *(*next)() = nullptr; \ + uintptr_t start = HostStackStart(); \ + if (start != 0) return reinterpret_cast(start); \ + if (next == nullptr) next = reinterpret_cast(dlsym(RTLD_NEXT, symbol)); \ + return next != nullptr ? next() : nullptr; \ + } +SCR_V8_STACK_START(ScrV8GetStackStart, "_ZN2v84base5Stack13GetStackStartEv") +SCR_V8_STACK_START(ScrV8GetStackStartUnchecked, "_ZN2v84base5Stack22GetStackStartUncheckedEv") +SCR_V8_STACK_START(ScrV8ObtainCurrentThreadStackStart, + "_ZN2v84base5Stack29ObtainCurrentThreadStackStartEv") +#undef SCR_V8_STACK_START +#endif + extern "C" __attribute__((visibility("default"))) const ScrNodeBridge *scr_node_bridge_v1() { return &kBridge; } diff --git a/packages/libnode-common/test/bridge.test.ts b/packages/libnode-common/test/bridge.test.ts index 1ab6ab95b..4c37b31a3 100644 --- a/packages/libnode-common/test/bridge.test.ts +++ b/packages/libnode-common/test/bridge.test.ts @@ -45,7 +45,7 @@ exports.quit = (code) => process.exit(code); expect(result.stderr).toBe(""); expect(result.stdout).toBe( [ - "abi 2 ok 1", + "abi 3 ok 1", "sync add -> R5", 'sync fails -> T{"error":{"kind":2,"message":"out of range","code":"ERR_X"}}', `sync missing -> T{"error":{"kind":1,"message":"'missing' exported by './smoke.cjs' is not a function"}}`, diff --git a/packages/node/README.md b/packages/node/README.md index 77002db49..7b1c3656d 100644 --- a/packages/node/README.md +++ b/packages/node/README.md @@ -10,6 +10,8 @@ const config = await call<{ regions: string[] }>("config-parser", "parse", sourc ``` - The module loads with `require()` from the calling module: relative specifiers resolve from its directory, bare ones from its `node_modules`. +- In scriptc builds the specifier is a string literal. The build collects the module into the executable: a module of the program with the modules its string-literal `require()` and `import` reach, or a whole package with its dependencies. The executable loads them from its own copy, never from the build machine's files. A module of the program that loads others through computed specifiers is refused (SC8006); a module the build cannot find throws `MODULE_NOT_FOUND` when the call runs. +- The executable extracts its modules into `$XDG_CACHE_HOME/scriptc/node-islands/` (or `~/.cache/scriptc/node-islands/`) and checks every file against the build's digests on each start. With `--libnode=beside` they are written next to the executable instead, in `.libnode/modules/`. - Arguments and results cross as JSON copies; scriptc checks the result against the declared type. - A thrown `Error` crosses as its nearest intrinsic error class with its `message` and string `code`. - The executable loads Node from the libnode pack recorded at build time (`SCRIPTC_LIBNODE` or `scriptc build --libnode=`; `--libnode=beside` copies the pack next to the executable). `SCRIPTC_LIBNODE` overrides it at run time. A program that never calls into Node never loads it. diff --git a/packages/node/index.d.ts b/packages/node/index.d.ts index 10642b098..87a8eb541 100644 --- a/packages/node/index.d.ts +++ b/packages/node/index.d.ts @@ -9,6 +9,8 @@ * - the module loads with require() from the calling module, so a relative * specifier resolves from the caller's directory and a bare one from its * node_modules (ES modules load through require(esm)); + * - in scriptc builds `specifier` is a string literal: the build ships the + * module, and what it loads, inside the executable; * - the arguments cross as a copy, `JSON.parse(JSON.stringify(args))`, and * the function runs with the module namespace as `this`; * - the result crosses as its JSON copy (undefined stays undefined), and diff --git a/packages/runtime/src/scr_node_bridge.h b/packages/runtime/src/scr_node_bridge.h index 8ccdeaf8d..f88764691 100644 --- a/packages/runtime/src/scr_node_bridge.h +++ b/packages/runtime/src/scr_node_bridge.h @@ -16,7 +16,7 @@ #include #include -#define SCR_NODE_BRIDGE_ABI 2u +#define SCR_NODE_BRIDGE_ABI 3u /* The symbol a bridge library exports: `const ScrNodeBridge *(void)`. */ #define SCR_NODE_BRIDGE_ENTRY "scr_node_bridge_v1" @@ -27,6 +27,12 @@ typedef struct ScrNodeBridgeHost { /* Node ends the process (process.exit, an uncaught exception): the * runtime runs its exit sequence with this code and never returns. */ void (*exit)(int code); + /* The start (highest address) of the stack the caller runs on when the + * runtime provides it (Node's own stack), 0 for a thread's own stack. + * V8 scans the stack conservatively from the current position up to the + * start it records for the thread, so it must not record the thread's + * when JavaScript runs on the runtime's stack. */ + uintptr_t (*stack_start)(void); } ScrNodeBridgeHost; typedef struct ScrNodeBridge { @@ -58,6 +64,10 @@ typedef struct ScrNodeBridge { void (*pump)(void); /* Tears Node down at process exit. */ void (*stop)(void); + /* The program ends with `code`, before stop: Node's 'beforeExit' and + * 'exit' run, and the result is the code Node's process ends with + * (process.exitCode in Node, else `code`). */ + int (*finish)(int code); } ScrNodeBridge; #endif /* SCR_NODE_BRIDGE_H */ diff --git a/packages/runtime/src/scr_node_island.c b/packages/runtime/src/scr_node_island.c index 55ef2ff9d..e32f8da41 100644 --- a/packages/runtime/src/scr_node_island.c +++ b/packages/runtime/src/scr_node_island.c @@ -45,8 +45,12 @@ #if !defined(_WIN32) && !defined(__wasi__) #include "scr_node_island_host.h" +#include #include +#include +#include #include +#include #include #include #if defined(__linux__) @@ -92,6 +96,15 @@ void scr_node_island_install(const char *libnode, const char *node_version) { SCR_COLD static void ni_oom(void) { scr_trap("scriptc: out of memory\n"); } +/* `len` bytes as a NUL-terminated malloc'd copy. */ +static char *ni_cstr_of(const char *bytes, size_t len) { + char *copy = malloc(len + 1); + if (copy == NULL) ni_oom(); + memcpy(copy, bytes, len); + copy[len] = '\0'; + return copy; +} + SCR_COLD static void ni_fatal(const char *message) { fflush(stdout); fputs(message, stderr); @@ -149,6 +162,550 @@ static const char *ni_pack_dir(void) { return ni_dir; } +/* ── the island module tree ───────────────────────────────────────────── */ + +/* The JavaScript the program's calls load (node-islands/modules.ts), named + * by a manifest of relative paths with their sizes and SHA-256 digests, plus + * the links between package directories. Before Node loads anything, the + * tree is made available at one directory, and every file is checked against + * the manifest: + * - beside the executable (`--libnode=beside`), at the recorded `$ORIGIN/` + * path, checked on every start; + * - otherwise from the executable itself, extracted once into + * $XDG_CACHE_HOME/scriptc/node-islands// (or + * ~/.cache/...), checked on every start; when that directory cannot be + * used, into a private directory under $TMPDIR removed at exit. + * Directories are created 0700 and owned by the process's user; no link is + * followed below the cache directory. A process with elevated privileges + * refuses the tree, as it refuses SCRIPTC_LIBNODE. A call names its module + * by its path relative to the tree. */ +static const char *ni_manifest = NULL; +static size_t ni_manifest_len = 0; +static const unsigned char *ni_modules = NULL; +static size_t ni_modules_len = 0; +static const char *ni_modules_location = NULL; +/* The tree's directory at run time; "/" when the calls load only Node's + * built-in modules. */ +static char ni_root[4096] = "/"; + +void scr_node_island_install_modules(const char *manifest, size_t manifest_len, + const unsigned char *data, size_t data_len, + const char *location) { + ni_manifest = manifest; + ni_manifest_len = manifest_len; + ni_modules = data; + ni_modules_len = data_len; + ni_modules_location = location; +} + +/* SHA-256 (FIPS 180-4), for the manifest's digests. */ +typedef struct { + uint32_t h[8]; + uint64_t bytes; + unsigned char tail[64]; + size_t tail_len; +} NiSha256; + +static const uint32_t ni_sha256_k[64] = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2}; + +static uint32_t ni_rotr(uint32_t x, unsigned n) { return (x >> n) | (x << (32 - n)); } + +static void ni_sha256_block(uint32_t h[8], const unsigned char *p) { + uint32_t w[64]; + for (int i = 0; i < 16; i++) + w[i] = ((uint32_t)p[i * 4] << 24) | ((uint32_t)p[i * 4 + 1] << 16) | + ((uint32_t)p[i * 4 + 2] << 8) | (uint32_t)p[i * 4 + 3]; + for (int i = 16; i < 64; i++) { + uint32_t s0 = ni_rotr(w[i - 15], 7) ^ ni_rotr(w[i - 15], 18) ^ (w[i - 15] >> 3); + uint32_t s1 = ni_rotr(w[i - 2], 17) ^ ni_rotr(w[i - 2], 19) ^ (w[i - 2] >> 10); + w[i] = w[i - 16] + s0 + w[i - 7] + s1; + } + uint32_t a = h[0], b = h[1], c = h[2], d = h[3], e = h[4], f = h[5], g = h[6], k = h[7]; + for (int i = 0; i < 64; i++) { + uint32_t t1 = k + (ni_rotr(e, 6) ^ ni_rotr(e, 11) ^ ni_rotr(e, 25)) + ((e & f) ^ (~e & g)) + + ni_sha256_k[i] + w[i]; + uint32_t t2 = (ni_rotr(a, 2) ^ ni_rotr(a, 13) ^ ni_rotr(a, 22)) + ((a & b) ^ (a & c) ^ (b & c)); + k = g; + g = f; + f = e; + e = d + t1; + d = c; + c = b; + b = a; + a = t1 + t2; + } + h[0] += a; + h[1] += b; + h[2] += c; + h[3] += d; + h[4] += e; + h[5] += f; + h[6] += g; + h[7] += k; +} + +static void ni_sha256_init(NiSha256 *s) { + static const uint32_t initial[8] = {0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, + 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19}; + memcpy(s->h, initial, sizeof initial); + s->bytes = 0; + s->tail_len = 0; +} + +static void ni_sha256_update(NiSha256 *s, const unsigned char *data, size_t len) { + s->bytes += len; + while (len > 0) { + if (s->tail_len == 0 && len >= 64) { + ni_sha256_block(s->h, data); + data += 64; + len -= 64; + continue; + } + size_t take = 64 - s->tail_len < len ? 64 - s->tail_len : len; + memcpy(s->tail + s->tail_len, data, take); + s->tail_len += take; + data += take; + len -= take; + if (s->tail_len == 64) { + ni_sha256_block(s->h, s->tail); + s->tail_len = 0; + } + } +} + +static void ni_sha256_final(NiSha256 *s, unsigned char out[32]) { + uint64_t bits = s->bytes * 8; + unsigned char pad[72] = {0x80}; + size_t padding = (s->tail_len < 56 ? 56 : 120) - s->tail_len; + for (int i = 0; i < 8; i++) pad[padding + i] = (unsigned char)(bits >> (56 - 8 * i)); + ni_sha256_update(s, pad, padding + 8); + for (int i = 0; i < 8; i++) + for (int b = 0; b < 4; b++) out[i * 4 + b] = (unsigned char)(s->h[i] >> (24 - 8 * b)); +} + +typedef struct { + char *path; + size_t size; + unsigned char sha256[32]; + size_t offset; /* into ni_modules */ +} NiFile; + +typedef struct { + char *path, *target; +} NiLink; + +static NiFile *ni_files = NULL; +static size_t ni_nfiles = 0; +static NiLink *ni_links = NULL; +static size_t ni_nlinks = 0; + +SCR_COLD static void ni_bad_manifest(void) { + ni_fatal("scriptc: this program's Node island module manifest is malformed\n"); +} + +static char *ni_dup_str(const ScrDyn *d) { + if (d == NULL || d->kind != SCR_DYN_STR) ni_bad_manifest(); + if (memchr(d->v.str->data, '\0', d->v.str->len) != NULL) ni_bad_manifest(); + return ni_cstr_of(d->v.str->data, d->v.str->len); +} + +/* A relative path inside the tree: no leading `/`, no empty, `.` or `..` + * component. */ +static bool ni_relative_inside(const char *path) { + if (path[0] == '\0' || path[0] == '/') return false; + for (const char *p = path;;) { + const char *slash = strchr(p, '/'); + size_t n = slash != NULL ? (size_t)(slash - p) : strlen(p); + if (n == 0 || (n == 1 && p[0] == '.') || (n == 2 && p[0] == '.' && p[1] == '.')) return false; + if (slash == NULL) return true; + p = slash + 1; + } +} + +/* A link target that, read from the link's directory, stays inside the + * tree. */ +static bool ni_link_inside(const char *link, const char *target) { + if (target[0] == '\0' || target[0] == '/') return false; + long depth = 0; + for (const char *p = link; (p = strchr(p, '/')) != NULL; p++) depth++; + for (const char *p = target;;) { + const char *slash = strchr(p, '/'); + size_t n = slash != NULL ? (size_t)(slash - p) : strlen(p); + if (n == 2 && p[0] == '.' && p[1] == '.') { + if (--depth < 0) return false; + } else if (n == 0 || !(n == 1 && p[0] == '.')) { + if (n == 0) return false; + depth++; + } + if (slash == NULL) return true; + p = slash + 1; + } +} + +static int ni_hex(char c) { + return c >= '0' && c <= '9' ? c - '0' : c >= 'a' && c <= 'f' ? c - 'a' + 10 : -1; +} + +static void ni_parse_manifest(void) { + ScrStr *text = scr_str_new(ni_manifest, ni_manifest_len); + ScrDyn *body = scr_json_parse(text); + scr_str_release(text); + if (body == NULL || body->kind != SCR_DYN_OBJ) ni_bad_manifest(); + ScrDyn *files = scr_dyn_obj_get(body, "files", 5); + ScrDyn *links = scr_dyn_obj_get(body, "links", 5); + if (files == NULL || files->kind != SCR_DYN_ARR || links == NULL || links->kind != SCR_DYN_ARR) + ni_bad_manifest(); + ni_nfiles = (size_t)scr_dyn_arr_len(files); + ni_nlinks = (size_t)scr_dyn_arr_len(links); + ni_files = calloc(ni_nfiles + 1, sizeof *ni_files); + ni_links = calloc(ni_nlinks + 1, sizeof *ni_links); + if (ni_files == NULL || ni_links == NULL) ni_oom(); + size_t offset = 0; + for (size_t i = 0; i < ni_nfiles; i++) { + ScrDyn *entry = scr_dyn_arr_at(files, (double)i); + if (entry->kind != SCR_DYN_ARR || scr_dyn_arr_len(entry) != 3) ni_bad_manifest(); + ScrDyn *path = scr_dyn_arr_at(entry, 0), *size = scr_dyn_arr_at(entry, 1), + *digest = scr_dyn_arr_at(entry, 2); + NiFile *f = &ni_files[i]; + f->path = ni_dup_str(path); + if (!ni_relative_inside(f->path) || size->kind != SCR_DYN_NUM || size->v.num < 0 || + digest->kind != SCR_DYN_STR || digest->v.str->len != 64) + ni_bad_manifest(); + f->size = (size_t)size->v.num; + for (int b = 0; b < 32; b++) { + int hi = ni_hex(digest->v.str->data[b * 2]), lo = ni_hex(digest->v.str->data[b * 2 + 1]); + if (hi < 0 || lo < 0) ni_bad_manifest(); + f->sha256[b] = (unsigned char)(hi * 16 + lo); + } + f->offset = offset; + offset += f->size; + scr_dyn_release(path); + scr_dyn_release(size); + scr_dyn_release(digest); + scr_dyn_release(entry); + } + if (ni_modules != NULL && offset != ni_modules_len) ni_bad_manifest(); + for (size_t i = 0; i < ni_nlinks; i++) { + ScrDyn *entry = scr_dyn_arr_at(links, (double)i); + if (entry->kind != SCR_DYN_ARR || scr_dyn_arr_len(entry) != 2) ni_bad_manifest(); + ScrDyn *path = scr_dyn_arr_at(entry, 0), *target = scr_dyn_arr_at(entry, 1); + ni_links[i].path = ni_dup_str(path); + ni_links[i].target = ni_dup_str(target); + if (!ni_relative_inside(ni_links[i].path) || + !ni_link_inside(ni_links[i].path, ni_links[i].target)) + ni_bad_manifest(); + scr_dyn_release(path); + scr_dyn_release(target); + scr_dyn_release(entry); + } + scr_dyn_release(body); +} + +/* The directory `name` in `dirfd`, never through a link; created 0700 when + * `create` and absent. -1 when it cannot be opened. */ +static int ni_dir_at(int dirfd, const char *name, bool create) { + if (create && mkdirat(dirfd, name, 0700) != 0 && errno != EEXIST) return -1; + return openat(dirfd, name, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); +} + +/* The parent directory of the relative `path` below `rootfd` (its last + * component is left in *base), opened component by component without + * following links; created when `create`. -1 when it cannot be opened. */ +static int ni_parent_at(int rootfd, const char *path, bool create, const char **base) { + int fd = dup(rootfd); + const char *p = path; + for (const char *slash; fd >= 0 && (slash = strchr(p, '/')) != NULL; p = slash + 1) { + char component[256]; + size_t n = (size_t)(slash - p); + if (n >= sizeof component) { + close(fd); + return -1; + } + memcpy(component, p, n); + component[n] = '\0'; + int next = ni_dir_at(fd, component, create); + close(fd); + fd = next; + } + *base = p; + return fd; +} + +/* A directory this process may trust: owned by its user, writable by no + * one else. */ +static bool ni_private_dir(int fd) { + struct stat st; + return fd >= 0 && fstat(fd, &st) == 0 && S_ISDIR(st.st_mode) && st.st_uid == geteuid() && + (st.st_mode & 022) == 0; +} + +static bool ni_digest_matches(const unsigned char *data, size_t len, + const unsigned char expected[32]) { + NiSha256 s; + unsigned char got[32]; + ni_sha256_init(&s); + ni_sha256_update(&s, data, len); + ni_sha256_final(&s, got); + return memcmp(got, expected, 32) == 0; +} + +/* Whether the tree below `rootfd` holds exactly the manifest's files (by + * size and digest) and links. */ +static bool ni_tree_matches(int rootfd) { + for (size_t i = 0; i < ni_nfiles; i++) { + const char *base; + int dirfd = ni_parent_at(rootfd, ni_files[i].path, false, &base); + if (dirfd < 0) return false; + int fd = openat(dirfd, base, O_RDONLY | O_NOFOLLOW | O_CLOEXEC); + close(dirfd); + struct stat st; + if (fd < 0 || fstat(fd, &st) != 0 || !S_ISREG(st.st_mode) || + (size_t)st.st_size != ni_files[i].size) { + if (fd >= 0) close(fd); + return false; + } + NiSha256 s; + ni_sha256_init(&s); + unsigned char buffer[65536]; + size_t total = 0; + for (;;) { + ssize_t n = read(fd, buffer, sizeof buffer); + if (n < 0 && errno == EINTR) continue; + if (n <= 0) break; + ni_sha256_update(&s, buffer, (size_t)n); + total += (size_t)n; + } + close(fd); + unsigned char got[32]; + ni_sha256_final(&s, got); + if (total != ni_files[i].size || memcmp(got, ni_files[i].sha256, 32) != 0) return false; + } + for (size_t i = 0; i < ni_nlinks; i++) { + const char *base; + int dirfd = ni_parent_at(rootfd, ni_links[i].path, false, &base); + if (dirfd < 0) return false; + char target[4096]; + ssize_t n = readlinkat(dirfd, base, target, sizeof target - 1); + close(dirfd); + if (n < 0) return false; + target[n] = '\0'; + if (strcmp(target, ni_links[i].target) != 0) return false; + } + return true; +} + +/* Writes the executable's tree below `rootfd` (an empty directory). */ +static bool ni_extract(int rootfd) { + for (size_t i = 0; i < ni_nfiles; i++) { + const unsigned char *data = ni_modules + ni_files[i].offset; + if (!ni_digest_matches(data, ni_files[i].size, ni_files[i].sha256)) return false; + const char *base; + int dirfd = ni_parent_at(rootfd, ni_files[i].path, true, &base); + if (dirfd < 0) return false; + int fd = openat(dirfd, base, O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW | O_CLOEXEC, 0600); + close(dirfd); + if (fd < 0) return false; + size_t done = 0; + while (done < ni_files[i].size) { + ssize_t n = write(fd, data + done, ni_files[i].size - done); + if (n < 0 && errno == EINTR) continue; + if (n <= 0) break; + done += (size_t)n; + } + if (close(fd) != 0 || done != ni_files[i].size) return false; + } + for (size_t i = 0; i < ni_nlinks; i++) { + const char *base; + int dirfd = ni_parent_at(rootfd, ni_links[i].path, true, &base); + if (dirfd < 0) return false; + int status = symlinkat(ni_links[i].target, dirfd, base); + close(dirfd); + if (status != 0) return false; + } + return true; +} + +/* Removes everything below `dirfd` and closes it, never following links. */ +static void ni_remove_below(int dirfd) { + DIR *dir = fdopendir(dirfd); + if (dir == NULL) { + close(dirfd); + return; + } + for (struct dirent *e; (e = readdir(dir)) != NULL;) { + if (strcmp(e->d_name, ".") == 0 || strcmp(e->d_name, "..") == 0) continue; + int sub = openat(dirfd, e->d_name, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); + if (sub >= 0) { + ni_remove_below(sub); + (void)unlinkat(dirfd, e->d_name, AT_REMOVEDIR); + } else { + (void)unlinkat(dirfd, e->d_name, 0); + } + } + closedir(dir); +} + + +static char ni_temp_root[4096] = ""; + +static void ni_remove_temp_root(void) { + int fd = open(ni_temp_root, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); + if (fd >= 0) ni_remove_below(fd); + (void)rmdir(ni_temp_root); +} + +static bool ni_set_root(const char *a, const char *b, const char *c) { + int n = snprintf(ni_root, sizeof ni_root, "%s%s%s", a, b, c); + return n > 0 && (size_t)n < sizeof ni_root; +} + +/* The tree in the cache directory, extracted there when absent. */ +static bool ni_cached_root(const char *digest) { + const char *xdg = getenv("XDG_CACHE_HOME"); + const char *home = getenv("HOME"); + char base[4096]; + int n; + if (xdg != NULL && xdg[0] == '/') + n = snprintf(base, sizeof base, "%s", xdg); + else if (home != NULL && home[0] == '/') + n = snprintf(base, sizeof base, "%s/.cache", home); + else + return false; + if (n <= 0 || (size_t)n >= sizeof base) return false; + (void)mkdir(base, 0700); + int basefd = open(base, O_RDONLY | O_DIRECTORY | O_CLOEXEC); + if (basefd < 0) return false; + int scriptc = ni_dir_at(basefd, "scriptc", true); + close(basefd); + int islands = ni_private_dir(scriptc) ? ni_dir_at(scriptc, "node-islands", true) : -1; + if (scriptc >= 0) close(scriptc); + if (!ni_private_dir(islands)) { + if (islands >= 0) close(islands); + return false; + } + bool ok = false; + int tree = ni_dir_at(islands, digest, false); + if (tree < 0 && errno == ENOENT) { + /* Extract into a fresh sibling, then move it into place. */ + char staging[128]; + snprintf(staging, sizeof staging, "%s.%ld.tmp", digest, (long)getpid()); + int fd = ni_dir_at(islands, staging, true); + if (ni_private_dir(fd) && ni_extract(fd) && ni_tree_matches(fd) && + renameat(islands, staging, islands, digest) == 0) { + close(fd); + tree = ni_dir_at(islands, digest, false); + } else { + if (fd >= 0) ni_remove_below(fd); + (void)unlinkat(islands, staging, AT_REMOVEDIR); + tree = ni_dir_at(islands, digest, false); /* another process may have placed it */ + } + } + if (ni_private_dir(tree) && ni_tree_matches(tree)) + ok = ni_set_root(base, "/scriptc/node-islands/", digest); + if (tree >= 0) close(tree); + close(islands); + return ok; +} + +/* The tree in a private directory of this run, removed at exit. */ +static bool ni_temporary_root(void) { + const char *tmp = getenv("TMPDIR"); + if (tmp == NULL || tmp[0] != '/') tmp = "/tmp"; + int n = snprintf(ni_temp_root, sizeof ni_temp_root, "%s/scriptc-node-islands-XXXXXX", tmp); + if (n <= 0 || (size_t)n >= sizeof ni_temp_root || mkdtemp(ni_temp_root) == NULL) return false; + scr_atexit(ni_remove_temp_root); + int fd = open(ni_temp_root, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); + bool ok = ni_private_dir(fd) && ni_extract(fd) && ni_tree_matches(fd); + if (fd >= 0) close(fd); + return ok && ni_set_root(ni_temp_root, "", ""); +} + +/* Makes the tree available and sets ni_root; fatal when it cannot. */ +static void ni_open_modules(void) { + if (ni_manifest == NULL) return; /* only built-in modules */ + if (ni_secure()) + ni_fatal("scriptc: a program running with elevated privileges cannot load Node island " + "modules\n"); + ni_parse_manifest(); + if (ni_modules_location != NULL && ni_modules_location[0] != '\0') { + static const char origin[] = "$ORIGIN/"; + if (strncmp(ni_modules_location, origin, sizeof origin - 1) != 0) ni_bad_manifest(); + ScrStr *self = scr_process_exec_path(); + size_t len = self->len; + while (len > 0 && self->data[len - 1] != '/') len--; + bool absolute = len > 0 && self->data[0] == '/'; + char dir[4096]; + int n = snprintf(dir, sizeof dir, "%.*s", (int)len, self->data); + scr_str_release(self); + if (!absolute || n <= 0 || (size_t)n >= sizeof dir || + !ni_set_root(dir, ni_modules_location + sizeof origin - 1, "")) + ni_fatal("scriptc: cannot locate this program's Node island modules\n"); + int fd = open(ni_root, O_RDONLY | O_DIRECTORY | O_CLOEXEC); + bool ok = fd >= 0 && ni_tree_matches(fd); + if (fd >= 0) close(fd); + if (!ok) { + char message[8192]; + snprintf(message, sizeof message, + "scriptc: the Node island modules at %s do not match this program\n", ni_root); + ni_fatal(message); + } + return; + } + NiSha256 s; + unsigned char digest[32]; + ni_sha256_init(&s); + ni_sha256_update(&s, (const unsigned char *)ni_manifest, ni_manifest_len); + ni_sha256_final(&s, digest); + char hex[65]; + for (int i = 0; i < 32; i++) snprintf(hex + i * 2, 3, "%02x", digest[i]); + if (ni_modules == NULL && ni_nfiles > 0) ni_bad_manifest(); + if (!ni_cached_root(hex) && !ni_temporary_root()) + ni_fatal("scriptc: cannot write this program's Node island modules to a private directory\n"); +} + +/* The calling module's path inside the tree: `/`. */ +static char *ni_caller_path(const ScrStr *relative) { + size_t root = strlen(ni_root); + bool slash = root > 0 && ni_root[root - 1] == '/'; + char *path = malloc(root + 1 + relative->len + 1); + if (path == NULL) ni_oom(); + memcpy(path, ni_root, root); + if (!slash) path[root++] = '/'; + memcpy(path + root, relative->data, relative->len); + path[root + relative->len] = '\0'; + return path; +} + +/* The bootstrap: the tree root for the host's resolution guard, then the + * host script. +1 malloc'd. */ +static char *ni_bootstrap(void) { + size_t host = strlen(scr_node_island_host_js); + size_t cap = 64 + strlen(ni_root) * 6 + host; + char *out = malloc(cap); + if (out == NULL) ni_oom(); + size_t n = (size_t)snprintf(out, cap, "globalThis.__scriptcNodeIslandRoot = \""); + for (const char *p = ni_root; *p != '\0'; p++) { + unsigned char c = (unsigned char)*p; + if (c == '"' || c == '\\') + n += (size_t)snprintf(out + n, cap - n, "\\%c", c); + else if (c < 0x20) + n += (size_t)snprintf(out + n, cap - n, "\\u%04x", c); + else + out[n++] = (char)c; + } + n += (size_t)snprintf(out + n, cap - n, "\";\n"); + memcpy(out + n, scr_node_island_host_js, host + 1); + return out; +} + /* ── outcomes ─────────────────────────────────────────────────────────── */ /* The result of an "R" outcome: its JSON copy, or undefined. +1, or NULL @@ -215,7 +772,10 @@ static void ni_exit(int code) { ni_fatal(""); } -static const ScrNodeBridgeHost ni_host = {&ni_settle, &ni_exit}; +/* The start of Node's stack for callers on it (V8's stack-scan bound). */ +static uintptr_t ni_stack_start(void); + +static const ScrNodeBridgeHost ni_host = {&ni_settle, &ni_exit, &ni_stack_start}; /* ── the loop slot ────────────────────────────────────────────────────── */ @@ -226,7 +786,27 @@ static void ni_on_node_stack(void (*job)(void *), void *arg); static void ni_pump(void) { ni_on_node_stack(ni_pump_job, NULL); } static int ni_backend_fd(void) { return ni_bridge->backend_fd(); } static int ni_backend_timeout(void) { return ni_bridge->backend_timeout(); } -static void ni_stop_atexit(void) { ni_on_node_stack(ni_stop_job, NULL); } +typedef struct { + int code; +} NiFinish; +static void ni_finish_job(void *p); + +/* At exit, after the program's own exit work: Node's process ends with + * the program's code ('beforeExit', then 'exit'), and Node stops. A code + * Node's process changed (process.exitCode in Node) is the program's. */ +static void ni_stop_atexit(void) { + int code = (int)scr_process_exit_code_or_zero(); + NiFinish finish = {code}; + /* Node writes to the same stdout. */ + fflush(stdout); + ni_on_node_stack(ni_finish_job, &finish); + ni_on_node_stack(ni_stop_job, NULL); + if (finish.code != code) { + fflush(NULL); + if (ni_temp_root[0] != '\0') ni_remove_temp_root(); + _Exit(finish.code & 0xff); + } +} /* ── Node's stack ─────────────────────────────────────────────────────── */ @@ -254,6 +834,13 @@ static void ni_trampoline(void) { } } +static uintptr_t ni_stack_start(void) { + uintptr_t sp = (uintptr_t)__builtin_frame_address(0); + return ni_stack != NULL && sp >= (uintptr_t)ni_stack && sp < (uintptr_t)ni_stack + NI_STACK + ? (uintptr_t)ni_stack + NI_STACK + : 0; +} + /* Runs job(arg) on Node's stack (directly when already there). */ static void ni_on_node_stack(void (*job)(void *), void *arg) { if (ni_on_stack) { @@ -329,11 +916,17 @@ static void ni_stop_job(void *p) { ni_bridge->stop(); } +static void ni_finish_job(void *p) { + NiFinish *a = p; + a->code = ni_bridge->finish(a->code); +} + /* ── loading Node ─────────────────────────────────────────────────────── */ static void ni_start(void) { if (ni_bridge != NULL) return; const char *dir = ni_pack_dir(); + ni_open_modules(); char path[4352]; int written = snprintf(path, sizeof path, "%s/%s", dir, NI_BRIDGE_FILE); if (written < 0 || (size_t)written >= sizeof path) @@ -374,8 +967,10 @@ static void ni_start(void) { /* Flush first: Node writes to the same stdout. */ fflush(stdout); ni_bridge = bridge; - NiStart start = {scr_node_island_host_js, argv0, NULL}; + char *bootstrap = ni_bootstrap(); + NiStart start = {bootstrap, argv0, NULL}; ni_on_node_stack(ni_start_job, &start); + free(bootstrap); char *error = start.error; if (error != NULL) { char message[8192]; @@ -401,17 +996,11 @@ static ScrDyn *ni_args_json(ScrDyn *args) { } /* A borrowed ScrStr as a NUL-terminated copy for the bridge. */ -static char *ni_cstr(const ScrStr *s) { - char *copy = malloc(s->len + 1); - if (copy == NULL) ni_oom(); - memcpy(copy, s->data, s->len); - copy[s->len] = '\0'; - return copy; -} +static char *ni_cstr(const ScrStr *s) { return ni_cstr_of(s->data, s->len); } static char *ni_call(uint32_t id, bool sync, ScrStr *file, ScrStr *specifier, ScrStr *name, ScrDyn *json, size_t *len) { - char *f = ni_cstr(file), *s = ni_cstr(specifier), *n = ni_cstr(name); + char *f = ni_caller_path(file), *s = ni_cstr(specifier), *n = ni_cstr(name); NiCall call = {id, sync ? 1 : 0, f, s, n, json->v.str->data, json->v.str->len, 0, NULL}; ni_on_node_stack(ni_call_job, &call); char *outcome = call.outcome; @@ -491,6 +1080,16 @@ void scr_node_island_install(const char *libnode, const char *node_version) { (void)node_version; } +void scr_node_island_install_modules(const char *manifest, size_t manifest_len, + const unsigned char *data, size_t data_len, + const char *location) { + (void)manifest; + (void)manifest_len; + (void)data; + (void)data_len; + (void)location; +} + SCR_COLD static void ni_unsupported(void) { fputs("scriptc: Node islands are not supported on this target\n", stderr); fflush(stderr); diff --git a/packages/runtime/src/scr_node_island.h b/packages/runtime/src/scr_node_island.h index 9904047f5..33d1d17b4 100644 --- a/packages/runtime/src/scr_node_island.h +++ b/packages/runtime/src/scr_node_island.h @@ -14,6 +14,15 @@ * before %main. */ void scr_node_island_install(const char *libnode, const char *node_version); +/* Records the island module tree (node-islands/modules.ts): its manifest, + * and its files concatenated in manifest order when the executable carries + * them (NULL when they ship beside it at `location`, a `$ORIGIN/` path; "" + * otherwise). All static. The emitted main calls it after + * scr_node_island_install. */ +void scr_node_island_install_modules(const char *manifest, size_t manifest_len, + const unsigned char *data, size_t data_len, + const char *location); + /* @scriptc/node callSync(specifier, name, ...args): `file` is the calling * module's path (relative specifiers resolve from it), `args` the boxed * argument array, copied as JSON. Returns the JSON copy of the result, +1, diff --git a/packages/runtime/src/scr_node_island_host.cjs b/packages/runtime/src/scr_node_island_host.cjs index d98b1c3ad..90c6187ab 100644 --- a/packages/runtime/src/scr_node_island_host.cjs +++ b/packages/runtime/src/scr_node_island_host.cjs @@ -13,10 +13,36 @@ * them in-process under Node: both must produce the same results, errors * and messages. Outcomes are "R" (returned; "R" alone is undefined) * or "T" (threw), the encoding of scr_node_bridge.h. */ -const { createRequire } = require("node:module"); +const { createRequire, registerHooks } = require("node:module"); const nativeHost = globalThis.__scriptcNodeIslandHost; +// The program's island module tree (scr_node_island.c prefixes this script +// with its root, then the bridge removes the global): modules resolve only +// to files inside it, never to a file elsewhere on the machine. +const islandRoot = globalThis.__scriptcNodeIslandRoot; +delete globalThis.__scriptcNodeIslandRoot; +if (typeof islandRoot === "string" && islandRoot !== "/") { + const { realpathSync } = require("node:fs"); + const { sep } = require("node:path"); + const { fileURLToPath } = require("node:url"); + const root = realpathSync(islandRoot); + const inside = root.endsWith(sep) ? root : root + sep; + registerHooks({ + resolve(specifier, context, nextResolve) { + const result = nextResolve(specifier, context); + if (result.url.startsWith("file:") && !fileURLToPath(result.url).startsWith(inside)) { + const error = new Error( + `Cannot find module '${specifier}': it is not one of the modules this program ships`, + ); + error.code = "MODULE_NOT_FOUND"; + throw error; + } + return result; + }, + }); +} + // Error classes that cross as copies, at their scr_error.h kind index. const ERROR_KINDS = [ [TypeError, 1], @@ -135,4 +161,28 @@ for (const stream of [process.stdout, process.stderr]) { stream._handle.setBlocking(true); } -globalThis.__scriptcNodeIsland = { call }; +// The program ends with `code` (the bridge then runs Node's loop to its +// end, which emits 'beforeExit' and 'exit'): Node's process shares it, as +// it shares one process under Node. +function finish(code) { + if (code !== 0) process.exitCode = code; +} + +// process.execPath is this program, not a node executable: a fork without +// an execPath of its own would start the program again. +const childProcess = require("node:child_process"); +const nodeFork = childProcess.fork; +childProcess.fork = function fork(modulePath, args, options) { + const forkOptions = Array.isArray(args) ? options : args; + const execPath = + forkOptions === null || typeof forkOptions !== "object" ? undefined : forkOptions.execPath; + if (execPath === undefined || execPath === process.execPath) { + throw new Error( + "child_process.fork() in a Node island needs options.execPath (a node executable): process.execPath is this program", + ); + } + return Reflect.apply(nodeFork, this, arguments); +}; +require("node:module").syncBuiltinESMExports(); + +globalThis.__scriptcNodeIsland = { call, finish }; diff --git a/packages/runtime/src/scr_node_island_host.h b/packages/runtime/src/scr_node_island_host.h index 5d2286450..ed78c8951 100644 --- a/packages/runtime/src/scr_node_island_host.h +++ b/packages/runtime/src/scr_node_island_host.h @@ -19,10 +19,36 @@ static const char scr_node_island_host_js[] = " * them in-process under Node: both must produce the same results, errors\012" " * and messages. Outcomes are \"R\" (returned; \"R\" alone is undefined)\012" " * or \"T\" (threw), the encoding of scr_node_bridge.h. */\012" - "const { createRequire } = require(\"node:module\");\012" + "const { createRequire, registerHooks } = require(\"node:module\");\012" "\012" "const nativeHost = globalThis.__scriptcNodeIslandHost;\012" "\012" + "// The program's island module tree (scr_node_island.c prefixes this script\012" + "// with its root, then the bridge removes the global): modules resolve only\012" + "// to files inside it, never to a file elsewhere on the machine.\012" + "const islandRoot = globalThis.__scriptcNodeIslandRoot;\012" + "delete globalThis.__scriptcNodeIslandRoot;\012" + "if (typeof islandRoot === \"string\" && islandRoot !== \"/\") {\012" + " const { realpathSync } = require(\"node:fs\");\012" + " const { sep } = require(\"node:path\");\012" + " const { fileURLToPath } = require(\"node:url\");\012" + " const root = realpathSync(islandRoot);\012" + " const inside = root.endsWith(sep) \077 root : root + sep;\012" + " registerHooks({\012" + " resolve(specifier, context, nextResolve) {\012" + " const result = nextResolve(specifier, context);\012" + " if (result.url.startsWith(\"file:\") && !fileURLToPath(result.url).startsWith(inside)) {\012" + " const error = new Error(\012" + " `Cannot find module '${specifier}': it is not one of the modules this program ships`,\012" + " );\012" + " error.code = \"MODULE_NOT_FOUND\";\012" + " throw error;\012" + " }\012" + " return result;\012" + " },\012" + " });\012" + "}\012" + "\012" "// Error classes that cross as copies, at their scr_error.h kind index.\012" "const ERROR_KINDS = [\012" " [TypeError, 1],\012" @@ -141,6 +167,30 @@ static const char scr_node_island_host_js[] = " stream._handle.setBlocking(true);\012" "}\012" "\012" - "globalThis.__scriptcNodeIsland = { call };\012"; + "// The program ends with `code` (the bridge then runs Node's loop to its\012" + "// end, which emits 'beforeExit' and 'exit'): Node's process shares it, as\012" + "// it shares one process under Node.\012" + "function finish(code) {\012" + " if (code !== 0) process.exitCode = code;\012" + "}\012" + "\012" + "// process.execPath is this program, not a node executable: a fork without\012" + "// an execPath of its own would start the program again.\012" + "const childProcess = require(\"node:child_process\");\012" + "const nodeFork = childProcess.fork;\012" + "childProcess.fork = function fork(modulePath, args, options) {\012" + " const forkOptions = Array.isArray(args) \077 options : args;\012" + " const execPath =\012" + " forkOptions === null || typeof forkOptions !== \"object\" \077 undefined : forkOptions.execPath;\012" + " if (execPath === undefined || execPath === process.execPath) {\012" + " throw new Error(\012" + " \"child_process.fork() in a Node island needs options.execPath (a node executable): process.execPath is this program\",\012" + " );\012" + " }\012" + " return Reflect.apply(nodeFork, this, arguments);\012" + "};\012" + "require(\"node:module\").syncBuiltinESMExports();\012" + "\012" + "globalThis.__scriptcNodeIsland = { call, finish };\012"; #endif /* SCR_NODE_ISLAND_HOST_H */ diff --git a/tests/corpus/7661-node-call-modules/lib/entry.cjs b/tests/corpus/7661-node-call-modules/lib/entry.cjs new file mode 100644 index 000000000..3fa55af87 --- /dev/null +++ b/tests/corpus/7661-node-call-modules/lib/entry.cjs @@ -0,0 +1,14 @@ +"use strict"; +const { label } = require("./util"); +const settings = require("../settings.json"); + +exports.describe = (name) => `${label(name)} (${settings.mode})`; + +exports.hasOptional = () => { + try { + require("./not-shipped.cjs"); + return true; + } catch (error) { + return error.code !== "MODULE_NOT_FOUND"; + } +}; diff --git a/tests/corpus/7661-node-call-modules/lib/esm.mjs b/tests/corpus/7661-node-call-modules/lib/esm.mjs new file mode 100644 index 000000000..467987471 --- /dev/null +++ b/tests/corpus/7661-node-call-modules/lib/esm.mjs @@ -0,0 +1,5 @@ +import { factor } from "./factor.mjs"; + +export function twice(n) { + return n * factor; +} diff --git a/tests/corpus/7661-node-call-modules/lib/factor.mjs b/tests/corpus/7661-node-call-modules/lib/factor.mjs new file mode 100644 index 000000000..adb2e40cb --- /dev/null +++ b/tests/corpus/7661-node-call-modules/lib/factor.mjs @@ -0,0 +1 @@ +export const factor = 2; diff --git a/tests/corpus/7661-node-call-modules/lib/util.js b/tests/corpus/7661-node-call-modules/lib/util.js new file mode 100644 index 000000000..ea7711f79 --- /dev/null +++ b/tests/corpus/7661-node-call-modules/lib/util.js @@ -0,0 +1,5 @@ +// An ES module by the nearest package.json ("type": "module"), loaded with +// require() from a CommonJS module. +export function label(name) { + return `module of ${name}`; +} diff --git a/tests/corpus/7661-node-call-modules/main.ts b/tests/corpus/7661-node-call-modules/main.ts new file mode 100644 index 000000000..7f6f4ac76 --- /dev/null +++ b/tests/corpus/7661-node-call-modules/main.ts @@ -0,0 +1,12 @@ +// @requires-libnode +// The modules island calls load ship with the program: a module of the +// program and the modules it loads, and whole packages with their +// dependencies, including files a package reads or loads by computed names. +import { callSync } from "@scriptc/node"; + +console.log(callSync("./lib/entry.cjs", "describe", "scriptc")); +console.log(callSync("greeter", "greet", "world")); +console.log(callSync("greeter", "fromData")); +console.log(callSync("greeter", "byName", "shout", "computed")); +console.log(callSync("./lib/entry.cjs", "hasOptional")); +console.log(callSync("./lib/esm.mjs", "twice", 21)); diff --git a/tests/corpus/7661-node-call-modules/node_modules/greeter/data/message.txt b/tests/corpus/7661-node-call-modules/node_modules/greeter/data/message.txt new file mode 100644 index 000000000..8e4d1b0c2 --- /dev/null +++ b/tests/corpus/7661-node-call-modules/node_modules/greeter/data/message.txt @@ -0,0 +1 @@ +read from a file the package ships diff --git a/tests/corpus/7661-node-call-modules/node_modules/greeter/index.js b/tests/corpus/7661-node-call-modules/node_modules/greeter/index.js new file mode 100644 index 000000000..4e3f7bc55 --- /dev/null +++ b/tests/corpus/7661-node-call-modules/node_modules/greeter/index.js @@ -0,0 +1,8 @@ +"use strict"; +const fs = require("node:fs"); +const path = require("node:path"); +const shout = require("shout"); + +exports.greet = (who) => shout(`hello, ${who}`); +exports.fromData = () => fs.readFileSync(path.join(__dirname, "data", "message.txt"), "utf8").trim(); +exports.byName = (name, text) => require(name)(text); diff --git a/tests/corpus/7661-node-call-modules/node_modules/greeter/package.json b/tests/corpus/7661-node-call-modules/node_modules/greeter/package.json new file mode 100644 index 000000000..07b35f1af --- /dev/null +++ b/tests/corpus/7661-node-call-modules/node_modules/greeter/package.json @@ -0,0 +1 @@ +{ "name": "greeter", "version": "1.0.0", "main": "index.js", "dependencies": { "shout": "1.0.0" } } diff --git a/tests/corpus/7661-node-call-modules/node_modules/shout/index.js b/tests/corpus/7661-node-call-modules/node_modules/shout/index.js new file mode 100644 index 000000000..6401aa6da --- /dev/null +++ b/tests/corpus/7661-node-call-modules/node_modules/shout/index.js @@ -0,0 +1,2 @@ +"use strict"; +module.exports = (text) => `${text.toUpperCase()}!`; diff --git a/tests/corpus/7661-node-call-modules/node_modules/shout/package.json b/tests/corpus/7661-node-call-modules/node_modules/shout/package.json new file mode 100644 index 000000000..d32731e21 --- /dev/null +++ b/tests/corpus/7661-node-call-modules/node_modules/shout/package.json @@ -0,0 +1 @@ +{ "name": "shout", "version": "1.0.0", "main": "index.js" } diff --git a/tests/corpus/7661-node-call-modules/settings.json b/tests/corpus/7661-node-call-modules/settings.json new file mode 100644 index 000000000..42201c528 --- /dev/null +++ b/tests/corpus/7661-node-call-modules/settings.json @@ -0,0 +1 @@ +{ "mode": "shipped" } diff --git a/tests/corpus/7662-node-call-exit-events/listeners.cjs b/tests/corpus/7662-node-call-exit-events/listeners.cjs new file mode 100644 index 000000000..c21ad324f --- /dev/null +++ b/tests/corpus/7662-node-call-exit-events/listeners.cjs @@ -0,0 +1,13 @@ +"use strict"; +let beforeExit = 0; + +exports.register = () => { + process.on("beforeExit", (code) => { + beforeExit++; + console.log("beforeExit", code, beforeExit); + if (beforeExit === 1) setTimeout(() => console.log("work from beforeExit"), 1); + }); + process.on("exit", (code) => console.log("exit", code)); + process.exitCode = 4; + return true; +}; diff --git a/tests/corpus/7662-node-call-exit-events/main.ts b/tests/corpus/7662-node-call-exit-events/main.ts new file mode 100644 index 000000000..909550588 --- /dev/null +++ b/tests/corpus/7662-node-call-exit-events/main.ts @@ -0,0 +1,8 @@ +// @requires-libnode +// The program and the Node.js it loads end as one process: Node's +// 'beforeExit' and 'exit' run when the program ends, work that 'beforeExit' +// schedules runs first, and process.exitCode set in Node is the program's. +import { callSync } from "@scriptc/node"; + +console.log("registered", callSync("./listeners.cjs", "register")); +console.log("main done"); diff --git a/tests/fixtures/node-island/gc-stress.ts b/tests/fixtures/node-island/gc-stress.ts new file mode 100644 index 000000000..8c71099d1 --- /dev/null +++ b/tests/fixtures/node-island/gc-stress.ts @@ -0,0 +1,24 @@ +// Full garbage collections in Node during island calls +// (tests/harness/node-island.test.ts): from the main stack, from deep native +// recursion, from async functions and from timers, with deep JavaScript +// recursion on Node's side. +import { call, callSync } from "@scriptc/node"; + +function nested(depth: number): number { + if (depth === 0) return callSync("./gc.cjs", "deep", 2000); + return nested(depth - 1) + 1; +} + +async function inFiber(n: number): Promise { + const later = await call("./gc.cjs", "later", n); + return later + callSync("./gc.cjs", "churn", 1); +} + +console.log("churn", callSync("./gc.cjs", "churn", 20)); +console.log("nested", nested(3000)); +console.log("deep", callSync("./gc.cjs", "deep", 8000)); +const results = await Promise.all([1, 2, 3, 4].map((n) => inFiber(n))); +console.log("fibers", results.join(",")); +setTimeout(() => { + console.log("timer", callSync("./gc.cjs", "churn", 5)); +}, 1); diff --git a/tests/fixtures/node-island/gc.cjs b/tests/fixtures/node-island/gc.cjs new file mode 100644 index 000000000..f8f5a32fa --- /dev/null +++ b/tests/fixtures/node-island/gc.cjs @@ -0,0 +1,41 @@ +"use strict"; +// Garbage-collection pressure inside Node island calls +// (tests/harness/node-island.test.ts): full collections while the call +// runs on Node's stack, at depth on both sides. +const v8 = require("node:v8"); +const vm = require("node:vm"); +v8.setFlagsFromString("--expose-gc"); +const gc = vm.runInNewContext("gc"); + +const kept = []; + +// Allocates `rounds` batches of objects that survive until the next batch, +// with a full collection after each. +exports.churn = (rounds) => { + let total = 0; + for (let r = 0; r < rounds; r++) { + const batch = []; + for (let i = 0; i < 100000; i++) batch.push({ i, s: `item ${i}`, tags: [i, i + 1] }); + kept[r % 4] = batch; + total += batch.length; + gc(); + } + return total; +}; + +// Recurses `depth` frames in JavaScript and collects at the bottom. +exports.deep = (depth) => { + if (depth === 0) { + gc(); + return 0; + } + const local = { depth, text: String(depth) }; + return 1 + exports.deep(depth - 1) + (local.depth > 0 ? 0 : 1); +}; + +// Collects after a timer, from Node's loop. +exports.later = async (n) => { + await new Promise((resolve) => setTimeout(resolve, 1)); + exports.churn(1); + return n; +}; diff --git a/tests/fixtures/node-island/helper.cjs b/tests/fixtures/node-island/helper.cjs index 2268d324b..92c4ac77b 100644 --- a/tests/fixtures/node-island/helper.cjs +++ b/tests/fixtures/node-island/helper.cjs @@ -11,3 +11,13 @@ exports.unreadable = () => { }); throw error; }; +// A fork without an execPath of its own: in a program it would start the +// program again. +exports.forkSelf = () => { + try { + require("node:child_process").fork(__filename); + return "forked"; + } catch (error) { + return error.message; + } +}; diff --git a/tests/fixtures/node-island/lifecycle.ts b/tests/fixtures/node-island/lifecycle.ts index 7bac91a40..262ab7a6b 100644 --- a/tests/fixtures/node-island/lifecycle.ts +++ b/tests/fixtures/node-island/lifecycle.ts @@ -16,6 +16,7 @@ if (mode !== "never") { console.log("mapped after", mapped()); } if (mode === "exit") process.exit(5); +if (mode === "fork") console.log(callSync("./helper.cjs", "forkSelf")); if (mode === "unreadable") { try { callSync("./helper.cjs", "unreadable"); diff --git a/tests/harness/ci-stack-scope.test.ts b/tests/harness/ci-stack-scope.test.ts index 4abd321d0..9408433c4 100644 --- a/tests/harness/ci-stack-scope.test.ts +++ b/tests/harness/ci-stack-scope.test.ts @@ -62,6 +62,22 @@ test.each([ ["bottom layer", "pull_request", stacked("main", 1), true], ["new bottom after a merge", "pull_request", stacked("main", 2), true], ["upper layer", "pull_request", upper, false], + [ + "a PR into the default branch whose stack names another base", + "pull_request", + { + ...stacked("main"), + repository: { full_name: repository, default_branch: "main" }, + pull_request: { ...stacked("main").pull_request, stack: { base: { ref: "ni/base" } } }, + }, + true, + ], + [ + "upper layer with the default branch known", + "pull_request", + { ...upper, repository: { full_name: repository, default_branch: "main" } }, + false, + ], ["main push", "push", {}, true], ["push with stack metadata", "push", upper, true], ["merge queue group", "merge_group", upper, true], diff --git a/tests/harness/corpus-costs/7661-node-call-modules/main.ts.json b/tests/harness/corpus-costs/7661-node-call-modules/main.ts.json new file mode 100644 index 000000000..a970c2f16 --- /dev/null +++ b/tests/harness/corpus-costs/7661-node-call-modules/main.ts.json @@ -0,0 +1 @@ +{ "plain": 6000, "san": 9000 } diff --git a/tests/harness/corpus-costs/7662-node-call-exit-events/main.ts.json b/tests/harness/corpus-costs/7662-node-call-exit-events/main.ts.json new file mode 100644 index 000000000..a970c2f16 --- /dev/null +++ b/tests/harness/corpus-costs/7662-node-call-exit-events/main.ts.json @@ -0,0 +1 @@ +{ "plain": 6000, "san": 9000 } diff --git a/tests/harness/node-island.test.ts b/tests/harness/node-island.test.ts index b67f0c84b..132af41f0 100644 --- a/tests/harness/node-island.test.ts +++ b/tests/harness/node-island.test.ts @@ -6,7 +6,18 @@ * (6800-6803) pins the call semantics against Node. Needs a libnode pack * in SCRIPTC_LIBNODE; required in the Linux x64 CI lanes (libnode.ts). */ import { execFile } from "node:child_process"; -import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { + chmodSync, + copyFileSync, + cpSync, + existsSync, + mkdirSync, + readFileSync, + readdirSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; import { basename, join } from "node:path"; import { promisify } from "node:util"; import { beforeAll, describe, expect, test } from "vitest"; @@ -36,10 +47,13 @@ let binary = ""; const before = `mapped before ${process.platform === "linux" ? "false" : "unknown"}`; const after = `mapped after ${process.platform === "linux" ? "true" : "unknown"}`; +// The island module cache of these runs (scr_node_island.c). +const cacheHome = join(outDir, "cache"); + async function run( path: string, args: string[], - env: NodeJS.ProcessEnv = process.env, + env: NodeJS.ProcessEnv = { ...process.env, XDG_CACHE_HOME: cacheHome }, ): Promise { try { const { stdout, stderr } = await execFileAsync(path, args, { env, encoding: "utf8" }); @@ -125,18 +139,57 @@ describe.skipIf(skip)("Node islands in-process", () => { expect(result.stdout).toBe(`${before}\n42\n${after}\n`); }); - test("beside packaging carries its own pack", async () => { + test("beside packaging carries its own pack and island modules", async () => { const beside = await build("beside", "beside"); expect(existsSync(join(outDir, "beside/lifecycle.libnode/manifest.json"))).toBe(true); - const env = { ...process.env }; + const helper = join(outDir, "beside/lifecycle.libnode/modules/helper.cjs"); + expect(readFileSync(helper, "utf8")).toBe( + readFileSync(join(repoRoot, "tests/fixtures/node-island/helper.cjs"), "utf8"), + ); + const env = { ...process.env, XDG_CACHE_HOME: cacheHome }; delete env["SCRIPTC_LIBNODE"]; const result = await run(beside, [], env); expect(result.stdout).toBe(`${before}\n42\n${after}\ndone\n`); expect(result.code).toBe(0); + // Modules that no longer match the executable are refused. + writeFileSync(helper, `${readFileSync(helper, "utf8")}\n// edited\n`); + const edited = await run(beside, [], env); + expect(edited.code).toBe(1); + expect(edited.stderr).toBe( + `scriptc: the Node island modules at ${join(outDir, "beside/lifecycle.libnode/modules")} do not match this program\n`, + ); }, 300_000); + test("the executable carries its island modules and no path of the build machine", async () => { + const bytes = readFileSync(binary); + expect(bytes.includes(Buffer.from(repoRoot))).toBe(false); + // A copy elsewhere runs with the modules it carries, extracted into a + // private cache directory named by the manifest's digest. + const moved = join(outDir, "moved"); + mkdirSync(moved, { recursive: true }); + copyFileSync(binary, join(moved, "app")); + chmodSync(join(moved, "app"), 0o755); + const cache = join(outDir, "moved-cache"); + rmSync(cache, { recursive: true, force: true }); + const env = { ...process.env, XDG_CACHE_HOME: cache }; + expect((await run(join(moved, "app"), [], env)).stdout).toBe(`${before}\n42\n${after}\ndone\n`); + const islands = join(cache, "scriptc/node-islands"); + const [digest] = readdirSync(islands); + expect(digest).toMatch(/^[0-9a-f]{64}$/); + expect(statSync(islands).mode & 0o777).toBe(0o700); + expect(statSync(join(islands, digest!)).mode & 0o777).toBe(0o700); + // An entry that no longer matches is not used: the run extracts the + // modules into a directory of its own instead. + writeFileSync(join(islands, digest!, "helper.cjs"), "exports.answer = () => 0;\n"); + expect((await run(join(moved, "app"), [], env)).stdout).toBe(`${before}\n42\n${after}\ndone\n`); + }); + test("a missing pack fails the first call", async () => { - const result = await run(binary, [], { ...process.env, SCRIPTC_LIBNODE: "/nonexistent/pack" }); + const result = await run(binary, [], { + ...process.env, + XDG_CACHE_HOME: cacheHome, + SCRIPTC_LIBNODE: "/nonexistent/pack", + }); expect(result.code).toBe(1); expect(result.stdout).toBe(`${before}\n`); expect(result.stderr).toMatch( @@ -145,7 +198,11 @@ describe.skipIf(skip)("Node islands in-process", () => { }); test("SCRIPTC_LIBNODE must be an absolute path", async () => { - const result = await run(binary, [], { ...process.env, SCRIPTC_LIBNODE: "relative/pack" }); + const result = await run(binary, [], { + ...process.env, + XDG_CACHE_HOME: cacheHome, + SCRIPTC_LIBNODE: "relative/pack", + }); expect(result.code).toBe(1); expect(result.stdout).toBe(`${before}\n`); expect(result.stderr).toBe( @@ -175,9 +232,54 @@ describe.skipIf(skip)("Node islands in-process", () => { }); }, 300_000); + test("full garbage collections during calls, at depth on both sides", async () => { + const stress = await build( + "gc-stress", + undefined, + join(repoRoot, "tests/fixtures/node-island/gc-stress.ts"), + ); + expect(await run(stress, [])).toEqual({ + stdout: + "churn 2000000\nnested 5000\ndeep 8000\nfibers 100001,100002,100003,100004\ntimer 500000\n", + stderr: "", + code: 0, + }); + }, 300_000); + + test("Node ignores NODE_OPTIONS from the program's environment", async () => { + const result = await run(binary, [], { + ...process.env, + XDG_CACHE_HOME: cacheHome, + NODE_OPTIONS: "--require /nonexistent/preload.cjs", + }); + expect(result).toEqual({ stdout: `${before}\n42\n${after}\ndone\n`, stderr: "", code: 0 }); + }); + + test("child_process.fork() needs an execPath of its own", async () => { + expect((await run(binary, ["fork"])).stdout).toBe( + `${before}\n42\n${after}\nchild_process.fork() in a Node island needs options.execPath (a node executable): process.execPath is this program\ndone\n`, + ); + }); + + test("a cached build follows a change of the recorded libnode pack", async () => { + // The same program built against a copy of the pack elsewhere records + // the copy: the executable cache does not serve the first build's entry. + const copy = join(outDir, "pack-copy"); + rmSync(copy, { recursive: true, force: true }); + cpSync(pack!, copy, { recursive: true }); + const first = readFileSync(await build("cached-pack", pack!)); + const second = readFileSync(await build("cached-pack", copy)); + expect(first.includes(Buffer.from(copy))).toBe(false); + expect(second.includes(Buffer.from(copy))).toBe(true); + }, 300_000); + test("a pack of another Node version fails the first call", async () => { const dir = await fakePack("old-node", "SCR_NODE_BRIDGE_ABI", "22.11.0"); - const result = await run(binary, [], { ...process.env, SCRIPTC_LIBNODE: dir }); + const result = await run(binary, [], { + ...process.env, + XDG_CACHE_HOME: cacheHome, + SCRIPTC_LIBNODE: dir, + }); expect(result.code).toBe(1); expect(result.stderr).toMatch( new RegExp( @@ -188,10 +290,14 @@ describe.skipIf(skip)("Node islands in-process", () => { test("a bridge of another ABI fails the first call", async () => { const dir = await fakePack("other-abi", "999u", "24.22.0"); - const result = await run(binary, [], { ...process.env, SCRIPTC_LIBNODE: dir }); + const result = await run(binary, [], { + ...process.env, + XDG_CACHE_HOME: cacheHome, + SCRIPTC_LIBNODE: dir, + }); expect(result.code).toBe(1); expect(result.stderr).toBe( - `scriptc: the libnode pack at ${dir} has an incompatible bridge (ABI 999; this program needs 2)\n`, + `scriptc: the libnode pack at ${dir} has an incompatible bridge (ABI 999; this program needs 3)\n`, ); }); From 0af86e2e3ef7862ca6d91b8917ff6ccf36735fd1 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 12:58:05 -0500 Subject: [PATCH 49/53] Version two-counter loops over typed arrays A loop with two counters closing in on each other, such as `for (let i = lo, j = hi; i < j; i++, j--)`, checked the bounds of every typed-array element it read or wrote at either counter. Within the body the low counter only rises and stays below the high one, which only falls, so every such index lies between the two starts. Typed-array loop versions now cover these loops. The guard tests that both starts are exact integers, that the low start is at least 0, and that the high start is below the length of each view the loop indexes at a counter. The fast copy then reads and writes those elements without checks, and the checked copy runs whenever the guard fails. In the fast copy the guard also stands for the loop's own start test. The exact-integer test of a start is now one function, shared by the typed-array version guards and the dense array two-counter guard. The dense guard, which compared the round trip with `fcmp oeq`, now also leaves a -0 start to the checked loop. --- .../backend/llvm/bytes-loop-versions.test.ts | 77 ++++++++++++++- .../src/backend/llvm/bytes-loop-versions.ts | 97 ++++++++++++------- .../src/backend/llvm/counted-loops.ts | 33 +++++++ .../src/backend/llvm/dense-array-loops.ts | 31 ++---- .../src/backend/llvm/emit-statements.ts | 11 ++- packages/compiler/src/backend/llvm/emitter.ts | 30 +++--- .../7981-typed-array-paired-loops.ts.json | 6 ++ tests/corpus/7981-typed-array-paired-loops.ts | 57 +++++++++++ .../7981-typed-array-paired-loops.ts.json | 1 + 9 files changed, 265 insertions(+), 78 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7981-typed-array-paired-loops.ts.json create mode 100644 tests/corpus/7981-typed-array-paired-loops.ts create mode 100644 tests/harness/corpus-costs/7981-typed-array-paired-loops.ts.json diff --git a/packages/compiler/src/backend/llvm/bytes-loop-versions.test.ts b/packages/compiler/src/backend/llvm/bytes-loop-versions.test.ts index 34b75861c..de71a6ef0 100644 --- a/packages/compiler/src/backend/llvm/bytes-loop-versions.test.ts +++ b/packages/compiler/src/backend/llvm/bytes-loop-versions.test.ts @@ -11,7 +11,7 @@ import { type IrType, } from "../../ir/ir.js"; import { analyzeIntegerRanges } from "../../ir/integer-ranges.js"; -import { matchIntegerCountedForLoop } from "../../ir/integer-loops.js"; +import { matchIntegerCountedForLoop, matchIntegerPairedForLoop } from "../../ir/integer-loops.js"; import { findInitializedByteLoopBindings } from "./byte-windows.js"; import { matchBytesLoopVersion, type BytesLoopVersion } from "./bytes-loop-versions.js"; import { emitLlvmModule } from "./emitter.js"; @@ -623,3 +623,78 @@ test("while loop counters below a string's length version its reads", () => { expect(version.bounded.map((c) => c.slack)).toEqual([0]); expect(version.fixed).toEqual([]); }); + +test("a two-counter loop versions the typed arrays it indexes at either counter", () => { + // for (let i = 0, j = k; i < j; i++, j--) { const t = out[i]; out[i] = out[j]; out[j] = t; } + const loop: IrStmt & { kind: "for" } = { + kind: "for", + init: { + kind: "block", + body: [ + { kind: "varDecl", localId: "i", init: num(0), loc }, + { kind: "varDecl", localId: "j", init: ref("k"), loc }, + ], + loc, + }, + cond: { kind: "bin", op: "<", left: ref("i"), right: ref("j"), type: BOOL, loc }, + update: { + kind: "block", + body: [ + { kind: "assign", localId: "i", value: bin("+", ref("i"), num(1)), loc }, + { kind: "assign", localId: "j", value: bin("-", ref("j"), num(1)), loc }, + ], + loc, + }, + body: [ + { kind: "varDecl", localId: "t", init: get(ref("out", f64s), ref("i")), loc }, + set(ref("out", f64s), ref("i"), get(ref("out", f64s), ref("j"))), + set(ref("out", f64s), ref("j"), ref("t")), + // Another index stays checked. + set(ref("src", f64s), bin("+", ref("i"), num(1)), num(0)), + ], + loc, + }; + const kernel = fn([loop], [local("t", F64)]); + const ranges = analyzeIntegerRanges(kernel); + const locals = new Map(kernel.locals.map((l) => [l.id, l])); + const paired = matchIntegerPairedForLoop(loop, locals, ranges)!; + expect(paired).not.toBeNull(); + const version = matchBytesLoopVersion( + loop, + null, + { + locals, + invariantGlobals: new Set(), + captured: new Set(), + ranges, + initialized: findInitializedByteLoopBindings(kernel).get(loop) ?? new Set(), + proven: new Set(), + workers: false, + owned: new Set(), + scalarStrings: new Set(), + }, + paired, + )!; + expect(version.paired?.loop).toBe(paired); + expect(version.paired?.receivers.map((r) => version.receivers[r]!.ref)).toEqual([ + ref("out", f64s), + ]); + expect(version.bounds.size).toBe(4); + const mod: IrModule = { + irVersion: 17, + sourceFile: loc.file, + entry: "__main", + functions: [ + { name: "__main", params: [], returnType: VOID, locals: [], body: [], loc }, + kernel, + ], + }; + const body = /^define internal [^\n]*@sc_bf_kernel[^]*?^}/m.exec(emitLlvmModule(mod))![0]; + const guard = body.slice(0, body.search(/^bytes\.version\.(fast|slow)\d+:$/m)); + expect(guard).toMatch(/icmp sge i64 %\S+, 0/); + expect(guard).toMatch(/icmp slt i64 %\S+, %\S+/); + const fast = /^bytes\.version\.fast\d+:$[^]*?^bytes\.version\.done\d+:$/m.exec(body)![0]; + // The fast copy keeps only the `src[i + 1]` check, and no second guard. + expect(fast.match(/bytes\.index\.invalid\d+:/g)).toHaveLength(1); + expect(fast).not.toContain("counted.fast"); +}); diff --git a/packages/compiler/src/backend/llvm/bytes-loop-versions.ts b/packages/compiler/src/backend/llvm/bytes-loop-versions.ts index 704b6723a..2013d03e4 100644 --- a/packages/compiler/src/backend/llvm/bytes-loop-versions.ts +++ b/packages/compiler/src/backend/llvm/bytes-loop-versions.ts @@ -1,11 +1,15 @@ import { F64, type IrExpr, type IrLocal, type IrStmt } from "../../ir/ir.js"; import type { IntegerRange, IntegerRanges } from "../../ir/integer-ranges.js"; -import type { IntegerCountedForLoop } from "../../ir/integer-loops.js"; +import type { IntegerCountedForLoop, IntegerPairedForLoop } from "../../ir/integer-loops.js"; import type { LlvmEmitterContext } from "./expr-context.js"; import { byteNumberAccess } from "../../ir/byte-numbers.js"; import { everyExprChild, everyStmtChild } from "../../ir/traverse.js"; import { exactInteger, widenInteger } from "./integer-values.js"; -import { emitCountedLoopLimit, emitCountedLoopSafety } from "./counted-loops.js"; +import { + emitCountedLoopLimit, + emitCountedLoopSafety, + emitExactIntegerTest, +} from "./counted-loops.js"; import { emitBytesHeaderLength, emitBytesSharedStorage } from "./expr-bytes.js"; import { f64Lit } from "./common.js"; import { emitStringAsciiFlag, emitStringByteLength } from "./string-reads.js"; @@ -79,6 +83,10 @@ export interface BytesLoopVersion { guarded: ReadonlySet; /** The counted loop whose iteration space the affine checks cover. */ counted: IntegerCountedForLoop | null; + /** A two-counter loop (`for (let i = lo, j = hi; i < j; i++, j--)`) and + * the receivers it indexes at either counter. Both counters stay between + * the two starts, so the guard requires `lo >= 0` and `hi < length`. */ + paired: { loop: IntegerPairedForLoop; receivers: number[] } | null; /** The guard also requires every counter value within [-bound, bound]. */ counterBound: number | null; /** Integer ranges of affine index operands that hold in the unchecked @@ -153,6 +161,7 @@ export function matchBytesLoopVersion( loop: IrStmt & { kind: "for" | "while" }, counted: IntegerCountedForLoop | null, facts: BytesLoopFacts, + paired: IntegerPairedForLoop | null = null, ): BytesLoopVersion | null { const written = new Set(); const temps = new Map(); @@ -438,6 +447,9 @@ export function matchBytesLoopVersion( string, { check: BoundedCheck; nodes: { node: Node; access: WhileAccess }[] } >(); + const pairedCounters = + paired !== null && counted === null ? new Set([paired.low.localId, paired.high.localId]) : null; + const pairedNodes = new Map(); const access = (node: Node, array: IrExpr, index: IrExpr, width: number): void => { const ref = root(array); if (!ref) return; @@ -448,6 +460,20 @@ export function matchBytesLoopVersion( else owned.set(receiver, [node]); } if (facts.proven.has(node)) return; + // An element at either counter of a two-counter loop. + if ( + pairedCounters !== null && + width === 1 && + ref.type.kind === "bytes" && + index.kind === "varRef" && + pairedCounters.has(index.localId) + ) { + const receiver = receiverOf(ref); + const nodes = pairedNodes.get(receiver); + if (nodes) nodes.push(node); + else pairedNodes.set(receiver, [node]); + return; + } const shape = counter !== null ? affine(index) : null; if (shape && shape.counter !== 0) { const operands = new Set(); @@ -543,11 +569,16 @@ export function matchBytesLoopVersion( Math.max(0, MAX_CHECKS - fixedKept.length - affineKept.length), ); const ownedKept = [...owned.entries()].slice(0, MAX_CHECKS); + const pairedKept = [...pairedNodes.entries()].slice( + 0, + Math.max(0, MAX_CHECKS - fixedKept.length - affineKept.length - boundedKept.length), + ); if ( fixedKept.length === 0 && affineKept.length === 0 && boundedKept.length === 0 && - ownedKept.length === 0 + ownedKept.length === 0 && + pairedKept.length === 0 ) return null; const bounds = new Set(); @@ -616,6 +647,10 @@ export function matchBytesLoopVersion( if (access.index !== access.ref) narrow(access.index, { min: 0, max: SAFE }); } } + for (const [receiver, nodes] of pairedKept) { + used.add(receiver); + for (const node of nodes) bounds.add(node); + } const counterChecks: CounterCheck[] = []; for (const [id, min] of counterMins) { const info = whileAccesses!.counters.get(id)!; @@ -658,6 +693,10 @@ export function matchBytesLoopVersion( counters: counterChecks, guarded, counted: affineKept.length > 0 ? counted : null, + paired: + paired !== null && pairedKept.length > 0 + ? { loop: paired, receivers: pairedKept.map(([receiver]) => renumber.get(receiver)!) } + : null, counterBound: needsBound ? bound : null, ranges, }; @@ -1143,6 +1182,25 @@ export function emitBytesLoopVersionGuard( conditions.push(both); } } + if (version.paired !== null) { + // Both counters stay within [lo, hi] (exact integers below 2^53): every + // access at either one is in bounds when `lo >= 0` and `hi < length`. + const { loop } = version.paired; + const [low, high] = [loop.low.start, loop.high.start].map((expr) => { + const tested = emitExactIntegerTest(host, host.emitExpr(expr).name, -SAFE, SAFE); + conditions.push(tested.ok); + return tested.integer; + }); + const nonNegative = B.tmp(); + B.line(`${nonNegative} = icmp sge i64 ${low}, 0`); + conditions.push(nonNegative); + for (const receiver of version.paired.receivers) { + // Lengths are at most 2^53 elements: a signed comparison is exact. + const below = B.tmp(); + B.line(`${below} = icmp slt i64 ${high}, ${lengths[receiver]}`); + conditions.push(below); + } + } let valid = "true"; for (const cond of conditions) { if (valid === "true") valid = cond; @@ -1156,39 +1214,6 @@ export function emitBytesLoopVersionGuard( else B.condBr(valid, fast, slow); } -/** Tests that a double is an exact integer within [min, max] (not -0; NaN - * fails the ordered comparisons) and gives its i64. The integer is frozen, - * so it is a fixed value even when the test fails. */ -function emitExactIntegerTest( - host: LlvmEmitterContext, - value: string, - min: number, - max: number, -): { ok: string; integer: string } { - const B = host.B; - const low = B.tmp(), - high = B.tmp(), - inRange = B.tmp(), - raw = B.tmp(), - integer = B.tmp(), - back = B.tmp(), - bits = B.tmp(), - backBits = B.tmp(), - exact = B.tmp(), - ok = B.tmp(); - B.line(`${low} = fcmp oge double ${value}, ${f64Lit(min)}`); - B.line(`${high} = fcmp ole double ${value}, ${f64Lit(max)}`); - B.line(`${inRange} = and i1 ${low}, ${high}`); - B.line(`${raw} = fptosi double ${value} to i64`); - B.line(`${integer} = freeze i64 ${raw}`); - B.line(`${back} = sitofp i64 ${integer} to double`); - B.line(`${bits} = bitcast double ${value} to i64`); - B.line(`${backBits} = bitcast double ${back} to i64`); - B.line(`${exact} = icmp eq i64 ${bits}, ${backBits}`); - B.line(`${ok} = and i1 ${inRange}, ${exact}`); - return { ok, integer }; -} - function scaleCounter(host: LlvmEmitterContext, value: string, scale: number): string { if (scale === 1) return value; const scaled = host.B.tmp(); diff --git a/packages/compiler/src/backend/llvm/counted-loops.ts b/packages/compiler/src/backend/llvm/counted-loops.ts index ca2dd6b92..1ff4c6580 100644 --- a/packages/compiler/src/backend/llvm/counted-loops.ts +++ b/packages/compiler/src/backend/llvm/counted-loops.ts @@ -151,3 +151,36 @@ function emitExactStart(host: LlvmEmitterContext, expr: IrExpr): string { B.line(`${ok} = and i1 ${inRange}, ${exact}`); return ok; } + +/** Tests that a double is an exact integer within [min, max] (not -0; NaN + * fails the ordered comparisons) and gives its i64. The integer is frozen, + * so it is a fixed value even when the test fails. */ +export function emitExactIntegerTest( + host: LlvmEmitterContext, + value: string, + min: number, + max: number, +): { ok: string; integer: string } { + const B = host.B; + const low = B.tmp(), + high = B.tmp(), + inRange = B.tmp(), + raw = B.tmp(), + integer = B.tmp(), + back = B.tmp(), + bits = B.tmp(), + backBits = B.tmp(), + exact = B.tmp(), + ok = B.tmp(); + B.line(`${low} = fcmp oge double ${value}, ${f64Lit(min)}`); + B.line(`${high} = fcmp ole double ${value}, ${f64Lit(max)}`); + B.line(`${inRange} = and i1 ${low}, ${high}`); + B.line(`${raw} = fptosi double ${value} to i64`); + B.line(`${integer} = freeze i64 ${raw}`); + B.line(`${back} = sitofp i64 ${integer} to double`); + B.line(`${bits} = bitcast double ${value} to i64`); + B.line(`${backBits} = bitcast double ${back} to i64`); + B.line(`${exact} = icmp eq i64 ${bits}, ${backBits}`); + B.line(`${ok} = and i1 ${inRange}, ${exact}`); + return { ok, integer }; +} diff --git a/packages/compiler/src/backend/llvm/dense-array-loops.ts b/packages/compiler/src/backend/llvm/dense-array-loops.ts index 384c31b0c..b806939b5 100644 --- a/packages/compiler/src/backend/llvm/dense-array-loops.ts +++ b/packages/compiler/src/backend/llvm/dense-array-loops.ts @@ -36,8 +36,7 @@ import { everyExprChild, everyStmtChild } from "../../ir/traverse.js"; import type { IntegerCountedForLoop, IntegerPairedForLoop } from "../../ir/integer-loops.js"; import type { LlvmEmitterContext } from "./expr-context.js"; import { loadArrCount, SCR_ARR_FIELD } from "./common.js"; -import { emitCountedLoopLimit } from "./counted-loops.js"; -import { f64Lit } from "./common.js"; +import { emitCountedLoopLimit, emitExactIntegerTest } from "./counted-loops.js"; export const INT32_MAX = 2147483647; @@ -453,28 +452,10 @@ function emitPairedDenseGuard( header: (array: string, field: number) => string, ): string { const B = host.B; - const integer = (expr: IrExpr, min: number, max: number): { value: string; ok: string } => { - const start = host.emitExpr(expr); - const low = B.tmp(), - high = B.tmp(), - raw = B.tmp(), - value = B.tmp(), - back = B.tmp(), - exact = B.tmp(), - inRange = B.tmp(), - ok = B.tmp(); - B.line(`${low} = fcmp oge double ${start.name}, ${f64Lit(min)}`); - B.line(`${high} = fcmp ole double ${start.name}, ${f64Lit(max)}`); - B.line(`${inRange} = and i1 ${low}, ${high}`); - B.line(`${raw} = fptosi double ${start.name} to i64`); - B.line(`${value} = freeze i64 ${raw}`); - B.line(`${back} = sitofp i64 ${value} to double`); - B.line(`${exact} = fcmp oeq double ${back}, ${start.name}`); - B.line(`${ok} = and i1 ${inRange}, ${exact}`); - return { value, ok }; - }; - const low = integer(pair.lowStart, 0, INT32_MAX - 1); - const high = integer(pair.highStart, -1, INT32_MAX - 1); + const start = (expr: IrExpr, min: number, max: number) => + emitExactIntegerTest(host, host.emitExpr(expr).name, min, max); + const low = start(pair.lowStart, 0, INT32_MAX - 1); + const high = start(pair.highStart, -1, INT32_MAX - 1); let valid = B.tmp(); B.line(`${valid} = and i1 ${low.ok}, ${high.ok}`); for (const ref of loop.arrays.values()) { @@ -486,7 +467,7 @@ function emitPairedDenseGuard( both = B.tmp(); B.line(`${filled} = load i32, ptr ${filledPtr}${host.fieldAliasAttachment(filledPtr)}`); B.line(`${wide} = zext i32 ${filled} to i64`); - B.line(`${covered} = icmp slt i64 ${high.value}, ${wide}`); + B.line(`${covered} = icmp slt i64 ${high.integer}, ${wide}`); B.line(`${both} = and i1 ${valid}, ${covered}`); valid = both; } diff --git a/packages/compiler/src/backend/llvm/emit-statements.ts b/packages/compiler/src/backend/llvm/emit-statements.ts index aef5ccf30..55e31219c 100644 --- a/packages/compiler/src/backend/llvm/emit-statements.ts +++ b/packages/compiler/src/backend/llvm/emit-statements.ts @@ -1051,15 +1051,17 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { LOOPS.of(emitter).unaliasedArrays, ) : countedLoop; - const version = versionable ? emitter.matchBytesLoopVersion(s, versionCounted) : null; + const version = versionable + ? emitter.matchBytesLoopVersion(s, versionCounted, versionCounted ? null : pairedLoop) + : null; if (version) { emitVersionedLoop(emitter, s, version); break; } let countedJoin: string | null = null; if ( - (countedLoop?.guarded && !LOOPS.of(emitter).provenCountedGuards.has(s)) || - pairedLoop?.guarded + (countedLoop?.guarded || pairedLoop?.guarded) && + !LOOPS.of(emitter).provenCountedGuards.has(s) ) { const safe = countedLoop ? emitCountedLoopSafety(emitter, countedLoop) @@ -2094,7 +2096,8 @@ function emitVersionedLoop( if (version.owned.size > 0) emitter.bytesOwned = new Set([...originalOwned, ...version.owned]); if (version.ranges.size > 0) emitter.integerRanges = new Map([...originalRanges, ...version.ranges]); - if (version.counted?.guarded) LOOPS.of(emitter).provenCountedGuards.add(s); + if (version.counted?.guarded || version.paired?.loop.guarded) + LOOPS.of(emitter).provenCountedGuards.add(s); emitter.emitStmt(s); B.br(done); } finally { diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 4aa9b65ad..ad59c3b1b 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -60,7 +60,7 @@ import { captureFreeFunctions, observesStacks } from "../stack-observations.js"; import { findInitializedByteLoopBindings } from "./byte-windows.js"; import { matchBytesLoopVersion } from "./bytes-loop-versions.js"; import { globalSlotAlias } from "./bytes-alias.js"; -import type { IntegerCountedForLoop } from "../../ir/integer-loops.js"; +import type { IntegerCountedForLoop, IntegerPairedForLoop } from "../../ir/integer-loops.js"; import { materializeSplitPiece, type SplitSpan, @@ -4263,6 +4263,7 @@ export class LlEmitter { matchBytesLoopVersion( s: IrStmt & { kind: "for" | "while" }, counted: IntegerCountedForLoop | null, + paired: IntegerPairedForLoop | null = null, ): ReturnType { const fn = LOOPS.of(this).bytesLoopFunction; if (!fn) return null; @@ -4272,17 +4273,22 @@ export class LlEmitter { const invariantGlobals = new Set(this.stableGlobals); for (const global of this.initializerBindings.get(fn.name) ?? []) invariantGlobals.add(global.id); - return matchBytesLoopVersion(s, counted, { - locals: this.currentLocals, - invariantGlobals, - captured: this.captureIds, - ranges: this.integerRanges, - initialized, - proven: this.bytesBounds, - workers: this.mod.workers === true, - owned: this.bytesOwned, - scalarStrings: new Set(this.stringSlices.keys()), - }); + return matchBytesLoopVersion( + s, + counted, + { + locals: this.currentLocals, + invariantGlobals, + captured: this.captureIds, + ranges: this.integerRanges, + initialized, + proven: this.bytesBounds, + workers: this.mod.workers === true, + owned: this.bytesOwned, + scalarStrings: new Set(this.stringSlices.keys()), + }, + paired, + ); } /** The alias tag of a whole-slot load of a binding (module globals only). */ diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7981-typed-array-paired-loops.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7981-typed-array-paired-loops.ts.json new file mode 100644 index 000000000..71fa483a8 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7981-typed-array-paired-loops.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7981-typed-array-paired-loops.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7981-typed-array-paired-loops.ts b/tests/corpus/7981-typed-array-paired-loops.ts new file mode 100644 index 000000000..d13eecbd6 --- /dev/null +++ b/tests/corpus/7981-typed-array-paired-loops.ts @@ -0,0 +1,57 @@ +// Two-counter loops over typed arrays (`for (let i = lo, j = hi; i < j; +// i++, j--)`), with starts inside the arrays and outside them: past the +// end, negative, fractional and NaN. Views over one buffer, other element +// types, inclusive conditions and steps other than one. + +function reverse(a: Int32Array, lo: number, hi: number): string { + for (let i = lo, j = hi; i < j; i++, j--) { + const t = a[i]!; + a[i] = a[j]!; + a[j] = t; + } + return Array.from(a).join(); +} + +function mirror(a: Float64Array, b: Float64Array, hi: number): number { + let sum = 0; + for (let i = 0, j = hi; i <= j; i++, j--) { + b[i] = a[j]! * 2; + sum += a[i]! - b[j]!; + } + return sum; +} + +function skipping(a: Uint8ClampedArray, hi: number): string { + for (let i = 0, j = hi; i < j; i += 2, j -= 3) { + a[i] = a[j]! + 200; + a[j] = i - 10; + } + return Array.from(a).join(); +} + +function palindrome(text: Uint16Array, n: number): boolean { + for (let i = 0, j = n - 1; i < j; i++, j--) if (text[i] !== text[j]) return false; + return true; +} + +console.log(reverse(new Int32Array([1, 2, 3, 4, 5, 6]), 0, 5)); +console.log(reverse(new Int32Array([1, 2, 3, 4, 5, 6]), 1, 3)); +console.log(reverse(new Int32Array([1, 2, 3, 4]), 0, 6)); +console.log(reverse(new Int32Array([1, 2, 3, 4]), -2, 2)); +console.log(reverse(new Int32Array([1, 2, 3, 4]), 0.5, 3.5)); +console.log(reverse(new Int32Array([1, 2, 3, 4]), 0, NaN)); +console.log(reverse(new Int32Array([1, 2, 3, 4]), 3, 0)); + +const buffer = new ArrayBuffer(48); +const whole = new Float64Array(buffer); +for (let k = 0; k < whole.length; k++) whole[k] = k + 0.5; +const tail = new Float64Array(buffer, 16, 3); +console.log(mirror(whole, tail, 4), Array.from(whole).join()); +console.log(mirror(tail, tail, 2), Array.from(whole).join()); +console.log(mirror(tail, whole, 7), Array.from(whole).join()); + +console.log(skipping(new Uint8ClampedArray([10, 20, 30, 40, 50, 60, 70, 80]), 7)); +console.log(skipping(new Uint8ClampedArray([10, 20, 30]), 9)); + +const word = new Uint16Array([114, 97, 99, 101, 99, 97, 114]); +console.log(palindrome(word, 7), palindrome(word, 6), palindrome(word, 9), palindrome(word, 0)); diff --git a/tests/harness/corpus-costs/7981-typed-array-paired-loops.ts.json b/tests/harness/corpus-costs/7981-typed-array-paired-loops.ts.json new file mode 100644 index 000000000..c93369880 --- /dev/null +++ b/tests/harness/corpus-costs/7981-typed-array-paired-loops.ts.json @@ -0,0 +1 @@ +{ "plain": 3500, "san": 4500 } From 8b6d3329f4666d641fcbebaf8be501ee4b6d0131 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 13:00:50 -0500 Subject: [PATCH 50/53] Hold an element assignment's receiver against its index's effects When the index assigns the receiver's binding (`a[(a = b, 0)] = c[0]!`), holding the index alone left the receiver read at the store, after the index ran, so the value landed in the replacement array. The receiver is now held whenever the index or the value could change what it reads. --- .../compiler/src/frontend/lowering/array-values.ts | 14 ++++++++++---- tests/corpus/7642-element-assignment-order.ts | 8 ++++++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/packages/compiler/src/frontend/lowering/array-values.ts b/packages/compiler/src/frontend/lowering/array-values.ts index 9f23a860b..f191ad942 100644 --- a/packages/compiler/src/frontend/lowering/array-values.ts +++ b/packages/compiler/src/frontend/lowering/array-values.ts @@ -291,16 +291,22 @@ export function arrayValueStore( // JavaScript evaluates them first (`out[n++] = src[n]` reads src[1]), so // either may need its value held before the value runs. if (value.kind !== "varRef") { - const effects = valueEffects(lowerer, value); - const hold = (e: IrExpr, name: string): IrExpr => { + const valueFx = valueEffects(lowerer, value); + const hold = (e: IrExpr, name: string, effects: typeof valueFx): IrExpr => { if (unaffectedBy(e, effects) && !(effects.opaque && readsMutableLocal(lowerer, e))) return e; const temp = lowerer.declareHiddenLocal(name, e.type); prefix.push({ kind: "varDecl", localId: temp.id, init: e, loc }); return varRef(temp.id, temp.type, loc); }; - arr = hold(arr, "%arrayTarget"); - index = hold(index, "%arrayIndex"); + // The receiver runs before the index and the value, so the index's + // effects can change it too (`a[(a = b, 0)] = c[0]!`). + const indexFx = valueEffects(lowerer, index); + arr = hold(arr, "%arrayTarget", { + writes: new Set([...valueFx.writes, ...indexFx.writes]), + opaque: valueFx.opaque || indexFx.opaque, + }); + index = hold(index, "%arrayIndex", valueFx); } if (value.kind !== "varRef") { const temp = lowerer.declareHiddenLocal("%arrayValue", value.type); diff --git a/tests/corpus/7642-element-assignment-order.ts b/tests/corpus/7642-element-assignment-order.ts index f2f2fae7b..e020d05fc 100644 --- a/tests/corpus/7642-element-assignment-order.ts +++ b/tests/corpus/7642-element-assignment-order.ts @@ -35,3 +35,11 @@ function index(): number { const c: (number | undefined)[] = [0, 0]; c[index()] = optional[1]; console.log(c, reads); + +// The index assigns the receiver's binding: the store uses the receiver +// evaluated before it. +let receiver: (number | undefined)[] = [0, 0]; +const original = receiver; +const replacement: (number | undefined)[] = [7, 7]; +receiver[((receiver = replacement), 1)] = optional[1]; +console.log(original, replacement, receiver === replacement); From 285a92354e998dcb5a329f43f80bb158b4847211 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 13:21:29 -0500 Subject: [PATCH 51/53] Wake TransformStream pulls waiting on a static fetch body --- packages/runtime/src/scr_fetch.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/runtime/src/scr_fetch.c b/packages/runtime/src/scr_fetch.c index 9792f8bed..03de9eed2 100644 --- a/packages/runtime/src/scr_fetch.c +++ b/packages/runtime/src/scr_fetch.c @@ -3227,8 +3227,17 @@ static void sft_set_backpressure(SfTransform *t, bool backpressure) { } } +/* After an enqueue: whether a read is still waiting. A fetch upload's read + * was just satisfied by the chunk, and it reads again through the next + * pull, as a reader's next read() does. */ static bool sft_readable_wants_chunk(const SfStream *r) { - return sf_stream_has_consumer(r); + return r->collector || (r->reader && r->reader->pending_head); +} + +/* The readable side is closed or errored: a pull still waiting on + * bp_change can no longer receive a chunk, so complete it. */ +static void sft_finish_pull(SfTransform *t) { + if (t->bp_change && scr_promise_is_pending(t->bp_change)) scr_promise_fulfill_void(t->bp_change); } static bool sft_readable_can_enqueue(const SfStream *r) { @@ -3334,6 +3343,7 @@ static void sf_transform_step(SfTransform *t, int op) { } else { sf_stream_close(r); } + sft_finish_pull(t); if (error) sf_reject_promise_reason(t->finish, error); else scr_promise_fulfill_void(t->finish); if (t->in_flight_close) sfw_finish_close(t, error); @@ -3355,6 +3365,7 @@ static void sf_transform_step(SfTransform *t, int op) { : scr_dyn_retain(scr_dyn_undefined()); bool readable_errored = r->error != NULL; if (!readable_errored) sf_stream_error(r, reason); + sft_finish_pull(t); /* The sink abort algorithm's finish promise, which a * readable.cancel() after the abort answers too: rejected with the * readable side's stored error when it had already errored, else @@ -3395,6 +3406,7 @@ static void sf_transform_step(SfTransform *t, int op) { } else { sfw_error_if_needed(t, reason ? reason : scr_dyn_undefined()); if (t->t_backpressure) sft_set_backpressure(t, false); + sft_finish_pull(t); scr_promise_fulfill_void(t->finish); } scr_dyn_release(reason); From 26d2c8bae194b6f6c0bf84907341c2cd9c2cc1e6 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Fri, 23 Oct 2026 10:33:57 -0700 Subject: [PATCH 52/53] Iterate untyped arrays without the generic iterator protocol for...of, spread and destructuring over an untyped array opened the iterator by calling the builtin values() function through scr_dyn_call with the array as `this`, and stepped it through the general element path: retain the source, compute the live length, read the element with hole handling, release the source. An array whose iterator method is the builtin values() now gets the native iterator that call returns directly. A step over a plain array whose index is an own element, or past the end, answers without the general path; no getter can run there. Holes, keys(), entries(), typed array capsules and every other source keep the general path. zod 3, z.array(Order).parse over 500 orders: 187.3M -> 182.2M instructions per round (-2.7%). Co-Authored-By: Claude Opus 5.5 --- .../7794-untyped-array-iteration.js.json | 6 ++ packages/runtime/src/scr_json.c | 23 +++++ tests/corpus/7794-untyped-array-iteration.js | 86 +++++++++++++++++++ .../7794-untyped-array-iteration.js.json | 1 + 4 files changed, 116 insertions(+) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7794-untyped-array-iteration.js.json create mode 100644 tests/corpus/7794-untyped-array-iteration.js create mode 100644 tests/harness/corpus-costs/7794-untyped-array-iteration.js.json diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7794-untyped-array-iteration.js.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7794-untyped-array-iteration.js.json new file mode 100644 index 000000000..5ffe60f53 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7794-untyped-array-iteration.js.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7794-untyped-array-iteration.js" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index eed8a9da2..a00a18d82 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -5719,6 +5719,23 @@ static void scr_native_iterator_release(void *ptr) { * next() calls still receive a fresh, independently mutable result object. */ static ScrDyn *scr_native_iterator_value(ScrNativeIterator *iterator, bool *done_out) { ScrDyn *source = iterator->source; + /* An own element of an untyped array, or its end: no getter can run, so + * the general path below is not needed. */ + if (source && source->kind == SCR_DYN_ARR && iterator->kind == 0 && !iterator->array_like && + iterator->selection == 0) { + if (iterator->index >= (double)source->v.arr.len) { + iterator->source = NULL; + scr_dyn_release(source); + *done_out = true; + return scr_dyn_retain(scr_dyn_undefined()); + } + if (scr_dyn_arr_has_index(source, (size_t)iterator->index)) { + size_t index = (size_t)iterator->index; + iterator->index++; + *done_out = false; + return scr_dyn_retain(source->v.arr.items[index]); + } + } /* An inherited indexed getter can advance this same iterator recursively. * Keep the source alive even if that nested call exhausts the iterator. */ if (source) scr_dyn_retain(source); @@ -6863,6 +6880,12 @@ ScrDyn *scr_dyn_iterator(const ScrDyn *source, const ScrStr *spell) { scr_dyn_release(method); return scr_dyn_not_iterable(source, spell); } + /* An array whose iterator method is the builtin values(): what calling + * it with the array as `this` returns, without the call. */ + if (source->kind == SCR_DYN_ARR && method == scr_iterator_functions[1]) { + scr_dyn_release(method); + return scr_native_iterator_new((ScrDyn *)source); + } ScrDyn *iterator = scr_iterator_call(method, source, "value[Symbol.iterator]"); scr_dyn_release(method); if (!iterator) return NULL; diff --git a/tests/corpus/7794-untyped-array-iteration.js b/tests/corpus/7794-untyped-array-iteration.js new file mode 100644 index 000000000..ba7fe0810 --- /dev/null +++ b/tests/corpus/7794-untyped-array-iteration.js @@ -0,0 +1,86 @@ +"use strict"; +// for...of, spread and destructuring over untyped arrays: elements are read +// live, so pushes and truncation during the loop show, holes read as +// undefined, an own Symbol.iterator replaces the builtin one, and keys() +// and entries() keep their own shapes, as in Node. + +/** @param {any} list */ +function collect(list) { + const out = []; + for (const item of list) out.push(item); + return out; +} + +/** @param {any} list */ +function growWhileIterating(list) { + const seen = []; + for (const item of list) { + seen.push(item); + if (item === 2) list.push(99); + if (seen.length > 10) break; + } + return seen; +} + +/** @param {any} list */ +function shrinkWhileIterating(list) { + const seen = []; + for (const item of list) { + seen.push(item); + list.length = 2; + } + return seen; +} + +const parsed = JSON.parse('[1, "two", {"three": 3}, [4], null, true]'); +console.log(JSON.stringify(collect(parsed))); +console.log(JSON.stringify(collect([]))); +console.log(JSON.stringify(growWhileIterating(JSON.parse("[1, 2, 3]")))); +console.log(JSON.stringify(shrinkWhileIterating(JSON.parse("[1, 2, 3, 4, 5]")))); + +/** @type {any} */ +const holes = [1, , 3]; +holes.length = 5; +console.log(collect(holes).map((x) => String(x)).join(",")); + +/** @type {any} */ +const custom = JSON.parse("[1, 2, 3]"); +custom[Symbol.iterator] = function* () { + yield "custom"; + yield this.length; +}; +console.log(JSON.stringify(collect(custom))); + +/** @type {any} */ +const data = JSON.parse('[10, 20, 30]'); +console.log(JSON.stringify([...data]), JSON.stringify([...data.keys()]), JSON.stringify([...data.entries()])); +const [first, , third = "default", fourth = "missing"] = data; +console.log(first, third, fourth); + +let total = 0; +for (const value of data) { + if (value === 20) continue; + total += value; +} +console.log(total); + +/** @param {any} list */ +function nested(list) { + const pairs = []; + for (const a of list) for (const b of list) pairs.push(`${a}${b}`); + return pairs.join(" "); +} +console.log(nested(JSON.parse('["x", "y"]'))); + +/** @param {any} list */ +function earlyReturn(list) { + for (const item of list) if (item > 1) return item; + return -1; +} +console.log(earlyReturn(JSON.parse("[0, 1, 2, 3]")), earlyReturn(JSON.parse("[0]"))); + +try { + console.log(collect(null)); +} catch (e) { + console.log(e instanceof TypeError, e.message); +} diff --git a/tests/harness/corpus-costs/7794-untyped-array-iteration.js.json b/tests/harness/corpus-costs/7794-untyped-array-iteration.js.json new file mode 100644 index 000000000..6c25b05fd --- /dev/null +++ b/tests/harness/corpus-costs/7794-untyped-array-iteration.js.json @@ -0,0 +1 @@ +{ "plain": 3100, "san": 3700 } From 037b0b1dcc42435aa1c56a96efebe22244dc1da7 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 14:02:28 -0500 Subject: [PATCH 53/53] Keep libnode pack manifest paths inside the bundle --- .../frontend/lowering/lower-node-island.ts | 7 +++ .../compiler/src/frontend/lowering/lowerer.ts | 8 +++ .../compiler/src/node-islands/libnode.test.ts | 43 ++++++++++++++- packages/compiler/src/node-islands/libnode.ts | 35 +++++++++---- .../compiler/src/node-islands/modules.test.ts | 27 ++++++++++ packages/compiler/src/node-islands/modules.ts | 52 +++++++++++++++---- .../corpus/7662-node-call-exit-events/main.ts | 1 + 7 files changed, 152 insertions(+), 21 deletions(-) diff --git a/packages/compiler/src/frontend/lowering/lower-node-island.ts b/packages/compiler/src/frontend/lowering/lower-node-island.ts index bab5f15a1..fae8aa219 100644 --- a/packages/compiler/src/frontend/lowering/lower-node-island.ts +++ b/packages/compiler/src/frontend/lowering/lower-node-island.ts @@ -17,6 +17,7 @@ * thrown Error crosses as its nearest intrinsic class with its message and * string `code`. */ import { realpathSync } from "node:fs"; +import { isAbsolute } from "node:path"; import * as ts from "../ts7/adapter.js"; import { locOf } from "../program.js"; import type { NodeApiMember } from "../node-api-import.js"; @@ -108,6 +109,12 @@ export function lowerNodeApiCall( throw new PoisonError(); } const specifier = lowerer.lowerExprExpecting(specifierNode, STRING); + // An absolute specifier names a file of the build machine: the call names + // the shipped copy, relative to the caller inside the tree. + if (isAbsolute(specifierNode.text) && specifier.kind === "strLit") { + const target = lowerer.nodeIslandModules.resolvedPath(caller, specifierNode.text); + if (target !== null) lowerer.nodeIslandSpecifiers.push({ literal: specifier, caller, target }); + } const name = lowerer.lowerExprExpecting(argNodes[1]!, STRING); const values = argNodes.slice(2).map((arg) => { const value = lowerer.lowerExpr(arg); diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index 4b6b0acdf..6594d8175 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -3145,6 +3145,12 @@ export class Lowerer { readonly nodeIslandModules = new NodeIslandModuleCollector(); /** Each call's caller literal, set to its tree path when lowering ends. */ readonly nodeIslandCallers: { literal: IrExpr & { kind: "strLit" }; caller: string }[] = []; + /** Absolute specifiers, rewritten to the shipped file relative to their caller. */ + readonly nodeIslandSpecifiers: { + literal: IrExpr & { kind: "strLit" }; + caller: string; + target: string; + }[] = []; /** Per-directory answers of experimental-import.ts's declaration-file test. */ readonly experimentalFiles = new Map(); /** Interned LifetimeRegion.run helpers, by callback type key. */ @@ -8656,6 +8662,8 @@ export class Lowerer { const modules = this.nodeIslandModules.tree(); for (const { literal, caller } of this.nodeIslandCallers) literal.value = NodeIslandModuleCollector.callerPath(modules, caller); + for (const { literal, caller, target } of this.nodeIslandSpecifiers) + literal.value = NodeIslandModuleCollector.relativeSpecifier(caller, target); return { libnode: "", nodeVersion: "", ...(modules === null ? {} : { modules }) }; } diff --git a/packages/compiler/src/node-islands/libnode.test.ts b/packages/compiler/src/node-islands/libnode.test.ts index cd310e0e1..94add4f1c 100644 --- a/packages/compiler/src/node-islands/libnode.test.ts +++ b/packages/compiler/src/node-islands/libnode.test.ts @@ -1,4 +1,12 @@ -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, expect, test } from "vitest"; @@ -94,6 +102,39 @@ describe("the recorded libnode pack", () => { expect(existsSync(join(out, "app.libnode"))).toBe(false); }); + test("beside refuses manifest file names outside the pack", () => { + for (const name of ["..", ".", "../escaped", "/etc/passwd", "a/b", "a\\b", "x\u0000y"]) { + const dir = pack("24.22.0"); + writeFileSync( + join(dir, "manifest.json"), + JSON.stringify({ + nodeVersion: "24.22.0", + bridgeAbi: NODE_BRIDGE_ABI, + files: { [name]: {} }, + }), + ); + const out = mkdtempSync(join(tmpdir(), "scriptc-libnode-out-")); + expect(libnodeRecord("beside", join(out, "app"), { SCRIPTC_LIBNODE: dir })).toBe( + `the libnode pack at ${dir} lists a file outside the pack ('${name}')`, + ); + expect(existsSync(join(out, "app.libnode"))).toBe(false); + expect(existsSync(join(out, "escaped"))).toBe(false); + } + }); + + test("beside refuses pack files that are symbolic links", () => { + const dir = pack("24.22.0"); + const elsewhere = join(mkdtempSync(join(tmpdir(), "scriptc-libnode-elsewhere-")), "secret"); + writeFileSync(elsewhere, "secret"); + rmSync(join(dir, "libnode.so.137")); + symlinkSync(elsewhere, join(dir, "libnode.so.137")); + const out = mkdtempSync(join(tmpdir(), "scriptc-libnode-out-")); + expect(libnodeRecord("beside", join(out, "app"), { SCRIPTC_LIBNODE: dir })).toBe( + `the libnode pack at ${dir} holds libnode.so.137 as a symbolic link; pack files must be regular files`, + ); + expect(existsSync(join(out, "app.libnode"))).toBe(false); + }); + test("a pack the deployment places beside the executable needs no manifest at build time", () => { expect(libnodeRecord("$ORIGIN/node", "/out/app", {})).toEqual({ libnode: "$ORIGIN/node", diff --git a/packages/compiler/src/node-islands/libnode.ts b/packages/compiler/src/node-islands/libnode.ts index a0ae29d2f..dc6498924 100644 --- a/packages/compiler/src/node-islands/libnode.ts +++ b/packages/compiler/src/node-islands/libnode.ts @@ -14,13 +14,14 @@ * also the hook for bundling Node into the executable itself later. */ import { createHash } from "node:crypto"; import { + constants, copyFileSync, linkSync, + lstatSync, mkdirSync, readFileSync, realpathSync, rmSync, - statSync, } from "node:fs"; import { basename, isAbsolute, join, resolve, sep } from "node:path"; @@ -76,12 +77,15 @@ function readManifest(dir: string): PackManifest | string { } /** Copies the pack next to the executable (hard links where possible), or - * returns why it cannot: the target would replace the pack itself, or the - * manifest names a file outside the pack. */ + * returns why it cannot: the target would replace the pack itself, the + * manifest names a file outside the pack, or a pack file is not a regular + * file (a symbolic link could point anywhere). Every destination is checked + * to resolve inside the target directory. */ function placeBeside(source: string, manifest: PackManifest, target: string): string | undefined { const names = ["manifest.json", "LICENSE", ...Object.keys(manifest.files)]; const unsafe = names.find( - (name) => name === "" || name === "." || name === ".." || /[/\\]/.test(name), + (name) => + name === "" || name === "." || name === ".." || /[/\\\0]/.test(name) || isAbsolute(name), ); if (unsafe !== undefined) return `the libnode pack at ${source} lists a file outside the pack ('${unsafe}')`; @@ -89,19 +93,30 @@ function placeBeside(source: string, manifest: PackManifest, target: string): st const to = realpathOr(target); if (from === to || from.startsWith(to + sep) || to.startsWith(from + sep)) return `--libnode=beside would replace the libnode pack it copies (${source}); point SCRIPTC_LIBNODE at a pack outside ${target}`; - rmSync(target, { recursive: true, force: true }); - mkdirSync(target, { recursive: true }); + const present: string[] = []; for (const name of names) { - const file = join(source, name); + let info; try { - statSync(file); + info = lstatSync(join(source, name)); } catch { continue; } + if (!info.isFile()) + return `the libnode pack at ${source} holds ${name} as a ${info.isSymbolicLink() ? "symbolic link" : "non-regular file"}; pack files must be regular files`; + present.push(name); + } + rmSync(target, { recursive: true, force: true }); + mkdirSync(target, { recursive: true }); + const inside = resolve(target) + sep; + for (const name of present) { + const destination = resolve(target, name); + if (!destination.startsWith(inside)) + return `the libnode pack at ${source} lists a file outside the pack ('${name}')`; + const file = join(source, name); try { - linkSync(file, join(target, name)); + linkSync(file, destination); } catch { - copyFileSync(file, join(target, name)); + copyFileSync(file, destination, constants.COPYFILE_EXCL); } } return undefined; diff --git a/packages/compiler/src/node-islands/modules.test.ts b/packages/compiler/src/node-islands/modules.test.ts index 6badf376a..63201e1e0 100644 --- a/packages/compiler/src/node-islands/modules.test.ts +++ b/packages/compiler/src/node-islands/modules.test.ts @@ -167,6 +167,33 @@ describe("the collected module tree", () => { ]); }); + test("a require through a linked directory keeps the link in the tree", () => { + const root = project({ + "src/main.ts": "", + "src/entry.cjs": "module.exports = require('./lib/x.cjs');", + "shared/x.cjs": "exports.x = 1;", + }); + symlinkSync("../shared", join(root, "src/lib")); + const collector = new NodeIslandModuleCollector(); + expect(collector.add(join(root, "src/main.ts"), "./entry.cjs")).toBeNull(); + const tree = collector.tree()!; + expect(tree.links).toEqual([{ path: "src/lib", target: "../shared" }]); + expect(tree.files.map((f) => f.path)).toEqual(["shared/x.cjs", "src/entry.cjs"]); + }); + + test("an absolute specifier names the shipped file relative to its caller", () => { + const root = project({ "src/main.ts": "", "lib/m.cjs": "" }); + const collector = new NodeIslandModuleCollector(); + const target = collector.resolvedPath(join(root, "src/main.ts"), join(root, "lib/m.cjs"))!; + expect(target).toBe(join(root, "lib/m.cjs")); + expect(NodeIslandModuleCollector.relativeSpecifier(join(root, "src/main.ts"), target)).toBe( + "../lib/m.cjs", + ); + expect( + NodeIslandModuleCollector.relativeSpecifier(join(root, "main.ts"), join(root, "m.cjs")), + ).toBe("./m.cjs"); + }); + test("built-in modules alone ship no tree", () => { const root = project({ "main.ts": "" }); const collector = new NodeIslandModuleCollector(); diff --git a/packages/compiler/src/node-islands/modules.ts b/packages/compiler/src/node-islands/modules.ts index 0d63dc333..e1551c070 100644 --- a/packages/compiler/src/node-islands/modules.ts +++ b/packages/compiler/src/node-islands/modules.ts @@ -211,14 +211,34 @@ export class NodeIslandModuleCollector { return null; } - /** The real path of `path`, recording a link when `path` itself (not an - * ancestor) is a symbolic link. */ + /** The real path of `path`, recording a link for each component of it + * that is a symbolic link (a linked directory on the way, or the file + * itself), so that the path resolves inside the tree as it does here. */ private placed(path: string): string { - const target = trackedRealpath(path) ?? path; - const parent = trackedRealpath(dirname(path)) ?? dirname(path); - const link = join(parent, basename(path)); - if (link !== target) this.links.set(link, target); - return target; + let real: string = sep; + for (const part of resolve(path).split(sep)) { + if (part === "") continue; + const lexical = join(real, part); + const target = trackedRealpath(lexical) ?? lexical; + if (target !== lexical) this.links.set(lexical, target); + real = target; + } + return real; + } + + /** The real path of the file an absolute or relative `specifier` names + * from `fromFile`, or null when there is none. */ + resolvedPath(fromFile: string, specifier: string): string | null { + if (!isPathSpecifier(specifier)) return null; + const target = resolveFile(resolve(dirname(fromFile), specifier)); + return target === null ? null : (trackedRealpath(target) ?? target); + } + + /** A specifier that names `target` from `caller` inside the tree: the + * relative path between them, with `/` separators. */ + static relativeSpecifier(caller: string, target: string): string { + const path = portable(relative(dirname(caller), target)); + return path.startsWith("../") ? path : `./${path}`; } /** A module of the program: collected with the package.json that scopes @@ -609,17 +629,29 @@ export function placeNodeIslandModules( ): string | null { const data = readNodeIslandModules(modules); if (typeof data === "string") return data; + // Every path, and every link's target, stays inside the tree. + const inside = resolve(target) + sep; + const placed = (relativePath: string): string | null => { + const path = resolve(target, ...relativePath.split("/")); + return path.startsWith(inside) ? path : null; + }; + for (const entry of [...modules.files, ...modules.links]) { + const path = placed(entry.path); + if (path === null) return `the island module tree names a path outside it ('${entry.path}')`; + if ("target" in entry && !resolve(dirname(path), entry.target).startsWith(inside)) + return `the island module tree links outside it ('${entry.path}' to '${entry.target}')`; + } rmSync(target, { recursive: true, force: true }); mkdirSync(target, { recursive: true }); let offset = 0; for (const file of modules.files) { - const path = join(target, ...file.path.split("/")); + const path = placed(file.path)!; mkdirSync(dirname(path), { recursive: true }); - writeFileSync(path, data.subarray(offset, offset + file.size)); + writeFileSync(path, data.subarray(offset, offset + file.size), { flag: "wx" }); offset += file.size; } for (const link of modules.links) { - const path = join(target, ...link.path.split("/")); + const path = placed(link.path)!; mkdirSync(dirname(path), { recursive: true }); symlinkSync(link.target, path); } diff --git a/tests/corpus/7662-node-call-exit-events/main.ts b/tests/corpus/7662-node-call-exit-events/main.ts index 909550588..15bb6ad5d 100644 --- a/tests/corpus/7662-node-call-exit-events/main.ts +++ b/tests/corpus/7662-node-call-exit-events/main.ts @@ -1,3 +1,4 @@ +// @exit: 4 // @requires-libnode // The program and the Node.js it loads end as one process: Node's // 'beforeExit' and 'exit' run when the program ends, work that 'beforeExit'