From 8b1deff6e97c894b7dba91e84227ecbae0a87690 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 22:01:42 -0500 Subject: [PATCH 1/8] Step untyped loops over the program's generators on the typed path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A for…of or for await loop over an untyped iterable now tests, once it has its iterator and next method, whether the iterator is one of the program's own generators with its own next method. Each step of such a loop resumes the generator directly and boxes only the value, where the dynamic protocol boxed the result record and its promise and read them back. Every other iterator keeps the dynamic protocol, and early exits still close the iterator through it. The awaits are the protocol's: an async generator's next() promise is awaited, and a sync generator under for await awaits the result and then the value. --- .../lowering/generator-dynamic-dispatch.ts | 287 +++++++++++++++++- .../src/frontend/lowering/lower-stmts.ts | 18 ++ .../compiler/src/frontend/lowering/lowerer.ts | 20 ++ .../8380-untyped-generator-loops.mjs.json | 6 + .../8381-untyped-generator-stages.mjs.json | 6 + tests/corpus/8380-untyped-generator-loops.mjs | 156 ++++++++++ .../corpus/8381-untyped-generator-stages.mjs | 82 +++++ 7 files changed, 573 insertions(+), 2 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8380-untyped-generator-loops.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8381-untyped-generator-stages.mjs.json create mode 100644 tests/corpus/8380-untyped-generator-loops.mjs create mode 100644 tests/corpus/8381-untyped-generator-stages.mjs diff --git a/packages/compiler/src/frontend/lowering/generator-dynamic-dispatch.ts b/packages/compiler/src/frontend/lowering/generator-dynamic-dispatch.ts index db542412f..882013369 100644 --- a/packages/compiler/src/frontend/lowering/generator-dynamic-dispatch.ts +++ b/packages/compiler/src/frontend/lowering/generator-dynamic-dispatch.ts @@ -1,7 +1,8 @@ -import { dynUndefinedExpr, varRef } from "../../ir/build.js"; +import { dynUndefinedExpr, numLit, varRef } from "../../ir/build.js"; import { BOOL, DYN, + F64, STRING, canConvertToDyn, funcOf, @@ -16,7 +17,7 @@ import { import { transformStmtList } from "../../ir/traverse.js"; import { genResultRecord } from "../type-mapper.js"; import type { DispatchSites } from "./dispatch-sites.js"; -import type { Lowerer } from "./lowerer.js"; +import type { Lowerer, UntypedLoopSite } from "./lowerer.js"; type GeneratorType = Extract; type Invoke = Extract; @@ -401,6 +402,288 @@ export class GeneratorDynamicDispatch { }); changed = true; } + return this.stepUntypedLoops(lowerer, functions) || changed; + } + + /** The generator types a loop over an untyped iterable can step natively: + * sync generators for `for … of`, and both kinds for `for await` (a sync + * generator through the async-from-sync rules). Insertion order, so a + * type keeps its index as later waves add types. */ + private loopTypes(lowerer: Lowerer, site: UntypedLoopSite): [string, GeneratorType][] { + const out: [string, GeneratorType][] = []; + for (const [key, type] of this.generators) { + if (type.async === true && !site.async) continue; + const result = genResultRecord(type.yieldT, type.retT, lowerer.shapes, lowerer.unions); + if (result === null) continue; + // next() with no argument, as the protocol calls it. + const channel = type.nextT; + if (channel.kind !== "undefinedT" && channel.kind !== "void" && channel.kind !== "dyn") + continue; + if ( + !canConvertToDyn( + result, + (id) => lowerer.shapes.get(id), + (id) => lowerer.unions.get(id), + ) + ) + continue; + out.push([key, type]); + } + return out; + } + + /** Loops over untyped iterables step the program's own generators on the + * typed path. Once the loop has its iterator and next method, a kind local + * records which generator type the iterator is, when its next method is + * the generator's own (`%dyn.generator.method.next`); each step then + * resumes the generator directly and boxes only the value, where the + * protocol would box the result record and its promise and read them + * back. Every other iterator, and every early exit (return() through the + * protocol), keeps the dynamic path. The awaits are the protocol's: a + * `for await` over an async generator awaits next()'s promise, and over a + * sync generator awaits once for the result and once for the value. */ + private stepUntypedLoops(lowerer: Lowerer, functions: readonly IrFunction[]): boolean { + if (this.generators.size === 0 || lowerer.untypedLoops.size === 0) return false; + // The loop reads next through the getter only once it exists. + if (!this.properties.has("next")) return false; + let changed = false; + for (const fn of functions) { + if (this.generated.has(fn)) continue; + const byKind = new Map(); + const byValue = new Map(); + for (const local of fn.locals) { + const site = lowerer.untypedLoops.get(local); + if (site === undefined) continue; + byKind.set(site.kind, site); + byValue.set(site.value, site); + } + if (byKind.size === 0) continue; + fn.body = transformStmtList(fn.body, { + expr: (expr) => expr, + stmt: (stmt) => { + if (stmt.kind !== "varDecl" || stmt.init === null) return stmt; + const opened = byKind.get(stmt.localId); + if (opened !== undefined) { + const types = this.loopTypes(lowerer, opened); + const init = this.loopKind(opened, types, stmt.loc); + if (this.loopKindCount(stmt.init) === types.length) return stmt; + changed = true; + return { ...stmt, init }; + } + const site = byValue.get(stmt.localId); + if (site === undefined) return stmt; + const types = this.loopTypes(lowerer, site); + // The protocol step is the innermost else of the type chain. + let fallback = stmt.init; + let present = 0; + while ( + fallback.kind === "ternary" && + fallback.cond.kind === "bin" && + fallback.cond.left.kind === "varRef" && + fallback.cond.left.localId === site.kind + ) { + fallback = fallback.else_; + present++; + } + if (present === types.length) return stmt; + let init = fallback; + for (let i = types.length - 1; i >= 0; i--) { + const [, type] = types[i]!; + const record = `${site.kind}.r${i + 1}`; + init = { + kind: "ternary", + cond: { + kind: "bin", + op: "===", + left: varRef(site.kind, F64, stmt.loc), + right: numLit(i + 1, stmt.loc), + type: BOOL, + loc: stmt.loc, + }, + then: this.loopStep(lowerer, fn, site, type, record, stmt.loc), + else_: init, + type: DYN, + loc: stmt.loc, + }; + } + changed = true; + return { ...stmt, init }; + }, + }); + } return changed; } + + /** How many generator types a loop's kind initializer tests: the chain + * inside its own-method test (loopKind). */ + private loopKindCount(init: IrExpr): number { + if (init.kind !== "ternary") return 0; + let count = 0; + for (let chain = init.then; chain.kind === "ternary"; chain = chain.else_) count++; + return count; + } + + /** The loop's kind: the 1-based index of the generator type its iterator + * is, when the next method it read is that generator's own, else 0. */ + private loopKind( + site: UntypedLoopSite, + types: readonly [string, GeneratorType][], + loc: IrExpr["loc"], + ): IrExpr { + const own: IrExpr = { + kind: "libCall", + fn: "dyn.sameValue", + args: [ + varRef(site.next, DYN, loc), + { + kind: "dynFrom", + value: { + kind: "closure", + fnName: "%dyn.generator.method.next", + captures: [], + type: funcOf([DYN], DYN), + loc, + }, + type: DYN, + loc, + }, + ], + type: BOOL, + loc, + }; + let kind: IrExpr = numLit(0, loc); + for (let i = types.length - 1; i >= 0; i--) + kind = { + kind: "ternary", + cond: { + kind: "libCall", + fn: "dyn.typedRefIs", + args: [ + varRef(site.iterator, DYN, loc), + { kind: "strLit", value: types[i]![0], type: STRING, loc }, + ], + type: BOOL, + loc, + }, + then: numLit(i + 1, loc), + else_: kind, + type: F64, + loc, + }; + return types.length === 0 + ? kind + : { kind: "ternary", cond: own, then: kind, else_: numLit(0, loc), type: F64, loc }; + } + + /** One step of a loop over an untyped iterable that is a `type` + * generator: sets the loop's done flag and answers the value, boxed. */ + private loopStep( + lowerer: Lowerer, + fn: IrFunction, + site: UntypedLoopSite, + type: GeneratorType, + recordId: string, + loc: IrExpr["loc"], + ): IrExpr { + const gen: IrExpr = { kind: "dynCheck", value: varRef(site.iterator, DYN, loc), type, loc }; + const setDone = (value: IrExpr): IrStmt => ({ kind: "assign", localId: site.done, value, loc }); + const doneRef = varRef(site.done, BOOL, loc); + const result = genResultRecord(type.yieldT, type.retT, lowerer.shapes, lowerer.unions)!; + if (!fn.locals.some((local) => local.id === recordId)) + fn.locals.push({ id: recordId, name: recordId, type: result, mutable: false }); + const field = (name: "value" | "done", fieldType: IrType): IrExpr => ({ + kind: "recordGet", + obj: varRef(recordId, result, loc), + shapeId: result.shapeId, + field: name, + type: fieldType, + loc, + }); + const valueType = lowerer.shapes + .get(result.shapeId)! + .fields.find((f) => f.name === "value")!.type; + const boxed: IrExpr = { kind: "dynFrom", value: field("value", valueType), type: DYN, loc }; + const awaitDyn = (value: IrExpr): IrExpr => ({ + kind: "libCall", + fn: "async.awaitDyn", + args: [value], + type: DYN, + loc, + }); + const resumed: IrExpr = { + kind: "genResume", + mode: "next", + gen, + arg: null, + type: type.async === true ? { kind: "promise", inner: result } : result, + loc, + }; + if (!site.async) + return { + kind: "seqExpr", + stmts: [ + { kind: "varDecl", localId: recordId, init: resumed, loc }, + setDone(field("done", BOOL)), + ], + result: { + kind: "ternary", + cond: doneRef, + then: dynUndefinedExpr(loc), + else_: boxed, + type: DYN, + loc, + }, + type: DYN, + loc, + }; + if (type.async === true) + return { + kind: "seqExpr", + stmts: [ + { + kind: "varDecl", + localId: recordId, + init: { kind: "awaitExpr", value: resumed, type: result, loc }, + loc, + }, + setDone(field("done", BOOL)), + ], + result: { + kind: "ternary", + cond: doneRef, + then: dynUndefinedExpr(loc), + else_: boxed, + type: DYN, + loc, + }, + type: DYN, + loc, + }; + // for await over a sync generator: the result awaits once, then the + // value (the final one too), as the async-from-sync iterator does. + return { + kind: "seqExpr", + stmts: [ + { kind: "varDecl", localId: recordId, init: resumed, loc }, + setDone(field("done", BOOL)), + { kind: "exprStmt", expr: awaitDyn(dynUndefinedExpr(loc)), loc }, + ], + result: { + kind: "ternary", + cond: doneRef, + then: { + kind: "seqExpr", + stmts: [{ kind: "exprStmt", expr: awaitDyn(boxed), loc }], + result: dynUndefinedExpr(loc), + type: DYN, + loc, + }, + else_: awaitDyn(boxed), + type: DYN, + loc, + }, + type: DYN, + loc, + }; + } } diff --git a/packages/compiler/src/frontend/lowering/lower-stmts.ts b/packages/compiler/src/frontend/lowering/lower-stmts.ts index 84196476a..cf3dfcc5e 100644 --- a/packages/compiler/src/frontend/lowering/lower-stmts.ts +++ b/packages/compiler/src/frontend/lowering/lower-stmts.ts @@ -11624,6 +11624,7 @@ function lowerForOfDyn( const iterator = lowerer.declareHiddenLocal("%dofIterator", DYN); const next = lowerer.declareHiddenLocal("%dofNext", DYN); const fast = lowerer.declareHiddenLocal("%dofNative", BOOL); + const generator = lowerer.declareHiddenLocal("%dofGenerator", F64); const done = lowerer.declareHiddenLocal("%dofDone", BOOL); done.mutable = true; const value = lowerer.declareHiddenLocal("%dofValue", DYN); @@ -11633,6 +11634,15 @@ function lowerForOfDyn( needsClose.mutable = true; const error = lowerer.declareHiddenLocal("%dofError", CAUGHT); const syncIterator = asyncLoop ? lowerer.declareHiddenLocal("%dofSync", BOOL) : null; + lowerer.untypedLoops.set(generator, { + kind: generator.id, + value: value.id, + iterator: iterator.id, + next: next.id, + done: done.id, + async: asyncLoop, + sync: syncIterator?.id ?? null, + }); const awaitDyn = (value: IrExpr): IrExpr => ({ kind: "libCall", fn: "async.awaitDyn", @@ -11907,6 +11917,14 @@ function lowerForOfDyn( : iteratorCanStep(varRef(iterator.id, DYN, loc), varRef(next.id, DYN, loc), loc), loc, }, + // Which of the program's generator types the iterator is, once + // generator-dynamic-dispatch.ts knows them; 0 keeps the protocol. + { + kind: "varDecl", + localId: generator.id, + init: { kind: "numLit", value: 0, type: F64, loc }, + loc, + }, { kind: "varDecl", localId: done.id, diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index 22b0fdb99..764b559b9 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -1927,6 +1927,21 @@ function byName(fns: readonly IrFunction[]): IrFunction[] { ); } +/** A loop over an untyped iterable (lowerForOfDyn). `kind` is a local the + * loop sets once it has opened the iterator: 0 for the dynamic protocol, + * or the 1-based index of the program's generator type it steps natively. + * `value` is the per-step local whose initializer takes one step. */ +export interface UntypedLoopSite { + kind: string; + value: string; + iterator: string; + next: string; + done: string; + /** `for await`; `sync` names its async-from-sync flag. */ + async: boolean; + sync: string | null; +} + export class Lowerer { readonly frontendServices: FrontendServices | undefined; readonly checker: ts.TypeChecker; @@ -3371,6 +3386,11 @@ export class Lowerer { * values other than fresh literals (noteUntypedIntegrity). */ untypedIntegrity = false; readonly liftedFns: IrFunction[] = []; + /** `for … of` and `for await` loops over untyped iterables, by their + * generator-kind local (generator-dynamic-dispatch.ts steps the program's + * own generators there on the typed path). Local ids repeat across + * functions; the local itself is in its function's locals. */ + readonly untypedLoops = new Map(); /** The first lifted function of each name (liftedFnNamed), for the * liftedFns indexed so far. liftedFns only grows. */ private readonly liftedFnsByName = new Map(); diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8380-untyped-generator-loops.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8380-untyped-generator-loops.mjs.json new file mode 100644 index 000000000..91ef13397 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8380-untyped-generator-loops.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8380-untyped-generator-loops.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8381-untyped-generator-stages.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8381-untyped-generator-stages.mjs.json new file mode 100644 index 000000000..f272e7d88 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8381-untyped-generator-stages.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8381-untyped-generator-stages.mjs" + ], + "diags": [] +} diff --git a/tests/corpus/8380-untyped-generator-loops.mjs b/tests/corpus/8380-untyped-generator-loops.mjs new file mode 100644 index 000000000..2a477ca10 --- /dev/null +++ b/tests/corpus/8380-untyped-generator-loops.mjs @@ -0,0 +1,156 @@ +// Untyped for-of and for await loops over generator objects: values, +// completion values, early exits, errors, other iterables through the same +// loop, and the microtask order of each step. +function* numbers(n) { + for (let i = 0; i < n; i++) yield i; + return "done"; +} +function* mixed() { + yield "a"; + yield { k: 1 }; + yield undefined; + yield [1, 2]; + yield Promise.resolve(7); + return 99; +} +async function* asyncNumbers(n) { + for (let i = 0; i < n; i++) yield i * 10; + return "async done"; +} +async function* asyncMixed() { + yield "x"; + yield Promise.resolve(8); + yield null; +} + +function collect(it) { + const out = []; + for (const x of it) out.push(x); + return out; +} +async function collectAsync(it) { + const out = []; + for await (const x of it) out.push(x); + return out; +} +const show = (v) => JSON.stringify(v, (k, x) => (x instanceof Promise ? "promise" : x === undefined ? "undef" : x)); + +async function main() { + // The same loops over generators, arrays, Maps, strings and custom iterators. + console.log("sync", show(collect(numbers(4))), show(collect(mixed()))); + console.log("sync other", show(collect([1, 2])), show(collect(new Map([[1, 2]]))), show(collect("hé"))); + const custom = { + [Symbol.iterator]() { + let i = 0; + return { next: () => ({ value: i * i, done: i++ > 2 }) }; + }, + }; + console.log("sync custom", show(collect(custom))); + console.log("async", show(await collectAsync(asyncNumbers(3))), show(await collectAsync(asyncMixed()))); + console.log("async over sync", show(await collectAsync(mixed())), show(await collectAsync(numbers(3)))); + console.log("async other", show(await collectAsync([Promise.resolve(1), 2])), show(await collectAsync(custom))); + const asyncCustom = { + [Symbol.asyncIterator]() { + let i = 0; + return { next: async () => ({ value: `c${i}`, done: i++ > 1 }) }; + }, + }; + console.log("async custom", show(await collectAsync(asyncCustom))); + + // A generator that has started keeps its position; one that finished stays done. + const started = numbers(5); + started.next(); + console.log("started", show(collect(started)), show(started.next())); + + // Early exits close the generator: finally blocks run once. + function* guarded(label) { + try { + yield 1; + yield 2; + } finally { + console.log(label, "finally"); + } + } + const pick = (v) => v; + for (const x of pick(guarded("break"))) { + console.log("break at", x); + break; + } + try { + for (const x of pick(guarded("throw"))) throw new Error(`body ${x}`); + } catch (e) { + console.log("caught", e.message); + } + const g = pick(guarded("after")); + for (const x of g) { + if (x === 1) break; + } + console.log("after break", show(g.next())); + async function* aguarded(label) { + try { + yield 1; + yield 2; + } finally { + console.log(label, "finally"); + } + } + for await (const x of pick(aguarded("abreak"))) { + console.log("abreak at", x); + break; + } + for await (const x of pick(guarded("sync abreak"))) { + console.log("sync abreak at", x); + break; + } + + // Errors from the generator body reach the loop. + function* failing() { + yield 1; + throw new TypeError("generator failed"); + } + async function* afailing() { + yield 1; + throw new RangeError("async generator failed"); + } + try { + for (const x of pick(failing())) console.log("failing got", x); + } catch (e) { + console.log("caught", e.name, e.message); + } + try { + for await (const x of pick(afailing())) console.log("afailing got", x); + } catch (e) { + console.log("caught", e.name, e.message); + } + // Microtask order: each step against a promise chain ticking alongside. + const ticker = (label, n) => { + let p = Promise.resolve(); + for (let i = 0; i < n; i++) p = p.then(() => console.log(label, "tick", i)); + return p; + }; + const consume = async (label, it) => { + for await (const x of it) console.log(label, "value", x); + console.log(label, "end"); + }; + await Promise.all([consume("sync", pick(numbers(3))), ticker("sync", 10)]); + await Promise.all([consume("async", pick(asyncNumbers(3))), ticker("async", 10)]); + await Promise.all([consume("promises", pick(mixed())), ticker("promises", 14)]); + + // Stages chained through untyped parameters. + async function* double(src) { + for await (const x of src) yield x * 2; + } + async function* keepOdd(src) { + for await (const x of src) if (x % 4 !== 0) yield x; + } + let sum = 0; + for await (const x of keepOdd(double(asyncNumbers(50)))) sum += x; + console.log("stages", sum); + let syncSum = 0; + function* inc(src) { + for (const x of src) yield x + 1; + } + for (const x of inc(inc(numbers(100)))) syncSum += x; + console.log("sync stages", syncSum); +} +main(); diff --git a/tests/corpus/8381-untyped-generator-stages.mjs b/tests/corpus/8381-untyped-generator-stages.mjs new file mode 100644 index 000000000..d326a00c1 --- /dev/null +++ b/tests/corpus/8381-untyped-generator-stages.mjs @@ -0,0 +1,82 @@ +// Untyped stages chained through generator objects: a sync generator source +// consumed with for await, async generator transforms, an async function +// consumer, and the same stages over arrays and custom iterables. +function run(source, transform, destination) { + return destination(transform(source())); +} +const N = 2000; +const total = await run( + function* () { + for (let i = 0; i < N; i++) yield i; + }, + async function* (src) { + for await (const n of src) yield n * 2; + }, + async function (src) { + let sum = 0; + for await (const n of src) sum += n; + return sum; + }, +); +console.log("total", total); + +async function* map(it, f) { + for await (const x of it) yield await f(x); +} +async function* take(it, n) { + if (n <= 0) return; + for await (const x of it) { + yield x; + if (--n === 0) return; + } +} +async function* source(n) { + try { + for (let i = 0; i < n; i++) yield i; + } finally { + console.log("source closed"); + } +} +async function toArray(it) { + const out = []; + for await (const x of it) out.push(x); + return out; +} +const half = async (x) => x / 2; +console.log(await toArray(take(map(source(10), half), 3))); +console.log(await toArray(map([1, 2, 3], half))); +console.log(await toArray(map(new Set(["a", "b"]), async (s) => s.toUpperCase()))); +const words = { + *[Symbol.iterator]() { + yield "x"; + yield "y"; + }, +}; +console.log(await toArray(map(words, async (s) => s + s))); + +// A stage that switches iterables between calls of the same loop. +function* letters() { + yield "p"; + yield "q"; +} +async function* numbersAsync() { + yield 1; + yield 2; +} +const sources = [letters(), numbersAsync(), ["z"], letters()]; +for (const s of sources) console.log(await toArray(s)); + +// Sync stages: generator to generator through untyped parameters. +function* inc(src) { + for (const x of src) yield x + 1; +} +function* evens(src) { + for (const x of src) if (x % 2 === 0) yield x; +} +function* range(n) { + for (let i = 0; i < n; i++) yield i; + return "range done"; +} +let s = 0; +for (const x of evens(inc(range(1000)))) s += x; +console.log("sync", s, [...inc([10, 20])], Array.from(evens(range(7)))); From ddb4111ebdfa0aea8e27da93acb50848be3a4581 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 23:31:44 -0500 Subject: [PATCH 2/8] Address review feedback --- .../lowering/generator-dynamic-dispatch.ts | 36 +++++++++++++++---- tests/corpus/8380-untyped-generator-loops.mjs | 6 ++++ 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/packages/compiler/src/frontend/lowering/generator-dynamic-dispatch.ts b/packages/compiler/src/frontend/lowering/generator-dynamic-dispatch.ts index 882013369..459af8751 100644 --- a/packages/compiler/src/frontend/lowering/generator-dynamic-dispatch.ts +++ b/packages/compiler/src/frontend/lowering/generator-dynamic-dispatch.ts @@ -441,7 +441,8 @@ export class GeneratorDynamicDispatch { * back. Every other iterator, and every early exit (return() through the * protocol), keeps the dynamic path. The awaits are the protocol's: a * `for await` over an async generator awaits next()'s promise, and over a - * sync generator awaits once for the result and once for the value. */ + * sync generator awaits once for the result and, reached through + * Symbol.iterator, once for the value. */ private stepUntypedLoops(lowerer: Lowerer, functions: readonly IrFunction[]): boolean { if (this.generators.size === 0 || lowerer.untypedLoops.size === 0) return false; // The loop reads next through the getter only once it exists. @@ -659,8 +660,15 @@ export class GeneratorDynamicDispatch { type: DYN, loc, }; - // for await over a sync generator: the result awaits once, then the - // value (the final one too), as the async-from-sync iterator does. + // for await over a sync generator: the result awaits once. When the + // loop reached it through Symbol.iterator, the async-from-sync iterator + // also awaits the value (the final one too); a sync generator that an + // object's Symbol.asyncIterator returns passes the value as it is, as + // the protocol does. + const fromSync: IrExpr = + site.sync === null + ? { kind: "boolLit", value: false, type: BOOL, loc } + : varRef(site.sync, BOOL, loc); return { kind: "seqExpr", stmts: [ @@ -672,13 +680,27 @@ export class GeneratorDynamicDispatch { kind: "ternary", cond: doneRef, then: { - kind: "seqExpr", - stmts: [{ kind: "exprStmt", expr: awaitDyn(boxed), loc }], - result: dynUndefinedExpr(loc), + kind: "ternary", + cond: fromSync, + then: { + kind: "seqExpr", + stmts: [{ kind: "exprStmt", expr: awaitDyn(boxed), loc }], + result: dynUndefinedExpr(loc), + type: DYN, + loc, + }, + else_: dynUndefinedExpr(loc), + type: DYN, + loc, + }, + else_: { + kind: "ternary", + cond: fromSync, + then: awaitDyn(boxed), + else_: boxed, type: DYN, loc, }, - else_: awaitDyn(boxed), type: DYN, loc, }, diff --git a/tests/corpus/8380-untyped-generator-loops.mjs b/tests/corpus/8380-untyped-generator-loops.mjs index 2a477ca10..a6dd20ebe 100644 --- a/tests/corpus/8380-untyped-generator-loops.mjs +++ b/tests/corpus/8380-untyped-generator-loops.mjs @@ -56,6 +56,12 @@ async function main() { }, }; console.log("async custom", show(await collectAsync(asyncCustom))); + // Symbol.asyncIterator returning a sync generator: each result is awaited, + // its value is not, so a yielded promise reaches the loop as a promise. + const viaAsyncIterator = { [Symbol.asyncIterator]: () => mixed() }; + const seen = []; + for await (const x of viaAsyncIterator) seen.push(x instanceof Promise ? `promise ${await x}` : x); + console.log("async iterator sync", show(seen)); // A generator that has started keeps its position; one that finished stays done. const started = numbers(5); From 238cb0e7cd3fe23042aed09eee2f68ece1c59f63 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Mon, 26 Oct 2026 07:56:46 -0500 Subject: [PATCH 3/8] Free sanitized fibers' fake stacks --- packages/runtime/src/scr_async.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/packages/runtime/src/scr_async.c b/packages/runtime/src/scr_async.c index f9c6944c2..5d09abc96 100644 --- a/packages/runtime/src/scr_async.c +++ b/packages/runtime/src/scr_async.c @@ -1518,7 +1518,11 @@ static void scr_switch(ScrCtx *from, ScrCtx *to, ScrFiber *to_fiber) { (void)to; scr_trap("scriptc: internal error: native stack switch reached on WASI\n"); #elif defined(SCR_ASAN_FIBERS) - void **save = from_fiber ? &from_fiber->fake_stack : &scr_main_fake_stack; + /* A finished fiber leaves for the last time: a NULL slot has + * AddressSanitizer free its fake stack, which otherwise stays mapped + * until exit. */ + void **save = from_fiber ? (from_fiber->done ? NULL : &from_fiber->fake_stack) + : &scr_main_fake_stack; const void *bottom = to_fiber ? to_fiber->stack : NULL; size_t size = to_fiber ? SCR_FIBER_STACK : 0; __sanitizer_start_switch_fiber(save, bottom, size); @@ -1527,10 +1531,13 @@ static void scr_switch(ScrCtx *from, ScrCtx *to, ScrFiber *to_fiber) { #else swapcontext(from, to); #endif + /* Back in this frame: the context that saved into `save` above runs + * again, whichever fiber `scr_current` names (a finished fiber returns + * to its resumer with no fiber to switch to). */ const void *old_bottom; size_t old_size; - __sanitizer_finish_switch_fiber( - scr_current ? scr_current->fake_stack : scr_main_fake_stack, &old_bottom, &old_size); + __sanitizer_finish_switch_fiber(from_fiber ? from_fiber->fake_stack : scr_main_fake_stack, + &old_bottom, &old_size); #elif defined(SCR_FAST_CTX) scr_ctx_swap(from, to); #else @@ -2228,6 +2235,7 @@ static void scr_prefix_trampoline(void) { scr_als_active = &f->als; entry(f, argpack); scr_current = carrier; + carrier->done = true; /* its stack is freed once the spawner is back */ scr_switch(&carrier->ctx, carrier->return_to, NULL); /* unreachable */ } From 9d0ca5f641547b49fb05625aa9ead0115b8bda97 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Mon, 26 Oct 2026 20:50:23 -0500 Subject: [PATCH 4/8] Give the native layout and metadata emission test 600 seconds The sanitized run of "native LLVM layout and metadata emission (llvm)" takes 170 to 300 seconds in its CI shard beside the shard's other tests, against the suite's 300-second test timeout. On a quiet 8-vCPU machine it takes 130 to 160 seconds. --- tests/harness/self-hosting-emission.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/harness/self-hosting-emission.test.ts b/tests/harness/self-hosting-emission.test.ts index 3977ede29..e6364b855 100644 --- a/tests/harness/self-hosting-emission.test.ts +++ b/tests/harness/self-hosting-emission.test.ts @@ -59,7 +59,9 @@ test("the production layout and metadata stage lowers statically", async () => { }); for (const backend of ["llvm"] as const) { - test(`native LLVM layout and metadata emission (${backend})`, async () => { + // The sanitized dev build of the stage takes 170 to 300 s of CI time beside + // its shard's other tests. + test(`native LLVM layout and metadata emission (${backend})`, { timeout: 600_000 }, async () => { const dir = mkdtempSync( join(process.platform === "win32" ? tmpdir() : "/tmp", "scriptc-native-emission-"), ); From 48ccee523e8a96fb2f8e4e30a4d49f2d6141f4ea Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Mon, 26 Oct 2026 21:01:36 -0500 Subject: [PATCH 5/8] Allocate template literals at their exact length A template literal of strings and numbers reserved 32 bytes for each number, the longest number text, and kept that capacity for the result's lifetime: an 8-byte name built from two numbers used a 96-byte block. The numbers now format into a stack buffer first, so the result is allocated once at its exact length. --- packages/runtime/src/scr_string.c | 33 +++++++++++++++++--------- packages/runtime/test/string.test.ts | 7 ++++++ packages/runtime/test/test_string.c | 35 ++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 11 deletions(-) diff --git a/packages/runtime/src/scr_string.c b/packages/runtime/src/scr_string.c index 548a04fd5..62a155bd1 100644 --- a/packages/runtime/src/scr_string.c +++ b/packages/runtime/src/scr_string.c @@ -706,31 +706,42 @@ static inline size_t scr_utf16_units_short(const char *data, size_t len) { } /* A fresh concatenation (no head, the template-literal shape): numbers - * format straight into the result, which reserves scr_f64_to_str's whole - * 32-byte buffer for each. */ + * format into a stack buffer first, so the result is allocated at its exact + * length. */ #define SCR_F64_TEXT_MAX 32 static __attribute__((noinline)) ScrStr *scr_str_concat_fresh(ScrStr *const *parts, const double *nums, size_t count) { - size_t reserve = 0; + char texts[SCR_STR_MIXED_MAX * SCR_F64_TEXT_MAX]; + size_t text_len[SCR_STR_MIXED_MAX]; + size_t len = 0, used = 0; bool ascii = true; for (size_t i = 0; i < count; i++) { - size_t n = parts[i] ? parts[i]->len : SCR_F64_TEXT_MAX; - if (n > SCR_STR_MAX_CAP - reserve) scr_oom(); - reserve += n; - if (parts[i]) ascii &= parts[i]->ascii; + size_t n; + if (parts[i]) { + n = parts[i]->len; + ascii &= parts[i]->ascii; + } else { + n = scr_f64_to_str(nums[i], texts + used); + text_len[i] = n; + used += n; + } + if (n > SCR_STR_MAX_CAP - len) scr_oom(); + len += n; } - ScrStr *out = scr_str_take_spare(reserve); - if (!out) out = scr_str_alloc(reserve, reserve); + ScrStr *out = scr_str_take_spare(len); + if (!out) out = scr_str_alloc(len, len); char *w = out->data; + const char *text = texts; for (size_t i = 0; i < count; i++) { if (parts[i]) { scr_copy_bytes(w, parts[i]->data, parts[i]->len); w += parts[i]->len; } else { - w += scr_f64_to_str(nums[i], w); + scr_copy_bytes(w, text, text_len[i]); + w += text_len[i]; + text += text_len[i]; } } - size_t len = (size_t)(w - out->data); out->len = len; out->data[len] = '\0'; out->ascii = ascii; diff --git a/packages/runtime/test/string.test.ts b/packages/runtime/test/string.test.ts index 0ca58aaf4..cc6d4fbb6 100644 --- a/packages/runtime/test/string.test.ts +++ b/packages/runtime/test/string.test.ts @@ -88,6 +88,13 @@ test("string methods match Node on committed oracle cases", async () => { expect(r.stderr.trim()).toMatch(/^(\d+)\/\1 cases passed$/); }); +// A template literal of strings and numbers allocates exactly its length: +// numbers format before the result is sized, not into a reserved maximum. +test("template-literal concatenation allocates its exact length", async () => { + const { stderr } = await execFileAsync(bin, ["--concat-capacity"]); + expect(stderr.trim()).toBe("0 concat capacity failures"); +}); + // repeat(count) with count < 0 or Infinity is Node's catchable RangeError // ("Invalid count value: -1"); the runtime leaves it pending. test.each([ diff --git a/packages/runtime/test/test_string.c b/packages/runtime/test/test_string.c index f41c2de50..d0dbebf0d 100644 --- a/packages/runtime/test/test_string.c +++ b/packages/runtime/test/test_string.c @@ -982,7 +982,42 @@ static void ordering_asserts(void) { } } +/* Template-literal concatenations (no head) of strings and numbers: the + * result holds exactly its bytes, whatever the numbers format to. */ +static int check_concat_capacity(void) { + ScrStr *a = scr_str_new("f", 1), *b = scr_str_new("_d", 2), *c = scr_str_new("", 0); + struct { + ScrStr *parts[4]; + double nums[4]; + size_t count; + const char *expected; + } cases[] = { + {{a, NULL, b, NULL}, {0, 7, 0, 2230}, 4, "f7_d2230"}, + {{NULL, a}, {-1.5e-7, 0}, 2, "-1.5e-7f"}, + {{NULL, NULL, NULL}, {1.7976931348623157e308, 0.1, -0.0}, 3, "1.7976931348623157e+3080.10"}, + {{c, NULL, c}, {0, 42, 0}, 3, "42"}, + }; + int failed = 0; + for (size_t i = 0; i < sizeof cases / sizeof cases[0]; i++) { + ScrStr *r = scr_str_concat_mixed(NULL, cases[i].parts, cases[i].nums, cases[i].count); + size_t want = strlen(cases[i].expected); + if (r->len != want || memcmp(r->data, cases[i].expected, want) != 0 || r->data[want] != '\0' || + r->cap != r->len) { + fprintf(stderr, "concat case %zu: got \"%.*s\" len %zu cap %zu, want \"%s\" len %zu\n", i, + (int)r->len, r->data, r->len, (size_t)r->cap, cases[i].expected, want); + failed++; + } + scr_str_release(r); + } + scr_str_release(a); + scr_str_release(b); + scr_str_release(c); + fprintf(stderr, "%d concat capacity failures\n", failed); + return failed != 0; +} + int main(int argc, char **argv) { + if (argc > 1 && strcmp(argv[1], "--concat-capacity") == 0) return check_concat_capacity(); if (argc > 1 && strncmp(argv[1], "--crash-repeat", 14) == 0) { ScrStr *s = scr_str_new("ab", 2); double count = strcmp(argv[1], "--crash-repeat-inf") == 0 From 07f159accd4929cbb035f2f8b0bb5a95f05c3600 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Mon, 26 Oct 2026 21:06:07 -0500 Subject: [PATCH 6/8] Fail type checking when the generated IR field readers miss a field fields.generated.ts reads IR nodes' fields by name, so a field ir.ts gains after the last `pnpm generate:ir-fields` is skipped by the walkers until someone regenerates the file; `--check` in CI reports it, but only on the run that looks. The generator now ends the file with type-level checks, one per node kind and per nested record the walkers enumerate: a field outside the list the file was generated from fails type checking with the field's name, and so does a field read as a scalar (no string, no object) that can now hold a string or an object. Fields the file reads already fail type checking when they are removed or change type. The checks are types only and emit no code. --- packages/compiler/src/ir/fields.generated.ts | 249 +++++++++++++++++++ scripts/generate-ir-fields.mjs | 67 ++++- 2 files changed, 309 insertions(+), 7 deletions(-) diff --git a/packages/compiler/src/ir/fields.generated.ts b/packages/compiler/src/ir/fields.generated.ts index e9f5efaa3..86c6a15e5 100644 --- a/packages/compiler/src/ir/fields.generated.ts +++ b/packages/compiler/src/ir/fields.generated.ts @@ -2069,3 +2069,252 @@ export function walkTypeFields(node: IrType, walker: IrFieldWalker, skipType: bo } } } + +/** A field ir.ts declares that this file does not read. */ +type FieldsNotRead = Exclude; +/** The string or object values a field read as a scalar can hold. */ +type ValuesNotRead = Extract, string | object>; +/** Fails type checking unless `T` is never: run `pnpm generate:ir-fields`. */ +type Unread = T; + +/** Every walked object type's fields as ir.ts declared them when this file + * was generated. */ +export type IrFieldsCurrent = [ + Unread, "kind" | "method" | "receiver" | "args" | "type" | "loc">>, + Unread, "kind" | "arr" | "index" | "type" | "loc">>, + Unread, "kind" | "arr" | "index" | "type" | "loc">>, + Unread, "kind" | "elems" | "spreads" | "capacity" | "type" | "loc">>, + Unread["capacity"]>>, + Unread, "kind" | "length" | "type" | "loc">>, + Unread, "kind" | "arr" | "index" | "type" | "loc">>, + Unread, "kind" | "localId" | "value" | "type" | "loc">>, + Unread, "kind" | "value" | "type" | "adopt" | "loc">>, + Unread["adopt"]>>, + Unread, "kind" | "value" | "promiseTag" | "type" | "loc">>, + Unread["promiseTag"]>>, + Unread, "kind" | "op" | "left" | "right" | "type" | "loc">>, + Unread, "kind" | "value" | "type" | "loc">>, + Unread["value"]>>, + Unread, "kind" | "method" | "receiver" | "args" | "type" | "loc" | "invalidNaN">>, + Unread["invalidNaN"]>>, + Unread, "kind" | "source" | "from" | "type" | "loc">>, + Unread["from"]>>, + Unread, "kind" | "callee" | "args" | "type" | "loc">>, + Unread, "kind" | "callee" | "receiver" | "args" | "type" | "loc">>, + Unread, "kind" | "value" | "className" | "type" | "loc">>, + Unread, "kind" | "value" | "type" | "loc">>, + Unread, "kind" | "value" | "test" | "className" | "negated" | "type" | "loc">>, + Unread["negated"]>>, + Unread, "kind" | "value" | "type" | "loc">>, + Unread, "kind" | "id" | "type" | "loc">>, + Unread, "kind" | "className" | "captures" | "type" | "loc">>, + Unread, "kind" | "fnName" | "captures" | "adapts" | "type" | "loc">>, + Unread["adapts"]>>, + Unread, "kind" | "value" | "type" | "loc">>, + Unread, "kind" | "elems" | "type" | "loc">>, + Unread["spreads"]>, readonly unknown[]>[number]>, object>, "arg" | "what">>, + Unread["spreads"]>, readonly unknown[]>[number]>, object>["arg"]>>, + Unread, "kind" | "callee" | "receiver" | "calleeName" | "calleeNameValue" | "args" | "spreads" | "type" | "loc">>, + Unread, "kind" | "value" | "preserveRefs" | "adopt" | "exact" | "inferredSite" | "origin" | "type" | "loc">>, + Unread["preserveRefs"]>>, + Unread["adopt"]>>, + Unread["exact"]>>, + Unread["origin"]>>, + Unread, "kind" | "value" | "spelling" | "firstProp" | "type" | "loc">>, + Unread, "kind" | "value" | "fnName" | "fnLength" | "liveRef" | "type" | "loc">>, + Unread["fnLength"]>>, + Unread["liveRef"]>>, + Unread, "kind" | "key" | "negated" | "value" | "type" | "loc">>, + Unread["negated"]>>, + Unread, "kind" | "recv" | "method" | "calleeName" | "calleeNameValue" | "args" | "type" | "loc">>, + Unread, "kind" | "value" | "count" | "notIterableMessage" | "type" | "loc">>, + Unread["count"]>>, + Unread, "kind" | "key" | "optional" | "value" | "type" | "loc">>, + Unread["optional"]>>, + Unread["fields"]>, readonly unknown[]>[number]>, object>, "key" | "value">>, + Unread, "kind" | "fields" | "type" | "loc">>, + Unread, "kind" | "left" | "right" | "negated" | "type" | "loc">>, + Unread["negated"]>>, + Unread, "kind" | "test" | "bytesElem" | "negated" | "value" | "type" | "loc">>, + Unread["negated"]>>, + Unread, "kind" | "import" | "args" | "type" | "loc">>, + Unread, "kind" | "unionId" | "type" | "loc">>, + Unread, "kind" | "obj" | "className" | "field" | "type" | "loc">>, + Unread, "kind" | "op" | "prefix" | "obj" | "className" | "field" | "fieldDyn" | "type" | "loc">>, + Unread["prefix"]>>, + Unread["fieldDyn"]>>, + Unread, "kind" | "mode" | "gen" | "arg" | "type" | "loc">>, + Unread, "kind" | "op" | "prefix" | "localId" | "type" | "loc">>, + Unread["prefix"]>>, + Unread, "kind" | "value" | "className" | "type" | "loc">>, + Unread, "kind" | "value" | "classValue" | "type" | "loc">>, + Unread, "kind" | "name" | "args" | "type" | "loc">>, + Unread, "kind" | "value" | "indent" | "circular" | "type" | "loc">>, + Unread["circular"]>>, + Unread["assertOptions"]>, object>, "diff" | "skipPrototype">>, + Unread["assertOptions"]>, object>["skipPrototype"]>>, + Unread, "kind" | "fn" | "args" | "type" | "loc" | "prototypeIdentityOnly" | "realTimer" | "assertOptions">>, + Unread["prototypeIdentityOnly"]>>, + Unread["realTimer"]>>, + Unread, "kind" | "op" | "left" | "right" | "type" | "loc">>, + Unread, "kind" | "method" | "receiver" | "args" | "type" | "loc">>, + Unread["seed"]>, readonly unknown[]>[number]>, object>, "key" | "value">>, + Unread, "kind" | "seed" | "type" | "loc">>, + Unread, "kind" | "moduleId" | "type" | "loc">>, + Unread, "kind" | "className" | "args" | "type" | "loc">>, + Unread, "kind" | "executor" | "type" | "loc">>, + Unread, "kind" | "callee" | "args" | "type" | "loc">>, + Unread, "kind" | "left" | "right" | "type" | "loc">>, + Unread, "kind" | "value" | "spelling" | "type" | "loc">>, + Unread["value"]>>, + Unread, "kind" | "id" | "receiver" | "body" | "type" | "loc">>, + Unread, "kind" | "left" | "right" | "retag" | "type" | "loc">>, + Unread, "kind" | "value" | "type" | "loc">>, + Unread, "kind" | "type" | "loc">>, + Unread["overrides"]>, readonly unknown[]>[number]>, object>, "name" | "value">>, + Unread, "kind" | "source" | "overrides" | "type" | "loc">>, + Unread, "kind" | "obj" | "shapeId" | "field" | "type" | "loc">>, + Unread, "kind" | "obj" | "shapeId" | "field" | "type" | "loc">>, + Unread, "kind" | "obj" | "shapeId" | "key" | "overflowOnly" | "missNaN" | "type" | "loc">>, + Unread["overflowOnly"]>>, + Unread["missNaN"]>>, + Unread["fields"]>, readonly unknown[]>[number]>, object>, "name" | "value" | "overflow">>, + Unread["fields"]>, readonly unknown[]>[number]>, object>["overflow"]>>, + Unread, "kind" | "fields" | "nullProto" | "type" | "loc">>, + Unread["nullProto"]>>, + Unread, "kind" | "obj" | "shapeId" | "type" | "loc">>, + Unread, "kind" | "obj" | "shapeId" | "key" | "type" | "loc">>, + Unread, "kind" | "obj" | "shapeId" | "type" | "loc">>, + Unread, "kind" | "method" | "receiver" | "args" | "type" | "loc">>, + Unread, "kind" | "pattern" | "flags" | "type" | "loc">>, + Unread, "kind" | "type" | "loc">>, + Unread, "kind" | "stmts" | "result" | "generatorDelegate" | "diverges" | "type" | "loc">>, + Unread["generatorDelegate"]>>, + Unread["diverges"]>>, + Unread, "kind" | "method" | "receiver" | "args" | "type" | "loc">>, + Unread, "kind" | "seed" | "type" | "loc">>, + Unread, "kind" | "op" | "left" | "right" | "type" | "loc">>, + Unread, "kind" | "left" | "right" | "type" | "loc">>, + Unread, "kind" | "negated" | "left" | "right" | "type" | "loc">>, + Unread["negated"]>>, + Unread, "kind" | "method" | "receiver" | "args" | "type" | "loc">>, + Unread, "kind" | "value" | "type" | "loc">>, + Unread, "kind" | "key" | "cooked" | "type" | "loc">>, + Unread, "kind" | "cond" | "then" | "else_" | "type" | "loc">>, + Unread, "kind" | "operand" | "type" | "loc">>, + Unread, "kind" | "operand" | "type" | "loc">>, + Unread, "kind" | "op" | "operand" | "type" | "loc">>, + Unread, "kind" | "unionId" | "field" | "value" | "type" | "loc">>, + Unread, "kind" | "unionId" | "negated" | "sameValue" | "left" | "right" | "type" | "loc">>, + Unread["negated"]>>, + Unread["sameValue"]>>, + Unread, "kind" | "unionId" | "tag" | "union" | "func" | "negated" | "type" | "loc">>, + Unread["tag"]>>, + Unread["negated"]>>, + Unread, "kind" | "unionId" | "tag" | "negated" | "value" | "type" | "loc">>, + Unread["tag"]>>, + Unread["negated"]>>, + Unread, "kind" | "unionId" | "key" | "value" | "type" | "loc">>, + Unread, "kind" | "unionId" | "tag" | "value" | "type" | "loc">>, + Unread["tag"]>>, + Unread, "kind" | "fromId" | "unionId" | "tags" | "value" | "type" | "loc">>, + Unread, "kind" | "unionId" | "tag" | "value" | "type" | "loc">>, + Unread["tag"]>>, + Unread, "kind" | "unit" | "type" | "loc">>, + Unread, "kind" | "value" | "type" | "loc">>, + Unread, "kind" | "localId" | "type" | "loc">>, + Unread, "kind" | "className" | "method" | "args" | "type" | "loc" | "targets">>, + Unread["captureCompletion"]>, object>, "returnType">>, + Unread, "kind" | "value" | "awaited" | "delegated" | "captureCompletion" | "type" | "loc">>, + Unread["awaited"]>>, + Unread["delegated"]>>, + Unread, "kind" | "arr" | "index" | "loc">>, + Unread, "kind" | "arr" | "index" | "value" | "loc">>, + Unread, "kind" | "arr" | "length" | "loc" | "pop" | "reserve">>, + Unread["pop"]>>, + Unread["reserve"]>>, + Unread, "kind" | "arr" | "index" | "loc">>, + Unread, "kind" | "localId" | "value" | "initializes" | "loc">>, + Unread["initializes"]>>, + Unread, "kind" | "body" | "labels" | "loc">>, + Unread, "kind" | "label" | "loc">>, + Unread, "kind" | "arr" | "index" | "value" | "loc">>, + Unread, "kind" | "label" | "loc">>, + Unread, "kind" | "body" | "cond" | "labels" | "loc">>, + Unread, "kind" | "expr" | "loc">>, + Unread, "kind" | "obj" | "className" | "field" | "value" | "sloppy" | "define" | "attributesChecked" | "loc">>, + Unread["sloppy"]>>, + Unread["define"]>>, + Unread["attributesChecked"]>>, + Unread, "kind" | "init" | "cond" | "update" | "body" | "labels" | "loc">>, + Unread, "kind" | "localId" | "iterable" | "body" | "labels" | "loc">>, + Unread, "kind" | "cond" | "then" | "else_" | "loc">>, + Unread, "kind" | "obj" | "shapeId" | "key" | "loc">>, + Unread, "kind" | "obj" | "shapeId" | "key" | "value" | "overflowOnly" | "loc">>, + Unread["overflowOnly"]>>, + Unread, "kind" | "obj" | "shapeId" | "field" | "value" | "loc">>, + Unread, "kind" | "localId" | "loc">>, + Unread, "kind" | "value" | "awaited" | "loc">>, + Unread["awaited"]>>, + Unread, "kind" | "code" | "message" | "loc">>, + Unread["cases"]>, readonly unknown[]>[number]>, object>, "test" | "body">>, + Unread, "kind" | "disc" | "cases" | "labels" | "loc">>, + Unread, "kind" | "value" | "loc">>, + Unread, "kind" | "tryBody" | "catchBody" | "catchLocalId" | "finallyBody" | "suppressFinallyErrors" | "dropFinallyErrors" | "loc">>, + Unread["suppressFinallyErrors"]>>, + Unread["dropFinallyErrors"]>>, + Unread, "kind" | "localId" | "init" | "loc">>, + Unread, "kind" | "cond" | "body" | "labels" | "loc">>, + Unread, "kind" | "elem" | "optionalElements">>, + Unread["optionalElements"]>>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind" | "elem">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind" | "className">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind" | "params" | "ret" | "rest" | "restAbi" | "argumentsAll">>, + Unread["rest"]>>, + Unread["argumentsAll"]>>, + Unread, "kind" | "async" | "yieldT" | "retT" | "nextT">>, + Unread["async"]>>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind" | "key" | "value">>, + Unread, "kind" | "moduleId">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind" | "className">>, + Unread, "kind">>, + Unread, "kind" | "inner">>, + Unread, "kind" | "shapeId">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind" | "elem">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind">>, + Unread, "kind" | "unionId">>, + Unread, "kind">>, + Unread, "kind">>, +]; diff --git a/scripts/generate-ir-fields.mjs b/scripts/generate-ir-fields.mjs index 5b5ad011a..037cb615d 100644 --- a/scripts/generate-ir-fields.mjs +++ b/scripts/generate-ir-fields.mjs @@ -105,9 +105,23 @@ function irKind(type) { } let temp = 0; -/** Lines that walk `expr` of `type`: strings to `walker.string`, IR nodes to - * their walkers, arrays by element and other objects by field. */ -function walkLines(expr, type, indent, depth) { +/** The fields each walked object type declares, for the type-level check at + * the end of the file: `typeExpr` names the object type, `fields` every + * field it had when the file was generated and `scalars` the fields that held + * no string and no object then. */ +const checks = []; +/** Records `property list` of `object` (named `typeExpr`) for the check. */ +function noteFields(typeExpr, properties, scalarNames) { + checks.push({ + typeExpr, + fields: properties.map((property) => property.name), + scalars: scalarNames, + }); +} +/** Lines that walk `expr` of `type` (named `typeExpr` in the generated file): + * strings to `walker.string`, IR nodes to their walkers, arrays by element and + * other objects by field. */ +function walkLines(expr, type, typeExpr, indent, depth) { if (depth > 12) throw new Error(`field type nests too deep at ${expr}`); const parts = type.isUnion() ? type.types : [type]; const present = parts.filter((part) => (part.flags & nullish) === 0); @@ -127,19 +141,28 @@ function walkLines(expr, type, indent, depth) { if (objects.length !== 1) throw new Error(`several array types at ${expr}`); const item = `item${temp++}`; body.push(`for (const ${item} of ${expr}) {`); - body.push(...walkLines(item, checker.getTypeArguments(objects[0])[0], " ", depth + 1)); + const itemType = `Extract, readonly unknown[]>[number]`; + body.push( + ...walkLines(item, checker.getTypeArguments(objects[0])[0], itemType, " ", depth + 1), + ); body.push("}"); } else if (kinds.has(null) && objects.length === 1) { - for (const property of checker.getPropertiesOfType(objects[0])) { + const recordType = `Extract, object>`; + const properties = checker.getPropertiesOfType(objects[0]); + const scalarNames = []; + for (const property of properties) { if (property.name === "loc") continue; const fieldType = checker.getTypeOfSymbol(property); const field = `${expr}.${property.name}`; - const lines = walkLines(field, fieldType, "", depth + 1); + const fieldTypeExpr = `${recordType}[${JSON.stringify(property.name)}]`; + const lines = walkLines(field, fieldType, fieldTypeExpr, "", depth + 1); + if (lines.length === 0) scalarNames.push(property.name); if (lines.length === 0) continue; if (property.name === "type") body.push("if (!skipType) {", ...lines.map((l) => ` ${l}`), "}"); else body.push(...lines); } + noteFields(recordType, properties, scalarNames); } else throw new Error(`mixed object types at ${expr}: ${checker.typeToString(type)}`); } let lines = body; @@ -193,20 +216,50 @@ for (const name of ["IrExpr", "IrStmt", "IrType"]) { } for (const [kind, member] of [...byKind].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))) { lines.push(` case ${JSON.stringify(kind)}: {`); + const memberType = `Extract<${name}, { kind: ${JSON.stringify(kind)} }>`; + const scalarNames = []; for (const property of member.getProperties()) { if (property.name === "loc") continue; const fieldType = checker.getTypeOfSymbol(property); const field = `node.${property.name}`; - const walk = walkLines(field, fieldType, "", 0); + const fieldTypeExpr = `${memberType}[${JSON.stringify(property.name)}]`; + const walk = walkLines(field, fieldType, fieldTypeExpr, "", 0); + if (walk.length === 0) scalarNames.push(property.name); if (walk.length === 0) continue; if (property.name === "type") lines.push(" if (!skipType) {", ...walk.map((l) => ` ${l}`), " }"); else lines.push(...walk.map((l) => ` ${l}`)); } lines.push(" return;", " }"); + noteFields(memberType, member.getProperties(), scalarNames); } lines.push(" }", "}", ""); } + +// Type checking fails below when ir.ts gives a walked object a field this +// file does not know, or lets a field that held no string and no object hold +// one before this file is regenerated. +const quoted = (names) => names.map((name) => JSON.stringify(name)).join(" | "); +lines.push( + "/** A field ir.ts declares that this file does not read. */", + "type FieldsNotRead = Exclude;", + "/** The string or object values a field read as a scalar can hold. */", + "type ValuesNotRead = Extract, string | object>;", + "/** Fails type checking unless `T` is never: run `pnpm generate:ir-fields`. */", + "type Unread = T;", + "", + "/** Every walked object type's fields as ir.ts declared them when this file", + " * was generated. */", + "export type IrFieldsCurrent = [", +); +for (const { typeExpr, fields, scalars } of checks) { + lines.push( + ` Unread 0 ? quoted(fields) : "never"}>>,`, + ); + for (const scalarName of scalars) + lines.push(` Unread>,`); +} +lines.push("];", ""); const text = lines.join("\n"); if (process.argv.includes("--check")) { From e4fe30e08446700fe21983937b6570744ee52d4b Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Mon, 26 Oct 2026 21:16:15 -0500 Subject: [PATCH 7/8] Leave out a header the compiler rejects in the implicit toolchain probe The implicit toolchain probe preprocesses every system header that the runtime and vendored sources spell in one source file. When the compiler rejected one of them, the whole probe failed, and the build continued without the native cache and without program sharding. When the compiler rejects the probe source, each header is now preprocessed on its own, and a header that the compiler rejects on its own is left out of the dependency set, as an absent header is. The probe still fails when the compiler rejects every header. --- .../src/backend/native-toolchain.test.ts | 25 ++++++++ .../backend/native/compiler-fingerprint.ts | 61 +++++++++++++++---- 2 files changed, 73 insertions(+), 13 deletions(-) diff --git a/packages/compiler/src/backend/native-toolchain.test.ts b/packages/compiler/src/backend/native-toolchain.test.ts index 8fb86d4af..699847a2a 100644 --- a/packages/compiler/src/backend/native-toolchain.test.ts +++ b/packages/compiler/src/backend/native-toolchain.test.ts @@ -39,6 +39,8 @@ import { warmNativeCaches, } from "./native-toolchain.js"; import { splitLlvmProgram } from "./llvm/split.js"; +import { implicitToolchainFingerprints } from "./native/compiler-fingerprint.js"; +import { fingerprintDependencyPaths } from "./native/dependency-files.js"; const scratch: string[] = []; const TEST_CACHE_IDENTITY = "cc-cache-tests-v1"; @@ -672,6 +674,29 @@ test.skipIf(clangExecutable === undefined)( }, ); +test.skipIf(clangExecutable === undefined)( + "the implicit toolchain probe leaves out a header the compiler rejects", + async () => { + // A system header that the runtime spells, shadowed by one that is an + // #error, as clang 18's is outside RISC-V. The probe + // still covers the other headers. + const dir = await mkdtemp(join(tmpdir(), "scriptc-probe-header-")); + scratch.push(dir); + await mkdir(join(dir, "sys")); + const rejected = join(dir, "sys", "auxv.h"); + await writeFile(rejected, '#error "not for this target"\n'); + const includes = await implicitDependencyProbeIncludes(runtimeSrcDir()); + expect(includes).toContain(""); + const fingerprints = await implicitToolchainFingerprints( + { argv: [clangExecutable!, "-isystem", dir], targetArgs: [], target: null }, + "probe-rejected-header", + ); + const dependencies = fingerprintDependencyPaths(fingerprints.complete); + expect(dependencies).not.toContain(rejected); + expect(dependencies.some((path) => path.endsWith("/stdatomic.h"))).toBe(true); + }, +); + test.skipIf( process.platform === "win32" || clangExecutable === undefined || diff --git a/packages/compiler/src/backend/native/compiler-fingerprint.ts b/packages/compiler/src/backend/native/compiler-fingerprint.ts index 4be20a4d0..3df6cb449 100644 --- a/packages/compiler/src/backend/native/compiler-fingerprint.ts +++ b/packages/compiler/src/backend/native/compiler-fingerprint.ts @@ -399,11 +399,8 @@ async function implicitToolchainFingerprintsFresh( ]; const [dependencyResults, linker, assembler, compilerInvocation] = await Promise.all([ Promise.all( - sources.map((source) => - execFileAsync(compiler, [...probeArgs, source], { - cwd: probeDir, - maxBuffer: 16 * 1024 * 1024, - }), + sources.map((source, index) => + probeIncludeDependencies(compiler, probeArgs, probeDir, source, sourceGroups[index]!), ), ), execFileAsync(compiler, [...prefix, "-print-prog-name=ld"], { cwd: probeDir }), @@ -423,14 +420,7 @@ async function implicitToolchainFingerprintsFresh( const toolSpellings = [linker.stdout.trim(), assemblerSpelling].filter( (value, index, all) => value !== "" && all.indexOf(value) === index, ); - const sourceSet = new Set(sources); - const dependencyPaths = [ - ...new Set( - dependencyResults - .flatMap((dependencies) => parseMakeDependencies(dependencies.stdout)) - .filter((path) => !sourceSet.has(path)), - ), - ].sort(); + const dependencyPaths = [...new Set(dependencyResults.flat())].sort(); probe = { compilerIdentity: compilerIdentity ?? `\0${compiler}`, compilerInvocation: normalizedProbeInvocation(compilerInvocation, probeDir), @@ -513,6 +503,51 @@ async function implicitToolchainFingerprintsFresh( }; } +/** Headers the dependency probe preprocesses at a time when it probes them + * one by one. */ +const SINGLE_HEADER_PROBES = 8; + +/** The files the compiler reads to preprocess `source`, a probe source that + * includes `includes`, apart from the probe's own sources. When the compiler + * rejects `source`, for example for a header it finds that is an #error + * outside its target ( through clang 18 without the RISC-V + * vector extension), each header is preprocessed on its own, and a header + * that the compiler rejects on its own is left out, as an absent one is. + * Throws the compiler's error when it rejects every header. */ +async function probeIncludeDependencies( + compiler: string, + probeArgs: readonly string[], + probeDir: string, + source: string, + includes: readonly string[], +): Promise { + const dependencies = async (path: string): Promise => { + const result = await execFileAsync(compiler, [...probeArgs, path], { + cwd: probeDir, + maxBuffer: 16 * 1024 * 1024, + }); + return parseMakeDependencies(result.stdout).filter((dependency) => dependency !== path); + }; + try { + return await dependencies(source); + } catch (error) { + const found: string[][] = []; + for (let start = 0; start < includes.length; start += SINGLE_HEADER_PROBES) { + const batch = includes.slice(start, start + SINGLE_HEADER_PROBES); + const results = await Promise.all( + batch.map(async (include, offset) => { + const single = `${source.slice(0, -2)}-${start + offset}.c`; + await writeFile(single, implicitDependencyIncludeDirective(include)); + return dependencies(single).catch(() => null); + }), + ); + for (const result of results) if (result !== null) found.push(result); + } + if (found.length === 0) throw error; + return found.flat(); + } +} + const stableImplicitToolchainMemos = new Map>(); export function implicitToolchainFingerprints( driver: Pick, From 2c09b499b25394ee3a6f04c8246f798b09344020 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Mon, 26 Oct 2026 21:18:38 -0500 Subject: [PATCH 8/8] Check a nested record of scalar fields by its own fields The generator listed a field as a scalar when its walk read nothing, so a nested record holding only numbers and booleans got a ValuesNotRead check on the record itself, which fails because the record is an object, even after regenerating. A field is now listed as a scalar only when its type holds no string and no object; such a record is checked through its own fields, so a field added to it or a scalar in it that starts to hold a string still fails type checking with its name. The generator exports generateIrFields(irText) for the new test, which generates the readers for an ir.ts with such a record and type-checks them. The generated file for the current ir.ts is unchanged. --- scripts/generate-ir-fields.mjs | 506 ++++++++++++---------- tests/harness/ir-fields-generator.test.ts | 60 +++ 2 files changed, 327 insertions(+), 239 deletions(-) create mode 100644 tests/harness/ir-fields-generator.test.ts diff --git a/scripts/generate-ir-fields.mjs b/scripts/generate-ir-fields.mjs index 037cb615d..95ce5d217 100644 --- a/scripts/generate-ir-fields.mjs +++ b/scripts/generate-ir-fields.mjs @@ -14,255 +14,283 @@ const require = createRequire(join(root, "package.json")); const ts = require("typescript"); const irPath = join(root, "packages/compiler/src/ir/ir.ts"); const target = join(root, "packages/compiler/src/ir/fields.generated.ts"); +const samePath = (a, b) => a.replace(/\\/g, "/") === b.replace(/\\/g, "/"); -const program = ts.createProgram([irPath], { strict: true, target: ts.ScriptTarget.ES2022 }); -const checker = program.getTypeChecker(); -const source = program.getSourceFile(irPath); -const aliases = new Map(); -ts.forEachChild(source, (node) => { - if (ts.isTypeAliasDeclaration(node)) aliases.set(node.name.text, node); -}); - -/** Whether a value of `type` can be a string. */ -function holdsString(type) { - if (type.isUnion()) return type.types.some(holdsString); - return (type.flags & (ts.TypeFlags.StringLike | ts.TypeFlags.Any | ts.TypeFlags.Unknown)) !== 0; -} - -/** Whether every value of `type` is a string. */ -function onlyString(type) { - if (type.isUnion()) return type.types.every(onlyString); - return (type.flags & ts.TypeFlags.StringLike) !== 0; -} - -/** Each kind's string fields, in declaration order, merged over the unions: - * a kind both unions declare lists the fields of both. */ -const kinds = new Map(); -for (const name of ["IrExpr", "IrStmt"]) { - const alias = aliases.get(name); - const union = checker.getTypeAtLocation(alias.name); - for (const member of union.isUnion() ? union.types : [union]) { - const kindSymbol = member.getProperty("kind"); - const kindType = checker.getTypeOfSymbolAtLocation(kindSymbol, alias); - if (!kindType.isStringLiteral()) throw new Error(`${name} member without a literal kind`); - const fields = kinds.get(kindType.value) ?? []; - for (const property of member.getProperties()) { - const type = checker.getTypeOfSymbolAtLocation(property, alias); - if (!holdsString(type)) continue; - const known = fields.find((field) => field.name === property.name); - const always = onlyString(type) && (property.flags & ts.SymbolFlags.Optional) === 0; - if (known === undefined) fields.push({ name: property.name, always }); - else known.always &&= always; - } - kinds.set(kindType.value, fields); - } -} - -const lines = [ - "// Generated from packages/compiler/src/ir/ir.ts by scripts/generate-ir-fields.mjs.", - "// Run `pnpm generate:ir-fields` after changing the fields of an IR node.", - 'import type { IrExpr, IrStmt, IrType } from "./ir.js";', - "", - "/** Calls `visit` with each string field of `node` (including `kind`) and its", - " * name: the string values `for (const key in node)` finds on a node, read", - " * by name in the order ir.ts declares them. */", - "export function eachStringField(", - " node: IrExpr | IrStmt,", - " visit: (value: string, field: string) => void,", - "): void {", - " switch (node.kind) {", -]; -for (const [kind, fields] of [...kinds].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))) { - lines.push(` case ${JSON.stringify(kind)}: {`); - for (const { name, always } of fields) { - const read = `node.${name}`; - if (always) lines.push(` visit(${read}, ${JSON.stringify(name)});`); - else - lines.push(` if (typeof ${read} === "string") visit(${read}, ${JSON.stringify(name)});`); - } - lines.push(" return;", " }"); -} -lines.push(" }", "}", ""); - -// The deep walkers: every field of every object under an IR node, as a -// recursive key walk finds them, read by name. -const members = new Map(); -for (const name of ["IrExpr", "IrStmt", "IrType"]) { - const union = checker.getTypeAtLocation(aliases.get(name).name); - members.set(name, new Set(union.isUnion() ? union.types : [union])); -} -const nullish = ts.TypeFlags.Null | ts.TypeFlags.Undefined | ts.TypeFlags.Void; -const scalar = - ts.TypeFlags.NumberLike | - ts.TypeFlags.BooleanLike | - ts.TypeFlags.BigIntLike | - ts.TypeFlags.ESSymbolLike; - -/** The walker an IR union member belongs to, or null. */ -function irKind(type) { - for (const [name, set] of members) if (set.has(type)) return name; - return null; -} - -let temp = 0; -/** The fields each walked object type declares, for the type-level check at - * the end of the file: `typeExpr` names the object type, `fields` every - * field it had when the file was generated and `scalars` the fields that held - * no string and no object then. */ -const checks = []; -/** Records `property list` of `object` (named `typeExpr`) for the check. */ -function noteFields(typeExpr, properties, scalarNames) { - checks.push({ - typeExpr, - fields: properties.map((property) => property.name), - scalars: scalarNames, +/** The text of fields.generated.ts for `irText`, the source of ir.ts. */ +export function generateIrFields(irText = readFileSync(irPath, "utf8")) { + const options = { strict: true, target: ts.ScriptTarget.ES2022 }; + const host = ts.createCompilerHost(options); + const readSourceFile = host.getSourceFile; + // ir.ts is read from `irText`; the files it imports from disk. + host.getSourceFile = (file, languageVersion, ...rest) => + samePath(file, irPath) + ? ts.createSourceFile(file, irText, languageVersion) + : readSourceFile.call(host, file, languageVersion, ...rest); + const program = ts.createProgram([irPath], options, host); + const checker = program.getTypeChecker(); + const source = program.getSourceFile(irPath); + const aliases = new Map(); + ts.forEachChild(source, (node) => { + if (ts.isTypeAliasDeclaration(node)) aliases.set(node.name.text, node); }); -} -/** Lines that walk `expr` of `type` (named `typeExpr` in the generated file): - * strings to `walker.string`, IR nodes to their walkers, arrays by element and - * other objects by field. */ -function walkLines(expr, type, typeExpr, indent, depth) { - if (depth > 12) throw new Error(`field type nests too deep at ${expr}`); - const parts = type.isUnion() ? type.types : [type]; - const present = parts.filter((part) => (part.flags & nullish) === 0); - const optional = present.length !== parts.length; - const strings = present.filter((part) => (part.flags & ts.TypeFlags.StringLike) !== 0); - const objects = present.filter((part) => (part.flags & (ts.TypeFlags.StringLike | scalar)) === 0); - for (const part of present) - if ((part.flags & (ts.TypeFlags.Any | ts.TypeFlags.Unknown)) !== 0) - throw new Error(`untyped field ${expr}`); - const body = []; - if (objects.length > 0) { - const kinds = new Set(objects.map(irKind)); - if (kinds.size === 1 && !kinds.has(null)) { - const [kind] = kinds; - body.push(`walk${kind.slice(2)}Fields(${expr}, walker, skipType);`); - } else if (objects.every((part) => checker.isArrayType(part))) { - if (objects.length !== 1) throw new Error(`several array types at ${expr}`); - const item = `item${temp++}`; - body.push(`for (const ${item} of ${expr}) {`); - const itemType = `Extract, readonly unknown[]>[number]`; - body.push( - ...walkLines(item, checker.getTypeArguments(objects[0])[0], itemType, " ", depth + 1), - ); - body.push("}"); - } else if (kinds.has(null) && objects.length === 1) { - const recordType = `Extract, object>`; - const properties = checker.getPropertiesOfType(objects[0]); - const scalarNames = []; - for (const property of properties) { - if (property.name === "loc") continue; - const fieldType = checker.getTypeOfSymbol(property); - const field = `${expr}.${property.name}`; - const fieldTypeExpr = `${recordType}[${JSON.stringify(property.name)}]`; - const lines = walkLines(field, fieldType, fieldTypeExpr, "", depth + 1); - if (lines.length === 0) scalarNames.push(property.name); - if (lines.length === 0) continue; - if (property.name === "type") - body.push("if (!skipType) {", ...lines.map((l) => ` ${l}`), "}"); - else body.push(...lines); - } - noteFields(recordType, properties, scalarNames); - } else throw new Error(`mixed object types at ${expr}: ${checker.typeToString(type)}`); - } - let lines = body; - if (strings.length > 0 && objects.length > 0) - lines = [ - `if (typeof ${expr} === "string") walker.string(${expr});`, - ...(body.length > 0 ? [`else {`, ...body.map((l) => ` ${l}`), "}"] : []), - ]; - else if (strings.length > 0) lines = [`walker.string(${expr});`]; - if (lines.length > 0 && optional && !(strings.length > 0 && objects.length > 0)) - lines = [ - `if (${expr} !== undefined && ${expr} !== null) {`, - ...lines.map((l) => ` ${l}`), - "}", - ]; - else if (lines.length > 0 && optional) - lines = [ - `if (${expr} !== undefined && ${expr} !== null) {`, - ...lines.map((l) => ` ${l}`), - "}", - ]; - return lines.map((line) => indent + line); -} -lines.push( - "/** What a field walk reports (walkExprFields): each expression and", - " * statement node before its fields, and each string a field holds. */", - "export interface IrFieldWalker {", - " node(node: IrExpr | IrStmt): void;", - " string(value: string): void;", - "}", - "", -); -for (const name of ["IrExpr", "IrStmt", "IrType"]) { - const short = name.slice(2); - const nodeCall = name === "IrType" ? [] : [" walker.node(node);"]; - lines.push( - `/** Walks every field of \`node\` but \`loc\` (and \`type\` with \`skipType\`), and`, - " * every field of each object under it: the strings and nodes a recursive", - " * `for (const key in value)` walk with the same skips finds, read by name in", - " * the order ir.ts declares the fields. */", - `export function walk${short}Fields(node: ${name}, walker: IrFieldWalker, skipType: boolean): void {`, - ...nodeCall, - " switch (node.kind) {", - ); - const byKind = new Map(); - for (const member of members.get(name)) { - const kind = checker.getTypeOfSymbol(member.getProperty("kind")).value; - if (byKind.has(kind)) throw new Error(`${name} kind ${kind} declared twice`); - byKind.set(kind, member); + /** Whether a value of `type` can be a string. */ + function holdsString(type) { + if (type.isUnion()) return type.types.some(holdsString); + return (type.flags & (ts.TypeFlags.StringLike | ts.TypeFlags.Any | ts.TypeFlags.Unknown)) !== 0; } - for (const [kind, member] of [...byKind].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))) { + + /** Whether every value of `type` is a string. */ + function onlyString(type) { + if (type.isUnion()) return type.types.every(onlyString); + return (type.flags & ts.TypeFlags.StringLike) !== 0; + } + + /** Each kind's string fields, in declaration order, merged over the unions: + * a kind both unions declare lists the fields of both. */ + const kinds = new Map(); + for (const name of ["IrExpr", "IrStmt"]) { + const alias = aliases.get(name); + const union = checker.getTypeAtLocation(alias.name); + for (const member of union.isUnion() ? union.types : [union]) { + const kindSymbol = member.getProperty("kind"); + const kindType = checker.getTypeOfSymbolAtLocation(kindSymbol, alias); + if (!kindType.isStringLiteral()) throw new Error(`${name} member without a literal kind`); + const fields = kinds.get(kindType.value) ?? []; + for (const property of member.getProperties()) { + const type = checker.getTypeOfSymbolAtLocation(property, alias); + if (!holdsString(type)) continue; + const known = fields.find((field) => field.name === property.name); + const always = onlyString(type) && (property.flags & ts.SymbolFlags.Optional) === 0; + if (known === undefined) fields.push({ name: property.name, always }); + else known.always &&= always; + } + kinds.set(kindType.value, fields); + } + } + + const lines = [ + "// Generated from packages/compiler/src/ir/ir.ts by scripts/generate-ir-fields.mjs.", + "// Run `pnpm generate:ir-fields` after changing the fields of an IR node.", + 'import type { IrExpr, IrStmt, IrType } from "./ir.js";', + "", + "/** Calls `visit` with each string field of `node` (including `kind`) and its", + " * name: the string values `for (const key in node)` finds on a node, read", + " * by name in the order ir.ts declares them. */", + "export function eachStringField(", + " node: IrExpr | IrStmt,", + " visit: (value: string, field: string) => void,", + "): void {", + " switch (node.kind) {", + ]; + for (const [kind, fields] of [...kinds].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))) { lines.push(` case ${JSON.stringify(kind)}: {`); - const memberType = `Extract<${name}, { kind: ${JSON.stringify(kind)} }>`; - const scalarNames = []; - for (const property of member.getProperties()) { - if (property.name === "loc") continue; - const fieldType = checker.getTypeOfSymbol(property); - const field = `node.${property.name}`; - const fieldTypeExpr = `${memberType}[${JSON.stringify(property.name)}]`; - const walk = walkLines(field, fieldType, fieldTypeExpr, "", 0); - if (walk.length === 0) scalarNames.push(property.name); - if (walk.length === 0) continue; - if (property.name === "type") - lines.push(" if (!skipType) {", ...walk.map((l) => ` ${l}`), " }"); - else lines.push(...walk.map((l) => ` ${l}`)); + for (const { name, always } of fields) { + const read = `node.${name}`; + if (always) lines.push(` visit(${read}, ${JSON.stringify(name)});`); + else + lines.push( + ` if (typeof ${read} === "string") visit(${read}, ${JSON.stringify(name)});`, + ); } lines.push(" return;", " }"); - noteFields(memberType, member.getProperties(), scalarNames); } lines.push(" }", "}", ""); -} -// Type checking fails below when ir.ts gives a walked object a field this -// file does not know, or lets a field that held no string and no object hold -// one before this file is regenerated. -const quoted = (names) => names.map((name) => JSON.stringify(name)).join(" | "); -lines.push( - "/** A field ir.ts declares that this file does not read. */", - "type FieldsNotRead = Exclude;", - "/** The string or object values a field read as a scalar can hold. */", - "type ValuesNotRead = Extract, string | object>;", - "/** Fails type checking unless `T` is never: run `pnpm generate:ir-fields`. */", - "type Unread = T;", - "", - "/** Every walked object type's fields as ir.ts declared them when this file", - " * was generated. */", - "export type IrFieldsCurrent = [", -); -for (const { typeExpr, fields, scalars } of checks) { + // The deep walkers: every field of every object under an IR node, as a + // recursive key walk finds them, read by name. + const members = new Map(); + for (const name of ["IrExpr", "IrStmt", "IrType"]) { + const union = checker.getTypeAtLocation(aliases.get(name).name); + members.set(name, new Set(union.isUnion() ? union.types : [union])); + } + const nullish = ts.TypeFlags.Null | ts.TypeFlags.Undefined | ts.TypeFlags.Void; + const scalar = + ts.TypeFlags.NumberLike | + ts.TypeFlags.BooleanLike | + ts.TypeFlags.BigIntLike | + ts.TypeFlags.ESSymbolLike; + + /** Whether every value of `type` is a number, boolean, bigint, symbol, null + * or undefined. A record of such fields walks to nothing too, but it is an + * object: its own fields are checked instead. */ + function onlyScalar(type) { + const parts = type.isUnion() ? type.types : [type]; + return parts.every((part) => (part.flags & (scalar | nullish)) !== 0); + } + + /** The walker an IR union member belongs to, or null. */ + function irKind(type) { + for (const [name, set] of members) if (set.has(type)) return name; + return null; + } + + let temp = 0; + /** The fields each walked object type declares, for the type-level check at + * the end of the file: `typeExpr` names the object type, `fields` every + * field it had when the file was generated and `scalars` the fields that held + * no string and no object then. */ + const checks = []; + /** Records `property list` of `object` (named `typeExpr`) for the check. */ + function noteFields(typeExpr, properties, scalarNames) { + checks.push({ + typeExpr, + fields: properties.map((property) => property.name), + scalars: scalarNames, + }); + } + /** Lines that walk `expr` of `type` (named `typeExpr` in the generated file): + * strings to `walker.string`, IR nodes to their walkers, arrays by element and + * other objects by field. */ + function walkLines(expr, type, typeExpr, indent, depth) { + if (depth > 12) throw new Error(`field type nests too deep at ${expr}`); + const parts = type.isUnion() ? type.types : [type]; + const present = parts.filter((part) => (part.flags & nullish) === 0); + const optional = present.length !== parts.length; + const strings = present.filter((part) => (part.flags & ts.TypeFlags.StringLike) !== 0); + const objects = present.filter( + (part) => (part.flags & (ts.TypeFlags.StringLike | scalar)) === 0, + ); + for (const part of present) + if ((part.flags & (ts.TypeFlags.Any | ts.TypeFlags.Unknown)) !== 0) + throw new Error(`untyped field ${expr}`); + const body = []; + if (objects.length > 0) { + const kinds = new Set(objects.map(irKind)); + if (kinds.size === 1 && !kinds.has(null)) { + const [kind] = kinds; + body.push(`walk${kind.slice(2)}Fields(${expr}, walker, skipType);`); + } else if (objects.every((part) => checker.isArrayType(part))) { + if (objects.length !== 1) throw new Error(`several array types at ${expr}`); + const item = `item${temp++}`; + body.push(`for (const ${item} of ${expr}) {`); + const itemType = `Extract, readonly unknown[]>[number]`; + body.push( + ...walkLines(item, checker.getTypeArguments(objects[0])[0], itemType, " ", depth + 1), + ); + body.push("}"); + } else if (kinds.has(null) && objects.length === 1) { + const recordType = `Extract, object>`; + const properties = checker.getPropertiesOfType(objects[0]); + const scalarNames = []; + for (const property of properties) { + if (property.name === "loc") continue; + const fieldType = checker.getTypeOfSymbol(property); + const field = `${expr}.${property.name}`; + const fieldTypeExpr = `${recordType}[${JSON.stringify(property.name)}]`; + const lines = walkLines(field, fieldType, fieldTypeExpr, "", depth + 1); + if (onlyScalar(fieldType)) scalarNames.push(property.name); + if (lines.length === 0) continue; + if (property.name === "type") + body.push("if (!skipType) {", ...lines.map((l) => ` ${l}`), "}"); + else body.push(...lines); + } + noteFields(recordType, properties, scalarNames); + } else throw new Error(`mixed object types at ${expr}: ${checker.typeToString(type)}`); + } + let lines = body; + if (strings.length > 0 && objects.length > 0) + lines = [ + `if (typeof ${expr} === "string") walker.string(${expr});`, + ...(body.length > 0 ? [`else {`, ...body.map((l) => ` ${l}`), "}"] : []), + ]; + else if (strings.length > 0) lines = [`walker.string(${expr});`]; + if (lines.length > 0 && optional && !(strings.length > 0 && objects.length > 0)) + lines = [ + `if (${expr} !== undefined && ${expr} !== null) {`, + ...lines.map((l) => ` ${l}`), + "}", + ]; + else if (lines.length > 0 && optional) + lines = [ + `if (${expr} !== undefined && ${expr} !== null) {`, + ...lines.map((l) => ` ${l}`), + "}", + ]; + return lines.map((line) => indent + line); + } + lines.push( - ` Unread 0 ? quoted(fields) : "never"}>>,`, + "/** What a field walk reports (walkExprFields): each expression and", + " * statement node before its fields, and each string a field holds. */", + "export interface IrFieldWalker {", + " node(node: IrExpr | IrStmt): void;", + " string(value: string): void;", + "}", + "", ); - for (const scalarName of scalars) - lines.push(` Unread>,`); -} -lines.push("];", ""); -const text = lines.join("\n"); + for (const name of ["IrExpr", "IrStmt", "IrType"]) { + const short = name.slice(2); + const nodeCall = name === "IrType" ? [] : [" walker.node(node);"]; + lines.push( + `/** Walks every field of \`node\` but \`loc\` (and \`type\` with \`skipType\`), and`, + " * every field of each object under it: the strings and nodes a recursive", + " * `for (const key in value)` walk with the same skips finds, read by name in", + " * the order ir.ts declares the fields. */", + `export function walk${short}Fields(node: ${name}, walker: IrFieldWalker, skipType: boolean): void {`, + ...nodeCall, + " switch (node.kind) {", + ); + const byKind = new Map(); + for (const member of members.get(name)) { + const kind = checker.getTypeOfSymbol(member.getProperty("kind")).value; + if (byKind.has(kind)) throw new Error(`${name} kind ${kind} declared twice`); + byKind.set(kind, member); + } + for (const [kind, member] of [...byKind].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))) { + lines.push(` case ${JSON.stringify(kind)}: {`); + const memberType = `Extract<${name}, { kind: ${JSON.stringify(kind)} }>`; + const scalarNames = []; + for (const property of member.getProperties()) { + if (property.name === "loc") continue; + const fieldType = checker.getTypeOfSymbol(property); + const field = `node.${property.name}`; + const fieldTypeExpr = `${memberType}[${JSON.stringify(property.name)}]`; + const walk = walkLines(field, fieldType, fieldTypeExpr, "", 0); + if (onlyScalar(fieldType)) scalarNames.push(property.name); + if (walk.length === 0) continue; + if (property.name === "type") + lines.push(" if (!skipType) {", ...walk.map((l) => ` ${l}`), " }"); + else lines.push(...walk.map((l) => ` ${l}`)); + } + lines.push(" return;", " }"); + noteFields(memberType, member.getProperties(), scalarNames); + } + lines.push(" }", "}", ""); + } -if (process.argv.includes("--check")) { - if (readFileSync(target, "utf8") !== text) - throw new Error("fields.generated.ts is stale; run node scripts/generate-ir-fields.mjs"); -} else writeFileSync(target, text); + // Type checking fails below when ir.ts gives a walked object a field this + // file does not know, or lets a field that held no string and no object hold + // one before this file is regenerated. + const quoted = (names) => names.map((name) => JSON.stringify(name)).join(" | "); + lines.push( + "/** A field ir.ts declares that this file does not read. */", + "type FieldsNotRead = Exclude;", + "/** The string or object values a field read as a scalar can hold. */", + "type ValuesNotRead = Extract, string | object>;", + "/** Fails type checking unless `T` is never: run `pnpm generate:ir-fields`. */", + "type Unread = T;", + "", + "/** Every walked object type's fields as ir.ts declared them when this file", + " * was generated. */", + "export type IrFieldsCurrent = [", + ); + for (const { typeExpr, fields, scalars } of checks) { + lines.push( + ` Unread 0 ? quoted(fields) : "never"}>>,`, + ); + for (const scalarName of scalars) + lines.push(` Unread>,`); + } + lines.push("];", ""); + return lines.join("\n"); +} + +if (process.argv[1] !== undefined && samePath(process.argv[1], fileURLToPath(import.meta.url))) { + const text = generateIrFields(); + + if (process.argv.includes("--check")) { + if (readFileSync(target, "utf8") !== text) + throw new Error("fields.generated.ts is stale; run node scripts/generate-ir-fields.mjs"); + } else writeFileSync(target, text); +} diff --git a/tests/harness/ir-fields-generator.test.ts b/tests/harness/ir-fields-generator.test.ts new file mode 100644 index 000000000..f3cba57a1 --- /dev/null +++ b/tests/harness/ir-fields-generator.test.ts @@ -0,0 +1,60 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import ts from "typescript"; +import { expect, test } from "vitest"; +import { generateIrFields } from "../../scripts/generate-ir-fields.mjs"; + +const compilerDir = join(import.meta.dirname, "../../packages/compiler"); +const irPath = join(compilerDir, "src/ir/ir.ts"); +const generatedPath = join(compilerDir, "src/ir/fields.generated.ts"); +const irText = readFileSync(irPath, "utf8"); +const arrayGet = '{ kind: "arrayGet"; arr: IrExpr; index: IrExpr; type: IrType; loc: SrcLoc }'; + +/** ir.ts with `field` added to the arrayGet node. */ +function withArrayGetField(field: string): string { + expect(irText).toContain(arrayGet); + return irText.replace(arrayGet, arrayGet.replace("loc: SrcLoc }", `loc: SrcLoc; ${field} }`)); +} + +/** The type errors of `generated` as fields.generated.ts, with `ir` as ir.ts, + * under the compiler package's options. */ +function generatedErrors(ir: string, generated: string): string[] { + const parsed = ts.getParsedCommandLineOfConfigFile( + join(compilerDir, "tsconfig.json"), + { noEmit: true, incremental: false }, + { ...ts.sys, onUnRecoverableConfigFileDiagnostic: () => {} }, + )!; + const host = ts.createCompilerHost(parsed.options); + const readSourceFile = host.getSourceFile; + host.getSourceFile = (file, languageVersion, ...rest) => + file === irPath + ? ts.createSourceFile(file, ir, languageVersion) + : file === generatedPath + ? ts.createSourceFile(file, generated, languageVersion) + : readSourceFile.call(host, file, languageVersion, ...rest); + const program = ts.createProgram([generatedPath], parsed.options, host); + return ts + .getPreEmitDiagnostics(program, program.getSourceFile(generatedPath)) + .map((d) => ts.flattenDiagnosticMessageText(d.messageText, "\n")); +} + +test("a nested record of scalar fields passes the checks once the readers are regenerated", () => { + const ir = withArrayGetField("counts?: { hits: number; misses: number | undefined }"); + expect(generatedErrors(ir, generateIrFields(ir))).toEqual([]); +}); + +test("the record's own fields are checked: an added field or a widened scalar fails with its name", () => { + const generated = generateIrFields( + withArrayGetField("counts?: { hits: number; misses: number | undefined }"), + ); + const added = generatedErrors( + withArrayGetField("counts?: { hits: number; misses: number | undefined; label: string }"), + generated, + ); + expect(added).toEqual([`Type '"label"' does not satisfy the constraint 'never'.`]); + const widened = generatedErrors( + withArrayGetField("counts?: { hits: number | string; misses: number | undefined }"), + generated, + ); + expect(widened).toEqual([`Type 'string' does not satisfy the constraint 'never'.`]); +});