From d9c1b89b5d2cb083d32f0ce4af79ff398bf9f3ea Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Sun, 25 Oct 2026 02:54:46 -0700 Subject: [PATCH 1/4] Cut the shape overlay's compile cost; keep package JSDoc it does not replace Compile cost (Hono fixture: 53 s -> 99 s with the overlay before): - The overlay is planned once per program text: a reload that only drops refused annotations renders the cached plan (overlay.ts plan/render split) and reuses the cached structural rewrites, instead of analyzing, reporting and planning again. - The overlay's analyses run without per-call-site clones, which only the constant-graph report uses so far (SCRIPTC_SHAPE_SITE_CLONES opts in). - A failing round also drops what the deferred refusals blame, saving a reload; deferred refusals no longer blame every annotation of their enclosing function (most of them exist without the overlay). - A parameter whose `typeof` tests the proven values never pass stays untyped: the typed parameter would narrow to never in that branch, which the lowering refuses (zod 4's Doc.write). - Deferred refusals in methods blame the arguments of the method's call sites (implicit instantiations), like those in functions. Package JSDoc: the overlay neutralized the package's JSDoc in every overlaid file, so declarations it does not type lost their documented types and the lowering instantiated them per call site instead. In commander, this changed which refusal `--help` reaches: the run passed the refusal the default build stops at and failed later, in Command.helpInformation, at a checked conversion of the package's own documented contract (a help width that is undefined off a TTY). Nothing the analysis proved was contradicted (SCRIPTC_SHAPE_CHECK_LOG logs no inferred-fact failure). Only the leading JSDoc of the declarations the overlay annotates is neutralized now; commander's overlay build fails exactly where its default build does, and the OVERLAY_KNOWN entries go. The overlay npm suite (134 cases), the shape tests, lint and format pass. Co-Authored-By: Claude Opus 5.5 --- packages/compiler/src/frontend/pipeline.ts | 14 +- .../src/frontend/shape-inference/analyze.ts | 18 ++- .../src/frontend/shape-inference/fallback.ts | 19 ++- .../src/frontend/shape-inference/index.ts | 110 ++++++++++--- .../src/frontend/shape-inference/overlay.ts | 152 +++++++++++++++--- tests/harness/npm-suite.ts | 11 +- 6 files changed, 265 insertions(+), 59 deletions(-) diff --git a/packages/compiler/src/frontend/pipeline.ts b/packages/compiler/src/frontend/pipeline.ts index 48d4161f9..bb1053cb5 100644 --- a/packages/compiler/src/frontend/pipeline.ts +++ b/packages/compiler/src/frontend/pipeline.ts @@ -739,23 +739,35 @@ function loadFrontend( level = level === "annotations" ? "files" : "all"; changed = excludeRefusedOverlay(finalLoad.program, refusals, level); } + // The same round also drops what the deferred refusals blame, + // saving a reload. + if (level !== "all" && lowered.runtimeFences.length > 0) + changed = + excludeRefusedOverlay(finalLoad.program, lowered.runtimeFences, "fences") || changed; } if (process.env["SCRIPTC_DEBUG_SHAPE_FALLBACK"] !== undefined) process.stderr.write( `[shape-fallback] round ${round} level ${level}: ${refusals.length} refusals (${refusals - .slice(0, 3) + .slice(0, process.env["SCRIPTC_DEBUG_SHAPE_FALLBACK"] === "all" ? undefined : 3) .map((d) => `${d.loc.file.split("/").pop()}:${d.loc.start} ${d.message.slice(0, 60)}`) .join("; ")}), changed ${changed}\n`, ); if (!changed) break; + const t0 = performance.now(); finalLoad.dispose(); finalLoad = loadProgram(entryPath, { npmStatic: finalNpmStatic, externalTypes }); + const t1 = performance.now(); // The preflight fills the module order; the tsc world reads the // files as written, so it answers as the first load did. const reloadPreflight = checkPreflight(finalLoad); if (reloadPreflight.length > 0) return { ...lowered, module: null, diagnostics: reloadPreflight }; + const t2 = performance.now(); lowered = lowerOnce(); + if (process.env["SCRIPTC_DEBUG_SHAPE_FALLBACK"] !== undefined) + process.stderr.write( + `[shape-fallback] reload ${(t1 - t0).toFixed(0)} ms, preflight ${(t2 - t1).toFixed(0)} ms, lower ${(performance.now() - t2).toFixed(0)} ms\n`, + ); } // The declaration-typed boundary's notes (SC6102) join the // performance warnings; they never fail a build. diff --git a/packages/compiler/src/frontend/shape-inference/analyze.ts b/packages/compiler/src/frontend/shape-inference/analyze.ts index d783a2ad7..2dd53079c 100644 --- a/packages/compiler/src/frontend/shape-inference/analyze.ts +++ b/packages/compiler/src/frontend/shape-inference/analyze.ts @@ -358,8 +358,17 @@ const LOOP_CALLBACK_METHODS: ReadonlySet = new Set([ "replaceAll", ]); -export function analyzeShapes(program: ts.Program, checker: ts.TypeChecker): ShapeAnalysis { - return new Analyzer(program, checker).run(); +export interface AnalyzeOptions { + /** Per-call-site clones per function (default SCRIPTC_SHAPE_SITE_CLONES or 64). */ + readonly siteClones?: number; +} + +export function analyzeShapes( + program: ts.Program, + checker: ts.TypeChecker, + options: AnalyzeOptions = {}, +): ShapeAnalysis { + return new Analyzer(program, checker, options.siteClones ?? SITE_CLONES).run(); } class Analyzer { @@ -421,6 +430,7 @@ class Analyzer { constructor( private readonly program: ts.Program, private readonly checker: ts.TypeChecker, + private readonly siteCloneLimit: number, ) { this.top = this.atoms.intern("top", () => ({ kind: "top", label: "⊤" })); } @@ -1361,7 +1371,7 @@ class Analyzer { return inside ? this.ctx : ""; } private siteClone(info: FunctionInfo, site: Node, args: number[]): FunctionInfo | null { - if (SITE_CLONES === 0 || info.isGenerator || ts.isSourceFile(info.node)) return null; + if (this.siteCloneLimit === 0 || info.isGenerator || ts.isSourceFile(info.node)) return null; if (ts.isClassLikeDeclaration(info.node)) return null; const sc = this.argsFrom.get(args); if (sc === undefined || sc.inLoop || this.forceLoop || this.repeatingSites.has(site)) @@ -1380,7 +1390,7 @@ class Analyzer { const existing = this.functions.get(`${ctx}|${this.nodeKey(info.node)}`); if (existing !== undefined) return existing; const count = this.siteClones.get(info.node) ?? 0; - if (count >= SITE_CLONES) return null; + if (count >= this.siteCloneLimit) return null; this.siteClones.set(info.node, count + 1); return this.functionInfo(info.node, info.ownerClass, ctx); } diff --git a/packages/compiler/src/frontend/shape-inference/fallback.ts b/packages/compiler/src/frontend/shape-inference/fallback.ts index 6a2824990..50857fcfb 100644 --- a/packages/compiler/src/frontend/shape-inference/fallback.ts +++ b/packages/compiler/src/frontend/shape-inference/fallback.ts @@ -35,6 +35,10 @@ export function attributeOverlayRefusals( program: ts.Program, diagnostics: readonly ScrDiagnostic[], overlayFiles: ReadonlySet, + /** Failing refusals may blame every annotation in their enclosing + * function as a last resort; refusals deferred to run time do not (most + * exist without the overlay too). */ + wholeFunctions = true, ): OverlayAttribution { const annotations = new Map>(); const files = new Set(); @@ -124,10 +128,17 @@ export function attributeOverlayRefusals( const sf = program.getSourceFile(file); if (sf === undefined) continue; const visit = (n: ts.Node): void => { - if (ts.isCallExpression(n) && ts.isIdentifier(n.expression)) { + const callee = !ts.isCallExpression(n) + ? undefined + : ts.isIdentifier(n.expression) + ? n.expression + : ts.isPropertyAccessExpression(n.expression) && ts.isIdentifier(n.expression.name) + ? n.expression.name + : undefined; + if (callee !== undefined && ts.isCallExpression(n)) { let decl: ts.Node | undefined; try { - decl = checker!.getSymbolAtLocation(n.expression)?.declarations[0]; + decl = checker!.getSymbolAtLocation(callee)?.declarations[0]; } catch { decl = undefined; } @@ -177,7 +188,7 @@ export function attributeOverlayRefusals( } // Still nothing: the lowering of the enclosing function changed as a // whole (an instantiation the annotations inside it made possible). - if (blamed === before) { + if (blamed === before && wholeFunctions) { const fn = enclosingFunctionLike(node); if (fn !== undefined && overlayFiles.has(fn.getSourceFile().fileName)) { const visit = (n: ts.Node): void => { @@ -197,7 +208,7 @@ export function attributeOverlayRefusals( function enclosingFunction(node: ts.Node): ts.Node | undefined { for (let n: ts.Node | undefined = node.parent; n !== undefined; n = n.parent) - if (ts.isFunctionDeclaration(n)) return n; + if (ts.isFunctionDeclaration(n) || ts.isMethodDeclaration(n)) return n; else if (ts.isFunctionLike(n)) return undefined; return undefined; } diff --git a/packages/compiler/src/frontend/shape-inference/index.ts b/packages/compiler/src/frontend/shape-inference/index.ts index d8074a0a8..38dce5143 100644 --- a/packages/compiler/src/frontend/shape-inference/index.ts +++ b/packages/compiler/src/frontend/shape-inference/index.ts @@ -6,10 +6,17 @@ import type { IrFunction } from "../../ir/ir.js"; import type { ScrDiagnostic } from "../../diagnostics/diagnostic.js"; import { npmStaticPackageOfPath } from "../npm-static.js"; import { isShapelessPackage } from "../package-shims.js"; -import { analyzeShapes, isPackageJsFile } from "./analyze.js"; -import { OVERLAY_MARK, planShapeOverlay, type OverlayExclusions } from "./overlay.js"; +import { analyzeShapes, isPackageJsFile, type ShapeAnalysis } from "./analyze.js"; +import { createHash } from "node:crypto"; +import { + OVERLAY_MARK, + planShapeOverlay, + renderShapeOverlay, + type OverlayExclusions, + type ShapeOverlayPlan, +} from "./overlay.js"; import { attributeOverlayRefusals } from "./fallback.js"; -import { planStaticObjectRewrites } from "./static-objects.js"; +import { planStaticObjectRewrites, type StaticObjectRewrites } from "./static-objects.js"; import { isJsSourceFile } from "../program.js"; import { buildShapeReport } from "./report.js"; import { computeConstantGraphs, constantGraphReport } from "./constant-graphs.js"; @@ -93,6 +100,7 @@ let droppedAnnotations = 0; /** Starts a build: no annotation is excluded yet. */ export function beginShapeOverlayBuild(): void { + overlayCache = new Map(); exclusions = { annotations: new Map(), files: new Set() }; overlayDropped = false; droppedAnnotations = 0; @@ -127,7 +135,12 @@ export function excludeRefusedOverlay( droppedAnnotations = overlayAnnotations; return true; } - const blame = attributeOverlayRefusals(program, diagnostics, files); + const blame = attributeOverlayRefusals(program, diagnostics, files, level !== "fences"); + if (process.env["SCRIPTC_DEBUG_SHAPE_FALLBACK"] === "all") + for (const [file, ids] of blame.annotations) + process.stderr.write( + `[shape-fallback] ${level} blames ${file.split("/").slice(-2).join("/")}: ${[...ids].join(",")}\n`, + ); let changed = false; for (const file of blame.files) { const ids = blame.annotations.get(file); @@ -158,6 +171,44 @@ function overlayExclusions(): OverlayExclusions { return exclusions; } +/* Per build, the overlay's analysis-dependent results by program text and + * excluded files: a reload that only drops refused annotations renders the + * cached plan instead of analyzing again (fallback.ts). */ +interface OverlayCacheEntry { + budgetExceeded?: true; + rewrites?: StaticObjectRewrites; + plan?: ShapeOverlayPlan; +} +let overlayCache = new Map(); + +function programTextKey(program: ts.Program): string { + const hash = createHash("sha256"); + for (const sf of program.getSourceFiles()) { + if (sf.isDeclarationFile) continue; + hash.update(sf.fileName).update("\0").update(sf.text).update("\0"); + } + return hash.digest("hex"); +} + +/** Per-call-site clones serve the constant-graph facts (stage 3); the + * overlay's rewrites and types do not use them yet, so its analyses run + * without them unless SCRIPTC_SHAPE_SITE_CLONES asks. */ +function overlaySiteClones(): number { + const env = process.env["SCRIPTC_SHAPE_SITE_CLONES"]; + return env === undefined ? 0 : Number(env); +} + +function overlayJsFiles(analysis: ShapeAnalysis): ts.SourceFile[] { + // Package JavaScript of packages that keep shapes on, and the program's + // own JavaScript files. + return analysis.files.filter((sf) => { + if (sf.isDeclarationFile || !isJsSourceFile(sf)) return false; + if (!isPackageJsFile(sf)) return !sf.fileName.includes("/node_modules/"); + const pkg = npmStaticPackageOfPath(sf.fileName); + return pkg === null || !isShapelessPackage(pkg); + }); +} + /** loadProgram7's overlayer: analyze, plan, and register the overlay files. */ export function shapeOverlayer( program: ts.Program, @@ -165,23 +216,34 @@ export function shapeOverlayer( ): ReadonlyMap | null { if (round === 0) resetShapeOverlay(); if (overlayDropped) return null; - const analysis = analyzeShapes(program, program.getTypeChecker()); - if (analysis.budgetExceeded) return null; - // Package JavaScript of packages that keep shapes on, and the program's - // own JavaScript files. - const jsFiles = analysis.files.filter((sf) => { - if (sf.isDeclarationFile || !isJsSourceFile(sf)) return false; - if (!isPackageJsFile(sf)) return !sf.fileName.includes("/node_modules/"); - const pkg = npmStaticPackageOfPath(sf.fileName); - return pkg === null || !isShapelessPackage(pkg); - }); + const key = `${programTextKey(program)}|${[...exclusions.files].sort().join(",")}`; + const cached = overlayCache.get(key); + const entry: OverlayCacheEntry = cached ?? {}; + if (cached === undefined) overlayCache.set(key, entry); + if (entry.budgetExceeded === true) return null; + let analysis: ShapeAnalysis | null = null; + const analyzed = (): ShapeAnalysis | null => { + analysis ??= analyzeShapes(program, program.getTypeChecker(), { + siteClones: overlaySiteClones(), + }); + if (analysis.budgetExceeded) { + entry.budgetExceeded = true; + return null; + } + return analysis; + }; // Round 0: the structural rewrites (static-objects.ts); the program is // rebuilt from them and analyzed again before the typed overlay. if (round === 0 && process.env["SCRIPTC_SHAPE_STATIC_OBJECTS"] !== "0") { - const rewrites = planStaticObjectRewrites( - analysis, - jsFiles.filter((sf) => !exclusions.files.has(sf.fileName)), - ); + if (entry.rewrites === undefined) { + const a = analyzed(); + if (a === null) return null; + entry.rewrites = planStaticObjectRewrites( + a, + overlayJsFiles(a).filter((sf) => !exclusions.files.has(sf.fileName)), + ); + } + const rewrites = entry.rewrites; staticRewriteFiles = new Set(rewrites.files.keys()); staticObjectStats = { classes: rewrites.classes, @@ -198,8 +260,16 @@ export function shapeOverlayer( if (rewrites.files.size > 0) return rewrites.files; } if (round > 1) return null; - const report = buildShapeReport(analysis, "1"); - const overlay = planShapeOverlay(analysis, report.sites, jsFiles, overlayExclusions()); + if (entry.plan === undefined) { + const a = analyzed(); + if (a === null) return null; + const report = buildShapeReport(a, "1"); + entry.plan = planShapeOverlay(a, report.sites, overlayJsFiles(a), { + annotations: new Map(), + files: exclusions.files, + }); + } + const overlay = renderShapeOverlay(entry.plan, overlayExclusions()); overlayFiles = new Set(overlay.files.keys()); overlayAnnotations = overlay.annotations; overlayDeadBranches = overlay.deadBranches; diff --git a/packages/compiler/src/frontend/shape-inference/overlay.ts b/packages/compiler/src/frontend/shape-inference/overlay.ts index 023f55deb..2fa99de13 100644 --- a/packages/compiler/src/frontend/shape-inference/overlay.ts +++ b/packages/compiler/src/frontend/shape-inference/overlay.ts @@ -35,6 +35,27 @@ interface Edit { readonly text: string; /** Replaces pos..end instead of inserting at pos. */ readonly end?: number; + /** The annotation (or dead branch) the edit belongs to. */ + readonly id?: number; +} + +/** The overlay before annotation exclusions apply: per file, its text, + * edits and appended typedefs. It depends only on the analysis, so a build + * that drops refused annotations renders it again instead of analyzing + * again (renderShapeOverlay). */ +export interface ShapeOverlayPlan { + readonly parts: Map; +} + +interface OverlayPart { + readonly text: string; + readonly edits: Edit[]; + readonly suffix: string; + /** The leading trivia of each annotated declaration, by annotation id: + * the package's own JSDoc there is neutralized while the annotation is + * kept (package JSDoc is a hint; elsewhere it types the code as it does + * without the overlay). */ + readonly docRanges: { lo: number; hi: number; id: number }[]; } /** Marks the overlay's own comments (an unknown JSDoc tag, ignored). */ @@ -76,7 +97,7 @@ export function planShapeOverlay( verdicts: readonly SiteVerdict[], packageFiles: readonly ts.SourceFile[], exclusions: OverlayExclusions = NO_EXCLUSIONS, -): ShapeOverlay { +): ShapeOverlayPlan { const pSites = new Map(); const siteVerdicts = new Set(verdicts.map((v) => v.site.atom)); // Records and arrays print only for sites whose objects never escape: a @@ -93,11 +114,9 @@ export function planShapeOverlay( siteKeys.set(v.site.atom, keys); pSites.set(v.site.atom, v); } - const files = new Map(); + const parts = new Map(); const overlaid = new Set(packageFiles); const exportedClasses = new Map>(); - let annotations = 0; - let deadBranches = 0; debugIndex = 0; // `if` branches no condition value allowed, per file: the overlay // replaces each with a throwing check, so the checker and the lowering @@ -118,15 +137,18 @@ export function planShapeOverlay( const typedefs = new Map(); const typedefText: string[] = []; const edits: Edit[] = []; - const dropped = exclusions.annotations.get(sf.fileName); // One annotation: a JSDoc comment at `pos` carrying MARK and the // annotation's id (its position), so a refused lowering can name it and // the next load drop it. `close`: the text that closes a type cast. - const annotate = (pos: number, tags: string, close?: number): void => { - if (dropped?.has(pos) === true) return; - edits.push({ pos, text: `/** ${tags} ${MARK} #${pos} */ ${close !== undefined ? "(" : ""}` }); - if (close !== undefined) edits.push({ pos: close, text: ")" }); - annotations++; + const docRanges: { lo: number; hi: number; id: number }[] = []; + const annotate = (pos: number, tags: string, close?: number, from?: number): void => { + if (from !== undefined && from < pos) docRanges.push({ lo: from, hi: pos, id: pos }); + edits.push({ + pos, + id: pos, + text: `/** ${tags} ${MARK} #${pos} */ ${close !== undefined ? "(" : ""}`, + }); + if (close !== undefined) edits.push({ pos: close, id: pos, text: ")" }); }; const className = (decl: ts.Node): string | null => { @@ -461,7 +483,12 @@ export function planShapeOverlay( p.questionToken !== undefined ) return; - const t = typeOf(paramValues(fn, index), false, 0); + const values = paramValues(fn, index); + const t = typeOf(values, false, 0); + // A `typeof p === "…"` test the proven values never pass would + // narrow the typed parameter to never inside the branch, which the + // lowering refuses: such a parameter stays untyped. + if (t !== null && typeofTestsOutside(fn, p.name.text, values)) return; if (t !== null) tags.push(`@param {${t}} ${p.name.text}`); }); return tags.length > 0 ? tags.join(" ") : null; @@ -482,7 +509,7 @@ export function planShapeOverlay( ) { if (node.body !== undefined) { const tags = paramTags(node); - if (tags !== null) annotate(ts.tokenStartOfNode(node), tags); + if (tags !== null) annotate(ts.tokenStartOfNode(node), tags, undefined, node.pos); } // `this.f = …` fields of a constructor's top-level statements. if (ts.isConstructorDeclaration(node) && node.body !== undefined) { @@ -505,7 +532,7 @@ export function planShapeOverlay( if (seen.has(name)) continue; seen.add(name); const t = cls !== undefined ? fieldType(cls, name) : null; - if (t !== null) annotate(ts.tokenStartOfNode(st), `@type {${t}}`); + if (t !== null) annotate(ts.tokenStartOfNode(st), `@type {${t}}`, undefined, st.pos); } } } @@ -521,7 +548,7 @@ export function planShapeOverlay( const init = d.initializer; if (ts.isIdentifier(d.name) && init !== undefined && !literalInitializer(init)) { const t = typeOf(analysis.declValues(d), false, 0); - if (t !== null) annotate(ts.tokenStartOfNode(node), `@type {${t}}`); + if (t !== null) annotate(ts.tokenStartOfNode(node), `@type {${t}}`, undefined, node.pos); } } ts.forEachChild(node, (c) => { @@ -551,7 +578,6 @@ export function planShapeOverlay( const ranges: [number, number][] = []; for (const branch of dead) { const start = ts.tokenStartOfNode(branch); - if (dropped?.has(start) === true) continue; if (ranges.some(([lo, hi]) => start >= lo && start < hi)) continue; ranges.push([start, branch.end]); } @@ -570,9 +596,9 @@ export function planShapeOverlay( edits.push({ pos: lo, end: hi, + id: lo, text: `{ /** ${MARK} #${lo} */ throw new Error(${JSON.stringify(message)}); }`, }); - deadBranches++; } } // Debugging: SCRIPTC_SHAPE_OVERLAY_ONLY=a-b keeps the annotations @@ -589,7 +615,7 @@ export function planShapeOverlay( edits.push(...kept); } if (edits.length === 0) continue; - files.set(sf.fileName, neutralizeDocTags(applyEdits(sf.text, edits)) + typedefText.join("")); + parts.set(sf.fileName, { text: sf.text, edits, suffix: typedefText.join(""), docRanges }); } // Type aliases other files name these files' classes through. for (const [other, names] of exportedClasses) { @@ -597,14 +623,98 @@ export function planShapeOverlay( .sort() .map((n) => `/** @typedef {${n}} __shape_class_${n} ${MARK} */`) .join(""); - files.set( - other.fileName, - (files.get(other.fileName) ?? neutralizeDocTags(other.text)) + "\n" + aliases, - ); + const part = parts.get(other.fileName); + if (part !== undefined) + parts.set(other.fileName, { ...part, suffix: `${part.suffix}\n${aliases}` }); + else + parts.set(other.fileName, { + text: other.text, + edits: [], + suffix: `\n${aliases}`, + docRanges: [], + }); + } + return { parts }; +} + +/** The overlay's files with the annotations a build dropped left out. */ +export function renderShapeOverlay( + plan: ShapeOverlayPlan, + exclusions: OverlayExclusions = NO_EXCLUSIONS, +): ShapeOverlay { + const files = new Map(); + let annotations = 0; + let deadBranches = 0; + for (const [file, part] of plan.parts) { + const dropped = exclusions.annotations.get(file); + const edits = part.edits.filter((e) => e.id === undefined || dropped?.has(e.id) !== true); + for (const e of edits) { + if (e.id === undefined || e.text === ")") continue; + if (e.end !== undefined) deadBranches++; + else annotations++; + } + if (edits.length === 0 && part.suffix === "") continue; + let text = part.text; + for (const r of part.docRanges) + if (dropped?.has(r.id) !== true) + text = text.slice(0, r.lo) + neutralizeDocTags(text.slice(r.lo, r.hi)) + text.slice(r.hi); + files.set(file, applyEdits(text, edits) + part.suffix); } return { files, annotations, deadBranches }; } +/** The `typeof` results the values can have. */ +function typeofResults(values: ReadonlySet): Set { + const out = new Set(); + for (const a of values) { + if (a.kind === "prim") + out.add(a.label === "null" ? "object" : a.label === "symbol" ? "symbol" : a.label); + else if (a.kind === "function" || a.kind === "class") out.add("function"); + else out.add("object"); + } + return out; +} + +/** Whether the function's body compares `typeof name` with a result the + * values never have. */ +function typeofTestsOutside( + fn: ts.FunctionLikeDeclaration, + name: string, + values: ReadonlySet, +): boolean { + const results = typeofResults(values); + let outside = false; + const visit = (n: ts.Node): void => { + if (outside) return; + if ( + ts.isBinaryExpression(n) && + (n.operatorToken.kind === ts.SyntaxKind.EqualsEqualsEqualsToken || + n.operatorToken.kind === ts.SyntaxKind.EqualsEqualsToken || + n.operatorToken.kind === ts.SyntaxKind.ExclamationEqualsEqualsToken || + n.operatorToken.kind === ts.SyntaxKind.ExclamationEqualsToken) + ) { + for (const [op, lit] of [ + [n.left, n.right], + [n.right, n.left], + ] as const) + if ( + ts.isTypeOfExpression(op) && + ts.isIdentifier(op.expression) && + op.expression.text === name && + ts.isStringLiteral(lit) && + !results.has(lit.text) + ) + outside = true; + } + ts.forEachChild(n, (c) => { + visit(c); + return undefined; + }); + }; + if (fn.body !== undefined) visit(fn.body); + return outside; +} + function packageNameOf(fileName: string): string { const parts = (fileName.split("/node_modules/").pop() ?? "").split("/"); return parts[0]!.startsWith("@") ? `${parts[0]}/${parts[1]}` : parts[0]!; diff --git a/tests/harness/npm-suite.ts b/tests/harness/npm-suite.ts index b0f9bc536..d5bb5beee 100644 --- a/tests/harness/npm-suite.ts +++ b/tests/harness/npm-suite.ts @@ -112,15 +112,8 @@ const KNOWN: Record = { "workspace-unclean": { build: "SC2013" }, }; -/** Where the overlay changes how a case KNOWN refuses: the operation it - * fails at on an argv list that fails in both builds. */ -const OVERLAY_KNOWN: Record = { - // `--help` reaches an unsupported operation in both builds; with the - // overlay it fails at a checked conversion of an option value first - // (results/design-shape-inference/NOTES.md, open item). - "commander-calc": { run: "expected number at $, got undefined", argvs: [["--help"]] }, - "commander-calc-async": { run: "expected number at $, got undefined", argvs: [["--help"]] }, -}; +/** Where the overlay changes how a case KNOWN refuses (none today). */ +const OVERLAY_KNOWN: Record = {}; /** Per-case timeouts past the default 120 s: the undici cases compile the * built-in undici implementation's module graph, the zod 3 cases (zod3, From e324c07a6610f8a4c59f52d28a505e7cdbc5202b Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 10:02:52 -0500 Subject: [PATCH 2/4] Neutralize the overlay's doc ranges in one pass over the file renderShapeOverlay rebuilt the whole source text once per JSDoc range, so a file with many annotations cost time quadratic in its size. The ranges are leading trivia of declarations, equal or disjoint, so the text is now built once from the ranges in order. --- .../src/frontend/shape-inference/overlay.ts | 25 +++++++-- .../test/shape-overlay-render.test.ts | 53 +++++++++++++++++++ 2 files changed, 73 insertions(+), 5 deletions(-) create mode 100644 packages/compiler/test/shape-overlay-render.test.ts diff --git a/packages/compiler/src/frontend/shape-inference/overlay.ts b/packages/compiler/src/frontend/shape-inference/overlay.ts index 2fa99de13..24fdcefd9 100644 --- a/packages/compiler/src/frontend/shape-inference/overlay.ts +++ b/packages/compiler/src/frontend/shape-inference/overlay.ts @@ -654,15 +654,30 @@ export function renderShapeOverlay( else annotations++; } if (edits.length === 0 && part.suffix === "") continue; - let text = part.text; - for (const r of part.docRanges) - if (dropped?.has(r.id) !== true) - text = text.slice(0, r.lo) + neutralizeDocTags(text.slice(r.lo, r.hi)) + text.slice(r.hi); - files.set(file, applyEdits(text, edits) + part.suffix); + files.set(file, applyEdits(neutralizedDocRanges(part, dropped), edits) + part.suffix); } return { files, annotations, deadBranches }; } +/** The part's text with the package JSDoc neutralized in the leading trivia + * of the annotations a build kept. Each range is one declaration's leading + * trivia, so two ranges are equal or disjoint; the text is built once. */ +function neutralizedDocRanges(part: OverlayPart, dropped: ReadonlySet | undefined): string { + const ranges = part.docRanges + .filter((r) => dropped?.has(r.id) !== true) + .sort((a, b) => a.lo - b.lo || a.hi - b.hi); + if (ranges.length === 0) return part.text; + let out = ""; + let at = 0; + for (const r of ranges) { + const lo = Math.max(r.lo, at); + if (r.hi <= lo) continue; + out += part.text.slice(at, lo) + neutralizeDocTags(part.text.slice(lo, r.hi)); + at = r.hi; + } + return out + part.text.slice(at); +} + /** The `typeof` results the values can have. */ function typeofResults(values: ReadonlySet): Set { const out = new Set(); diff --git a/packages/compiler/test/shape-overlay-render.test.ts b/packages/compiler/test/shape-overlay-render.test.ts new file mode 100644 index 000000000..8aa539a74 --- /dev/null +++ b/packages/compiler/test/shape-overlay-render.test.ts @@ -0,0 +1,53 @@ +/* renderShapeOverlay neutralizes the package JSDoc in the leading trivia of + * the annotations a build keeps, and leaves the trivia of dropped ones. */ +import { expect, test } from "vitest"; +import { + neutralizeDocTags, + OVERLAY_MARK, + renderShapeOverlay, +} from "../src/frontend/shape-inference/overlay.js"; +import type { ShapeOverlayPlan } from "../src/frontend/shape-inference/overlay.js"; + +test("each kept annotation's leading JSDoc is neutralized, once per range", () => { + const decls = [ + "/** @param {string} a */\nfunction f(a) {}\n", + "/** @type {number} */\nvar x = 1;\n", + "/** @returns {void} */\nfunction g() {}\n", + ]; + const text = decls.join(""); + const docRanges: { lo: number; hi: number; id: number }[] = []; + const edits: { pos: number; id: number; text: string }[] = []; + let lo = 0; + for (const decl of decls) { + const hi = lo + decl.indexOf("\n") + 1; + docRanges.push({ lo, hi, id: hi }); + // A declaration's first child starts at the same position: its range is + // the same trivia. + docRanges.push({ lo, hi, id: hi }); + edits.push({ pos: hi, id: hi, text: `/** @type {any} ${OVERLAY_MARK} #${hi} */ ` }); + lo += decl.length; + } + const plan: ShapeOverlayPlan = { + parts: new Map([["a.js", { text, edits, suffix: "", docRanges }]]), + }; + const dropped = docRanges[2]!.id; + const overlay = renderShapeOverlay(plan, { + annotations: new Map([["a.js", new Set([dropped])]]), + files: new Set(), + }); + const annotated = (decl: string, keep: boolean, at: number): string => { + const head = decl.slice(0, decl.indexOf("\n") + 1); + const rest = decl.slice(head.length); + return keep + ? `${neutralizeDocTags(head)}/** @type {any} ${OVERLAY_MARK} #${at} */ ${rest}` + : decl; + }; + expect(overlay.files.get("a.js")).toBe( + decls + .map((decl, i) => annotated(decl, docRanges[2 * i]!.id !== dropped, docRanges[2 * i]!.id)) + .join(""), + ); + expect(overlay.files.get("a.js")).toContain("/** @type {number} */"); + expect(overlay.files.get("a.js")).toContain("/** -param {string} a */"); + expect(overlay.annotations).toBe(2); +}); From 3d886579b2f25b213a33eb9ea6de82fdb9e01745 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Sun, 25 Oct 2026 14:50:35 -0700 Subject: [PATCH 3/4] Remove the committed node_modules symlink Co-Authored-By: Claude Opus 5.5 --- packages/compiler/node_modules | 1 - 1 file changed, 1 deletion(-) delete mode 120000 packages/compiler/node_modules diff --git a/packages/compiler/node_modules b/packages/compiler/node_modules deleted file mode 120000 index 6b01b0eda..000000000 --- a/packages/compiler/node_modules +++ /dev/null @@ -1 +0,0 @@ -../../../shape-reg/packages/compiler/node_modules \ No newline at end of file From 53896238c46b3d5fc0ed3d726a75d0edad85224d Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Mon, 26 Oct 2026 12:18:56 -0500 Subject: [PATCH 4/4] Regenerate the IR field readers and fix the shard tests --- .../src/frontend/shape-inference/overlay.ts | 114 +++++++++++++++--- packages/compiler/src/ir/fields.generated.ts | 4 + .../test/shape-overlay-render.test.ts | 30 +++++ 3 files changed, 132 insertions(+), 16 deletions(-) diff --git a/packages/compiler/src/frontend/shape-inference/overlay.ts b/packages/compiler/src/frontend/shape-inference/overlay.ts index 9b35ac8a4..a0d28b87c 100644 --- a/packages/compiler/src/frontend/shape-inference/overlay.ts +++ b/packages/compiler/src/frontend/shape-inference/overlay.ts @@ -55,7 +55,23 @@ interface OverlayPart { * the package's own JSDoc there is neutralized while the annotation is * kept (package JSDoc is a hint; elsewhere it types the code as it does * without the overlay). */ - readonly docRanges: { lo: number; hi: number; id: number }[]; + readonly docRanges: DocRange[]; +} + +interface DocRange { + readonly lo: number; + readonly hi: number; + readonly id: number; + /** For a function's annotation, the tags it writes: only the package's + * tags for the same result and parameters (and a `@type` or `@overload` + * signature) are neutralized, so the parameters it leaves untyped keep + * their documented types. Unset: every type tag in the range is. */ + readonly replaces?: FunctionTags; +} + +interface FunctionTags { + readonly returns: boolean; + readonly params: readonly string[]; } /** Marks the overlay's own comments (an unknown JSDoc tag, ignored). */ @@ -140,9 +156,21 @@ export function planShapeOverlay( // One annotation: a JSDoc comment at `pos` carrying MARK and the // annotation's id (its position), so a refused lowering can name it and // the next load drop it. `close`: the text that closes a type cast. - const docRanges: { lo: number; hi: number; id: number }[] = []; - const annotate = (pos: number, tags: string, close?: number, from?: number): void => { - if (from !== undefined && from < pos) docRanges.push({ lo: from, hi: pos, id: pos }); + const docRanges: DocRange[] = []; + const annotate = ( + pos: number, + tags: string, + close?: number, + from?: number, + replaces?: FunctionTags, + ): void => { + if (from !== undefined && from < pos) + docRanges.push({ + lo: from, + hi: pos, + id: pos, + ...(replaces !== undefined ? { replaces } : {}), + }); edits.push({ pos, id: pos, @@ -460,8 +488,12 @@ export function planShapeOverlay( return out; }; - const paramTags = (fn: ts.FunctionLikeDeclaration): string | null => { + const paramTags = ( + fn: ts.FunctionLikeDeclaration, + ): { text: string; replaces: FunctionTags } | null => { const tags: string[] = []; + const params: string[] = []; + let returns = false; if (!ts.isConstructorDeclaration(fn) && !constructed.has(fn)) { const ret = returnValues(fn); const t = ret !== undefined && ret.size > 0 ? typeOf(ret, false, 0) : null; @@ -477,7 +509,10 @@ export function planShapeOverlay( process.stderr.write( `[shape-overlay-why] ${fn.name.text}: ${[...ret].map((a) => `${a.kind}:${a.label}${analysis.escaped.has(a) ? "(escaped)" : ""}`).join(" | ")}\n`, ); - if (t !== null) tags.push(`@returns {${t}}`); + if (t !== null) { + tags.push(`@returns {${t}}`); + returns = true; + } } list(fn.parameters).forEach((p, index) => { if ( @@ -493,9 +528,12 @@ export function planShapeOverlay( // narrow the typed parameter to never inside the branch, which the // lowering refuses: such a parameter stays untyped. if (t !== null && typeofTestsOutside(fn, p.name.text, values)) return; - if (t !== null) tags.push(`@param {${t}} ${p.name.text}`); + if (t !== null) { + tags.push(`@param {${t}} ${p.name.text}`); + params.push(p.name.text); + } }); - return tags.length > 0 ? tags.join(" ") : null; + return tags.length > 0 ? { text: tags.join(" "), replaces: { returns, params } } : null; }; const visit = (node: ts.Node): void => { @@ -513,7 +551,8 @@ export function planShapeOverlay( ) { if (node.body !== undefined) { const tags = paramTags(node); - if (tags !== null) annotate(ts.tokenStartOfNode(node), tags, undefined, node.pos); + if (tags !== null) + annotate(ts.tokenStartOfNode(node), tags.text, undefined, node.pos, tags.replaces); } // `this.f = …` fields of a constructor's top-level statements. if (ts.isConstructorDeclaration(node) && node.body !== undefined) { @@ -675,7 +714,8 @@ export function renderShapeOverlay( /** The part's text with the package JSDoc neutralized in the leading trivia * of the annotations a build kept. Each range is one declaration's leading - * trivia, so two ranges are equal or disjoint; the text is built once. */ + * trivia, so two ranges are equal or disjoint; the text is built once, with + * the tags of all the annotations of a range neutralized. */ function neutralizedDocRanges(part: OverlayPart, dropped: ReadonlySet | undefined): string { const ranges = part.docRanges .filter((r) => dropped?.has(r.id) !== true) @@ -683,10 +723,22 @@ function neutralizedDocRanges(part: OverlayPart, dropped: ReadonlySet | if (ranges.length === 0) return part.text; let out = ""; let at = 0; - for (const r of ranges) { + for (let i = 0; i < ranges.length; i++) { + const r = ranges[i]!; + let replaces = r.replaces; + for (; i + 1 < ranges.length && ranges[i + 1]!.lo === r.lo && ranges[i + 1]!.hi === r.hi; i++) { + const next = ranges[i + 1]!.replaces; + replaces = + replaces === undefined || next === undefined + ? undefined + : { + returns: replaces.returns || next.returns, + params: [...replaces.params, ...next.params], + }; + } const lo = Math.max(r.lo, at); if (r.hi <= lo) continue; - out += part.text.slice(at, lo) + neutralizeCommentTags(part.text.slice(lo, r.hi)); + out += part.text.slice(at, lo) + neutralizeCommentTags(part.text.slice(lo, r.hi), replaces); at = r.hi; } return out + part.text.slice(at); @@ -694,17 +746,47 @@ function neutralizedDocRanges(part: OverlayPart, dropped: ReadonlySet | /** A comment range's JSDoc type tags made plain text (the range is * trivia, so it holds no literal). The overlay's own comments carry MARK - * and are kept. Lengths are unchanged. */ -export function neutralizeCommentTags(text: string): string { + * and are kept. With `replaces`, only the tags a function's annotation + * replaces are. Lengths are unchanged. */ +export function neutralizeCommentTags(text: string, replaces?: FunctionTags): string { return text.replace(/\/\*\*[\s\S]*?\*\//g, (comment) => { if (comment.includes(MARK)) return comment; + if (replaces === undefined) + return comment.replace( + /@(param|type|typedef|returns?|template|this|callback|property|prop|arg|argument|enum|implements|extends|augments|satisfies|overload|import)\b/g, + "-$1", + ); return comment.replace( - /@(param|type|typedef|returns?|template|this|callback|property|prop|arg|argument|enum|implements|extends|augments|satisfies|overload|import)\b/g, - "-$1", + /@(param|arg|argument|returns?|type|overload)\b/g, + (tag: string, name: string, at: number) => { + if (name === "type" || name === "overload") return `-${name}`; + if (name === "returns" || name === "return") return replaces.returns ? `-${name}` : tag; + const param = paramTagName(comment, at + tag.length); + return param !== null && replaces.params.includes(param) ? `-${name}` : tag; + }, ); }); } +/** The parameter a `@param` tag ending at `from` names (`{type} name`, + * `[name=default]`, or `name.member` of a documented object), or null. */ +function paramTagName(comment: string, from: number): string | null { + let i = from; + while (i < comment.length && /\s/.test(comment[i]!)) i++; + if (comment[i] === "{") { + let depth = 0; + for (; i < comment.length; i++) { + if (comment[i] === "{") depth++; + else if (comment[i] === "}" && --depth === 0) break; + } + i++; + while (i < comment.length && /\s/.test(comment[i]!)) i++; + } + if (comment[i] === "[") i++; + const name = /^[A-Za-z_$][\w$]*/.exec(comment.slice(i, i + 256)); + return name !== null ? name[0] : null; +} + /** The `typeof` results the values can have. */ function typeofResults(values: ReadonlySet): Set { const out = new Set(); diff --git a/packages/compiler/src/ir/fields.generated.ts b/packages/compiler/src/ir/fields.generated.ts index a46259b6e..e9f5efaa3 100644 --- a/packages/compiler/src/ir/fields.generated.ts +++ b/packages/compiler/src/ir/fields.generated.ts @@ -167,6 +167,7 @@ export function eachStringField( } case "dynCheck": { visit(node.kind, "kind"); + if (typeof node.inferredSite === "string") visit(node.inferredSite, "inferredSite"); return; } case "dynDestrCheck": { @@ -866,6 +867,9 @@ export function walkExprFields(node: IrExpr, walker: IrFieldWalker, skipType: bo case "dynCheck": { walker.string(node.kind); walkExprFields(node.value, walker, skipType); + if (node.inferredSite !== undefined && node.inferredSite !== null) { + walker.string(node.inferredSite); + } if (!skipType) { walkTypeFields(node.type, walker, skipType); } diff --git a/packages/compiler/test/shape-overlay-render.test.ts b/packages/compiler/test/shape-overlay-render.test.ts index 6151cbe68..53b47092d 100644 --- a/packages/compiler/test/shape-overlay-render.test.ts +++ b/packages/compiler/test/shape-overlay-render.test.ts @@ -51,3 +51,33 @@ test("each kept annotation's leading JSDoc is neutralized, once per range", () = expect(overlay.files.get("a.js")).toContain("/** -param {string} a */"); expect(overlay.annotations).toBe(2); }); + +test("a function's annotation neutralizes only the tags it replaces", () => { + const head = + "/**\n * @template T\n * @param {{ a?: boolean }} options\n * @param {T} [b]\n * @param {string} options.c\n * @param {number} n\n * @returns {T}\n */\n"; + expect(neutralizeCommentTags(head, { returns: true, params: ["n"] })).toBe( + "/**\n * @template T\n * @param {{ a?: boolean }} options\n * @param {T} [b]\n * @param {string} options.c\n * -param {number} n\n * -returns {T}\n */\n", + ); + expect(neutralizeCommentTags(head, { returns: false, params: ["options", "b"] })).toBe( + "/**\n * @template T\n * -param {{ a?: boolean }} options\n * -param {T} [b]\n * -param {string} options.c\n * @param {number} n\n * @returns {T}\n */\n", + ); + const text = `${head}function f(options, b, n) {}\n`; + const plan: ShapeOverlayPlan = { + parts: new Map([ + [ + "a.js", + { + text, + edits: [{ pos: head.length, id: head.length, text: "/** @returns {undefined} */ " }], + suffix: "", + docRanges: [ + { lo: 0, hi: head.length, id: head.length, replaces: { returns: true, params: [] } }, + ], + }, + ], + ]), + }; + const out = renderShapeOverlay(plan).files.get("a.js")!; + expect(out).toContain("@param {{ a?: boolean }} options"); + expect(out).toContain("-returns {T}"); +});