diff --git a/packages/compiler/src/frontend/pipeline.ts b/packages/compiler/src/frontend/pipeline.ts index 48d4161f9..bb1053cb5 100644 --- a/packages/compiler/src/frontend/pipeline.ts +++ b/packages/compiler/src/frontend/pipeline.ts @@ -739,23 +739,35 @@ function loadFrontend( level = level === "annotations" ? "files" : "all"; changed = excludeRefusedOverlay(finalLoad.program, refusals, level); } + // The same round also drops what the deferred refusals blame, + // saving a reload. + if (level !== "all" && lowered.runtimeFences.length > 0) + changed = + excludeRefusedOverlay(finalLoad.program, lowered.runtimeFences, "fences") || changed; } if (process.env["SCRIPTC_DEBUG_SHAPE_FALLBACK"] !== undefined) process.stderr.write( `[shape-fallback] round ${round} level ${level}: ${refusals.length} refusals (${refusals - .slice(0, 3) + .slice(0, process.env["SCRIPTC_DEBUG_SHAPE_FALLBACK"] === "all" ? undefined : 3) .map((d) => `${d.loc.file.split("/").pop()}:${d.loc.start} ${d.message.slice(0, 60)}`) .join("; ")}), changed ${changed}\n`, ); if (!changed) break; + const t0 = performance.now(); finalLoad.dispose(); finalLoad = loadProgram(entryPath, { npmStatic: finalNpmStatic, externalTypes }); + const t1 = performance.now(); // The preflight fills the module order; the tsc world reads the // files as written, so it answers as the first load did. const reloadPreflight = checkPreflight(finalLoad); if (reloadPreflight.length > 0) return { ...lowered, module: null, diagnostics: reloadPreflight }; + const t2 = performance.now(); lowered = lowerOnce(); + if (process.env["SCRIPTC_DEBUG_SHAPE_FALLBACK"] !== undefined) + process.stderr.write( + `[shape-fallback] reload ${(t1 - t0).toFixed(0)} ms, preflight ${(t2 - t1).toFixed(0)} ms, lower ${(performance.now() - t2).toFixed(0)} ms\n`, + ); } // The declaration-typed boundary's notes (SC6102) join the // performance warnings; they never fail a build. diff --git a/packages/compiler/src/frontend/shape-inference/analyze.ts b/packages/compiler/src/frontend/shape-inference/analyze.ts index d783a2ad7..2dd53079c 100644 --- a/packages/compiler/src/frontend/shape-inference/analyze.ts +++ b/packages/compiler/src/frontend/shape-inference/analyze.ts @@ -358,8 +358,17 @@ const LOOP_CALLBACK_METHODS: ReadonlySet = new Set([ "replaceAll", ]); -export function analyzeShapes(program: ts.Program, checker: ts.TypeChecker): ShapeAnalysis { - return new Analyzer(program, checker).run(); +export interface AnalyzeOptions { + /** Per-call-site clones per function (default SCRIPTC_SHAPE_SITE_CLONES or 64). */ + readonly siteClones?: number; +} + +export function analyzeShapes( + program: ts.Program, + checker: ts.TypeChecker, + options: AnalyzeOptions = {}, +): ShapeAnalysis { + return new Analyzer(program, checker, options.siteClones ?? SITE_CLONES).run(); } class Analyzer { @@ -421,6 +430,7 @@ class Analyzer { constructor( private readonly program: ts.Program, private readonly checker: ts.TypeChecker, + private readonly siteCloneLimit: number, ) { this.top = this.atoms.intern("top", () => ({ kind: "top", label: "⊤" })); } @@ -1361,7 +1371,7 @@ class Analyzer { return inside ? this.ctx : ""; } private siteClone(info: FunctionInfo, site: Node, args: number[]): FunctionInfo | null { - if (SITE_CLONES === 0 || info.isGenerator || ts.isSourceFile(info.node)) return null; + if (this.siteCloneLimit === 0 || info.isGenerator || ts.isSourceFile(info.node)) return null; if (ts.isClassLikeDeclaration(info.node)) return null; const sc = this.argsFrom.get(args); if (sc === undefined || sc.inLoop || this.forceLoop || this.repeatingSites.has(site)) @@ -1380,7 +1390,7 @@ class Analyzer { const existing = this.functions.get(`${ctx}|${this.nodeKey(info.node)}`); if (existing !== undefined) return existing; const count = this.siteClones.get(info.node) ?? 0; - if (count >= SITE_CLONES) return null; + if (count >= this.siteCloneLimit) return null; this.siteClones.set(info.node, count + 1); return this.functionInfo(info.node, info.ownerClass, ctx); } diff --git a/packages/compiler/src/frontend/shape-inference/fallback.ts b/packages/compiler/src/frontend/shape-inference/fallback.ts index 6a2824990..50857fcfb 100644 --- a/packages/compiler/src/frontend/shape-inference/fallback.ts +++ b/packages/compiler/src/frontend/shape-inference/fallback.ts @@ -35,6 +35,10 @@ export function attributeOverlayRefusals( program: ts.Program, diagnostics: readonly ScrDiagnostic[], overlayFiles: ReadonlySet, + /** Failing refusals may blame every annotation in their enclosing + * function as a last resort; refusals deferred to run time do not (most + * exist without the overlay too). */ + wholeFunctions = true, ): OverlayAttribution { const annotations = new Map>(); const files = new Set(); @@ -124,10 +128,17 @@ export function attributeOverlayRefusals( const sf = program.getSourceFile(file); if (sf === undefined) continue; const visit = (n: ts.Node): void => { - if (ts.isCallExpression(n) && ts.isIdentifier(n.expression)) { + const callee = !ts.isCallExpression(n) + ? undefined + : ts.isIdentifier(n.expression) + ? n.expression + : ts.isPropertyAccessExpression(n.expression) && ts.isIdentifier(n.expression.name) + ? n.expression.name + : undefined; + if (callee !== undefined && ts.isCallExpression(n)) { let decl: ts.Node | undefined; try { - decl = checker!.getSymbolAtLocation(n.expression)?.declarations[0]; + decl = checker!.getSymbolAtLocation(callee)?.declarations[0]; } catch { decl = undefined; } @@ -177,7 +188,7 @@ export function attributeOverlayRefusals( } // Still nothing: the lowering of the enclosing function changed as a // whole (an instantiation the annotations inside it made possible). - if (blamed === before) { + if (blamed === before && wholeFunctions) { const fn = enclosingFunctionLike(node); if (fn !== undefined && overlayFiles.has(fn.getSourceFile().fileName)) { const visit = (n: ts.Node): void => { @@ -197,7 +208,7 @@ export function attributeOverlayRefusals( function enclosingFunction(node: ts.Node): ts.Node | undefined { for (let n: ts.Node | undefined = node.parent; n !== undefined; n = n.parent) - if (ts.isFunctionDeclaration(n)) return n; + if (ts.isFunctionDeclaration(n) || ts.isMethodDeclaration(n)) return n; else if (ts.isFunctionLike(n)) return undefined; return undefined; } diff --git a/packages/compiler/src/frontend/shape-inference/index.ts b/packages/compiler/src/frontend/shape-inference/index.ts index d8074a0a8..38dce5143 100644 --- a/packages/compiler/src/frontend/shape-inference/index.ts +++ b/packages/compiler/src/frontend/shape-inference/index.ts @@ -6,10 +6,17 @@ import type { IrFunction } from "../../ir/ir.js"; import type { ScrDiagnostic } from "../../diagnostics/diagnostic.js"; import { npmStaticPackageOfPath } from "../npm-static.js"; import { isShapelessPackage } from "../package-shims.js"; -import { analyzeShapes, isPackageJsFile } from "./analyze.js"; -import { OVERLAY_MARK, planShapeOverlay, type OverlayExclusions } from "./overlay.js"; +import { analyzeShapes, isPackageJsFile, type ShapeAnalysis } from "./analyze.js"; +import { createHash } from "node:crypto"; +import { + OVERLAY_MARK, + planShapeOverlay, + renderShapeOverlay, + type OverlayExclusions, + type ShapeOverlayPlan, +} from "./overlay.js"; import { attributeOverlayRefusals } from "./fallback.js"; -import { planStaticObjectRewrites } from "./static-objects.js"; +import { planStaticObjectRewrites, type StaticObjectRewrites } from "./static-objects.js"; import { isJsSourceFile } from "../program.js"; import { buildShapeReport } from "./report.js"; import { computeConstantGraphs, constantGraphReport } from "./constant-graphs.js"; @@ -93,6 +100,7 @@ let droppedAnnotations = 0; /** Starts a build: no annotation is excluded yet. */ export function beginShapeOverlayBuild(): void { + overlayCache = new Map(); exclusions = { annotations: new Map(), files: new Set() }; overlayDropped = false; droppedAnnotations = 0; @@ -127,7 +135,12 @@ export function excludeRefusedOverlay( droppedAnnotations = overlayAnnotations; return true; } - const blame = attributeOverlayRefusals(program, diagnostics, files); + const blame = attributeOverlayRefusals(program, diagnostics, files, level !== "fences"); + if (process.env["SCRIPTC_DEBUG_SHAPE_FALLBACK"] === "all") + for (const [file, ids] of blame.annotations) + process.stderr.write( + `[shape-fallback] ${level} blames ${file.split("/").slice(-2).join("/")}: ${[...ids].join(",")}\n`, + ); let changed = false; for (const file of blame.files) { const ids = blame.annotations.get(file); @@ -158,6 +171,44 @@ function overlayExclusions(): OverlayExclusions { return exclusions; } +/* Per build, the overlay's analysis-dependent results by program text and + * excluded files: a reload that only drops refused annotations renders the + * cached plan instead of analyzing again (fallback.ts). */ +interface OverlayCacheEntry { + budgetExceeded?: true; + rewrites?: StaticObjectRewrites; + plan?: ShapeOverlayPlan; +} +let overlayCache = new Map(); + +function programTextKey(program: ts.Program): string { + const hash = createHash("sha256"); + for (const sf of program.getSourceFiles()) { + if (sf.isDeclarationFile) continue; + hash.update(sf.fileName).update("\0").update(sf.text).update("\0"); + } + return hash.digest("hex"); +} + +/** Per-call-site clones serve the constant-graph facts (stage 3); the + * overlay's rewrites and types do not use them yet, so its analyses run + * without them unless SCRIPTC_SHAPE_SITE_CLONES asks. */ +function overlaySiteClones(): number { + const env = process.env["SCRIPTC_SHAPE_SITE_CLONES"]; + return env === undefined ? 0 : Number(env); +} + +function overlayJsFiles(analysis: ShapeAnalysis): ts.SourceFile[] { + // Package JavaScript of packages that keep shapes on, and the program's + // own JavaScript files. + return analysis.files.filter((sf) => { + if (sf.isDeclarationFile || !isJsSourceFile(sf)) return false; + if (!isPackageJsFile(sf)) return !sf.fileName.includes("/node_modules/"); + const pkg = npmStaticPackageOfPath(sf.fileName); + return pkg === null || !isShapelessPackage(pkg); + }); +} + /** loadProgram7's overlayer: analyze, plan, and register the overlay files. */ export function shapeOverlayer( program: ts.Program, @@ -165,23 +216,34 @@ export function shapeOverlayer( ): ReadonlyMap | null { if (round === 0) resetShapeOverlay(); if (overlayDropped) return null; - const analysis = analyzeShapes(program, program.getTypeChecker()); - if (analysis.budgetExceeded) return null; - // Package JavaScript of packages that keep shapes on, and the program's - // own JavaScript files. - const jsFiles = analysis.files.filter((sf) => { - if (sf.isDeclarationFile || !isJsSourceFile(sf)) return false; - if (!isPackageJsFile(sf)) return !sf.fileName.includes("/node_modules/"); - const pkg = npmStaticPackageOfPath(sf.fileName); - return pkg === null || !isShapelessPackage(pkg); - }); + const key = `${programTextKey(program)}|${[...exclusions.files].sort().join(",")}`; + const cached = overlayCache.get(key); + const entry: OverlayCacheEntry = cached ?? {}; + if (cached === undefined) overlayCache.set(key, entry); + if (entry.budgetExceeded === true) return null; + let analysis: ShapeAnalysis | null = null; + const analyzed = (): ShapeAnalysis | null => { + analysis ??= analyzeShapes(program, program.getTypeChecker(), { + siteClones: overlaySiteClones(), + }); + if (analysis.budgetExceeded) { + entry.budgetExceeded = true; + return null; + } + return analysis; + }; // Round 0: the structural rewrites (static-objects.ts); the program is // rebuilt from them and analyzed again before the typed overlay. if (round === 0 && process.env["SCRIPTC_SHAPE_STATIC_OBJECTS"] !== "0") { - const rewrites = planStaticObjectRewrites( - analysis, - jsFiles.filter((sf) => !exclusions.files.has(sf.fileName)), - ); + if (entry.rewrites === undefined) { + const a = analyzed(); + if (a === null) return null; + entry.rewrites = planStaticObjectRewrites( + a, + overlayJsFiles(a).filter((sf) => !exclusions.files.has(sf.fileName)), + ); + } + const rewrites = entry.rewrites; staticRewriteFiles = new Set(rewrites.files.keys()); staticObjectStats = { classes: rewrites.classes, @@ -198,8 +260,16 @@ export function shapeOverlayer( if (rewrites.files.size > 0) return rewrites.files; } if (round > 1) return null; - const report = buildShapeReport(analysis, "1"); - const overlay = planShapeOverlay(analysis, report.sites, jsFiles, overlayExclusions()); + if (entry.plan === undefined) { + const a = analyzed(); + if (a === null) return null; + const report = buildShapeReport(a, "1"); + entry.plan = planShapeOverlay(a, report.sites, overlayJsFiles(a), { + annotations: new Map(), + files: exclusions.files, + }); + } + const overlay = renderShapeOverlay(entry.plan, overlayExclusions()); overlayFiles = new Set(overlay.files.keys()); overlayAnnotations = overlay.annotations; overlayDeadBranches = overlay.deadBranches; diff --git a/packages/compiler/src/frontend/shape-inference/overlay.ts b/packages/compiler/src/frontend/shape-inference/overlay.ts index 023f55deb..2fa99de13 100644 --- a/packages/compiler/src/frontend/shape-inference/overlay.ts +++ b/packages/compiler/src/frontend/shape-inference/overlay.ts @@ -35,6 +35,27 @@ interface Edit { readonly text: string; /** Replaces pos..end instead of inserting at pos. */ readonly end?: number; + /** The annotation (or dead branch) the edit belongs to. */ + readonly id?: number; +} + +/** The overlay before annotation exclusions apply: per file, its text, + * edits and appended typedefs. It depends only on the analysis, so a build + * that drops refused annotations renders it again instead of analyzing + * again (renderShapeOverlay). */ +export interface ShapeOverlayPlan { + readonly parts: Map; +} + +interface OverlayPart { + readonly text: string; + readonly edits: Edit[]; + readonly suffix: string; + /** The leading trivia of each annotated declaration, by annotation id: + * the package's own JSDoc there is neutralized while the annotation is + * kept (package JSDoc is a hint; elsewhere it types the code as it does + * without the overlay). */ + readonly docRanges: { lo: number; hi: number; id: number }[]; } /** Marks the overlay's own comments (an unknown JSDoc tag, ignored). */ @@ -76,7 +97,7 @@ export function planShapeOverlay( verdicts: readonly SiteVerdict[], packageFiles: readonly ts.SourceFile[], exclusions: OverlayExclusions = NO_EXCLUSIONS, -): ShapeOverlay { +): ShapeOverlayPlan { const pSites = new Map(); const siteVerdicts = new Set(verdicts.map((v) => v.site.atom)); // Records and arrays print only for sites whose objects never escape: a @@ -93,11 +114,9 @@ export function planShapeOverlay( siteKeys.set(v.site.atom, keys); pSites.set(v.site.atom, v); } - const files = new Map(); + const parts = new Map(); const overlaid = new Set(packageFiles); const exportedClasses = new Map>(); - let annotations = 0; - let deadBranches = 0; debugIndex = 0; // `if` branches no condition value allowed, per file: the overlay // replaces each with a throwing check, so the checker and the lowering @@ -118,15 +137,18 @@ export function planShapeOverlay( const typedefs = new Map(); const typedefText: string[] = []; const edits: Edit[] = []; - const dropped = exclusions.annotations.get(sf.fileName); // One annotation: a JSDoc comment at `pos` carrying MARK and the // annotation's id (its position), so a refused lowering can name it and // the next load drop it. `close`: the text that closes a type cast. - const annotate = (pos: number, tags: string, close?: number): void => { - if (dropped?.has(pos) === true) return; - edits.push({ pos, text: `/** ${tags} ${MARK} #${pos} */ ${close !== undefined ? "(" : ""}` }); - if (close !== undefined) edits.push({ pos: close, text: ")" }); - annotations++; + const docRanges: { lo: number; hi: number; id: number }[] = []; + const annotate = (pos: number, tags: string, close?: number, from?: number): void => { + if (from !== undefined && from < pos) docRanges.push({ lo: from, hi: pos, id: pos }); + edits.push({ + pos, + id: pos, + text: `/** ${tags} ${MARK} #${pos} */ ${close !== undefined ? "(" : ""}`, + }); + if (close !== undefined) edits.push({ pos: close, id: pos, text: ")" }); }; const className = (decl: ts.Node): string | null => { @@ -461,7 +483,12 @@ export function planShapeOverlay( p.questionToken !== undefined ) return; - const t = typeOf(paramValues(fn, index), false, 0); + const values = paramValues(fn, index); + const t = typeOf(values, false, 0); + // A `typeof p === "…"` test the proven values never pass would + // narrow the typed parameter to never inside the branch, which the + // lowering refuses: such a parameter stays untyped. + if (t !== null && typeofTestsOutside(fn, p.name.text, values)) return; if (t !== null) tags.push(`@param {${t}} ${p.name.text}`); }); return tags.length > 0 ? tags.join(" ") : null; @@ -482,7 +509,7 @@ export function planShapeOverlay( ) { if (node.body !== undefined) { const tags = paramTags(node); - if (tags !== null) annotate(ts.tokenStartOfNode(node), tags); + if (tags !== null) annotate(ts.tokenStartOfNode(node), tags, undefined, node.pos); } // `this.f = …` fields of a constructor's top-level statements. if (ts.isConstructorDeclaration(node) && node.body !== undefined) { @@ -505,7 +532,7 @@ export function planShapeOverlay( if (seen.has(name)) continue; seen.add(name); const t = cls !== undefined ? fieldType(cls, name) : null; - if (t !== null) annotate(ts.tokenStartOfNode(st), `@type {${t}}`); + if (t !== null) annotate(ts.tokenStartOfNode(st), `@type {${t}}`, undefined, st.pos); } } } @@ -521,7 +548,7 @@ export function planShapeOverlay( const init = d.initializer; if (ts.isIdentifier(d.name) && init !== undefined && !literalInitializer(init)) { const t = typeOf(analysis.declValues(d), false, 0); - if (t !== null) annotate(ts.tokenStartOfNode(node), `@type {${t}}`); + if (t !== null) annotate(ts.tokenStartOfNode(node), `@type {${t}}`, undefined, node.pos); } } ts.forEachChild(node, (c) => { @@ -551,7 +578,6 @@ export function planShapeOverlay( const ranges: [number, number][] = []; for (const branch of dead) { const start = ts.tokenStartOfNode(branch); - if (dropped?.has(start) === true) continue; if (ranges.some(([lo, hi]) => start >= lo && start < hi)) continue; ranges.push([start, branch.end]); } @@ -570,9 +596,9 @@ export function planShapeOverlay( edits.push({ pos: lo, end: hi, + id: lo, text: `{ /** ${MARK} #${lo} */ throw new Error(${JSON.stringify(message)}); }`, }); - deadBranches++; } } // Debugging: SCRIPTC_SHAPE_OVERLAY_ONLY=a-b keeps the annotations @@ -589,7 +615,7 @@ export function planShapeOverlay( edits.push(...kept); } if (edits.length === 0) continue; - files.set(sf.fileName, neutralizeDocTags(applyEdits(sf.text, edits)) + typedefText.join("")); + parts.set(sf.fileName, { text: sf.text, edits, suffix: typedefText.join(""), docRanges }); } // Type aliases other files name these files' classes through. for (const [other, names] of exportedClasses) { @@ -597,14 +623,98 @@ export function planShapeOverlay( .sort() .map((n) => `/** @typedef {${n}} __shape_class_${n} ${MARK} */`) .join(""); - files.set( - other.fileName, - (files.get(other.fileName) ?? neutralizeDocTags(other.text)) + "\n" + aliases, - ); + const part = parts.get(other.fileName); + if (part !== undefined) + parts.set(other.fileName, { ...part, suffix: `${part.suffix}\n${aliases}` }); + else + parts.set(other.fileName, { + text: other.text, + edits: [], + suffix: `\n${aliases}`, + docRanges: [], + }); + } + return { parts }; +} + +/** The overlay's files with the annotations a build dropped left out. */ +export function renderShapeOverlay( + plan: ShapeOverlayPlan, + exclusions: OverlayExclusions = NO_EXCLUSIONS, +): ShapeOverlay { + const files = new Map(); + let annotations = 0; + let deadBranches = 0; + for (const [file, part] of plan.parts) { + const dropped = exclusions.annotations.get(file); + const edits = part.edits.filter((e) => e.id === undefined || dropped?.has(e.id) !== true); + for (const e of edits) { + if (e.id === undefined || e.text === ")") continue; + if (e.end !== undefined) deadBranches++; + else annotations++; + } + if (edits.length === 0 && part.suffix === "") continue; + let text = part.text; + for (const r of part.docRanges) + if (dropped?.has(r.id) !== true) + text = text.slice(0, r.lo) + neutralizeDocTags(text.slice(r.lo, r.hi)) + text.slice(r.hi); + files.set(file, applyEdits(text, edits) + part.suffix); } return { files, annotations, deadBranches }; } +/** The `typeof` results the values can have. */ +function typeofResults(values: ReadonlySet): Set { + const out = new Set(); + for (const a of values) { + if (a.kind === "prim") + out.add(a.label === "null" ? "object" : a.label === "symbol" ? "symbol" : a.label); + else if (a.kind === "function" || a.kind === "class") out.add("function"); + else out.add("object"); + } + return out; +} + +/** Whether the function's body compares `typeof name` with a result the + * values never have. */ +function typeofTestsOutside( + fn: ts.FunctionLikeDeclaration, + name: string, + values: ReadonlySet, +): boolean { + const results = typeofResults(values); + let outside = false; + const visit = (n: ts.Node): void => { + if (outside) return; + if ( + ts.isBinaryExpression(n) && + (n.operatorToken.kind === ts.SyntaxKind.EqualsEqualsEqualsToken || + n.operatorToken.kind === ts.SyntaxKind.EqualsEqualsToken || + n.operatorToken.kind === ts.SyntaxKind.ExclamationEqualsEqualsToken || + n.operatorToken.kind === ts.SyntaxKind.ExclamationEqualsToken) + ) { + for (const [op, lit] of [ + [n.left, n.right], + [n.right, n.left], + ] as const) + if ( + ts.isTypeOfExpression(op) && + ts.isIdentifier(op.expression) && + op.expression.text === name && + ts.isStringLiteral(lit) && + !results.has(lit.text) + ) + outside = true; + } + ts.forEachChild(n, (c) => { + visit(c); + return undefined; + }); + }; + if (fn.body !== undefined) visit(fn.body); + return outside; +} + function packageNameOf(fileName: string): string { const parts = (fileName.split("/node_modules/").pop() ?? "").split("/"); return parts[0]!.startsWith("@") ? `${parts[0]}/${parts[1]}` : parts[0]!; diff --git a/tests/harness/npm-suite.ts b/tests/harness/npm-suite.ts index b0f9bc536..d5bb5beee 100644 --- a/tests/harness/npm-suite.ts +++ b/tests/harness/npm-suite.ts @@ -112,15 +112,8 @@ const KNOWN: Record = { "workspace-unclean": { build: "SC2013" }, }; -/** Where the overlay changes how a case KNOWN refuses: the operation it - * fails at on an argv list that fails in both builds. */ -const OVERLAY_KNOWN: Record = { - // `--help` reaches an unsupported operation in both builds; with the - // overlay it fails at a checked conversion of an option value first - // (results/design-shape-inference/NOTES.md, open item). - "commander-calc": { run: "expected number at $, got undefined", argvs: [["--help"]] }, - "commander-calc-async": { run: "expected number at $, got undefined", argvs: [["--help"]] }, -}; +/** Where the overlay changes how a case KNOWN refuses (none today). */ +const OVERLAY_KNOWN: Record = {}; /** Per-case timeouts past the default 120 s: the undici cases compile the * built-in undici implementation's module graph, the zod 3 cases (zod3,