Cut the shape overlay's compile cost; keep package JSDoc it does not replace (#2223)

packages/compiler/src/frontend/pipeline.ts

@@ -841,23 +841,35 @@ function loadFrontend(
             level = level === "annotations" ? "files" : "all";
             changed = excludeRefusedOverlay(finalLoad.program, refusals, level);
           }
+          // The same round also drops what the deferred refusals blame,
+          // saving a reload.
+          if (level !== "all" && lowered.runtimeFences.length > 0)
+            changed =
+              excludeRefusedOverlay(finalLoad.program, lowered.runtimeFences, "fences") || changed;
         }
         if (process.env["SCRIPTC_DEBUG_SHAPE_FALLBACK"] !== undefined)
           process.stderr.write(
             `[shape-fallback] round ${round} level ${level}: ${refusals.length} refusals (${refusals
-              .slice(0, 3)
+              .slice(0, process.env["SCRIPTC_DEBUG_SHAPE_FALLBACK"] === "all" ? undefined : 3)
               .map((d) => `${d.loc.file.split("/").pop()}:${d.loc.start} ${d.message.slice(0, 60)}`)
               .join("; ")}), changed ${changed}\n`,
           );
         if (!changed) break;
+        const t0 = performance.now();
         finalLoad.dispose();
         finalLoad = loadProgram(entryPath, { npmStatic: finalNpmStatic, externalTypes });
+        const t1 = performance.now();
         // The preflight fills the module order; the tsc world reads the
         // files as written, so it answers as the first load did.
         const reloadPreflight = checkPreflight(finalLoad);
         if (reloadPreflight.length > 0)
           return { ...lowered, module: null, diagnostics: reloadPreflight };
+        const t2 = performance.now();
         lowered = lowerOnce();
+        if (process.env["SCRIPTC_DEBUG_SHAPE_FALLBACK"] !== undefined)
+          process.stderr.write(
+            `[shape-fallback] reload ${(t1 - t0).toFixed(0)} ms, preflight ${(t2 - t1).toFixed(0)} ms, lower ${(performance.now() - t2).toFixed(0)} ms\n`,
+          );
       }
       // The declaration-typed boundary's notes (SC6102) join the
       // performance warnings; they never fail a build.

packages/compiler/src/frontend/shape-inference/analyze.ts

@@ -413,8 +413,17 @@ const LOOP_CALLBACK_METHODS: ReadonlySet<string> = new Set([
   "replaceAll",
 ]);
 
-export function analyzeShapes(program: ts.Program, checker: ts.TypeChecker): ShapeAnalysis {
-  return new Analyzer(program, checker).run();
+export interface AnalyzeOptions {
+  /** Per-call-site clones per function (default SCRIPTC_SHAPE_SITE_CLONES or 64). */
+  readonly siteClones?: number;
+}
+
+export function analyzeShapes(
+  program: ts.Program,
+  checker: ts.TypeChecker,
+  options: AnalyzeOptions = {},
+): ShapeAnalysis {
+  return new Analyzer(program, checker, options.siteClones ?? SITE_CLONES).run();
 }
 
 class Analyzer {
@@ -492,6 +501,7 @@ class Analyzer {
   constructor(
     private readonly program: ts.Program,
     private readonly checker: ts.TypeChecker,
+    private readonly siteCloneLimit: number,
   ) {
     this.top = this.atoms.intern("top", () => ({ kind: "top", label: "⊤" }));
   }
@@ -1488,7 +1498,7 @@ class Analyzer {
     return inside ? this.ctx : "";
   }
   private siteClone(info: FunctionInfo, site: Node, args: number[]): FunctionInfo | null {
-    if (SITE_CLONES === 0 || info.isGenerator || ts.isSourceFile(info.node)) return null;
+    if (this.siteCloneLimit === 0 || info.isGenerator || ts.isSourceFile(info.node)) return null;
     if (ts.isClassLikeDeclaration(info.node)) return null;
     // A function walked in an enclosing clone's or receiver's context reads
     // and writes that context's bindings of the variables it captures. A
@@ -1516,7 +1526,7 @@ class Analyzer {
     const existing = this.functions.get(`${ctx}|${this.nodeKey(info.node)}`);
     if (existing !== undefined) return existing;
     const count = this.siteClones.get(info.node) ?? 0;
-    if (count >= SITE_CLONES) return null;
+    if (count >= this.siteCloneLimit) return null;
     this.siteClones.set(info.node, count + 1);
     // The clone keeps the function's receiver: an arrow's lexical `this`,
     // and whatever reaches a function's own `this`, such as the receiver

packages/compiler/src/frontend/shape-inference/fallback.ts

@@ -35,6 +35,10 @@ export function attributeOverlayRefusals(
   program: ts.Program,
   diagnostics: readonly ScrDiagnostic[],
   overlayFiles: ReadonlySet<string>,
+  /** Failing refusals may blame every annotation in their enclosing
+   * function as a last resort; refusals deferred to run time do not (most
+   * exist without the overlay too). */
+  wholeFunctions = true,
 ): OverlayAttribution {
   const annotations = new Map<string, Set<number>>();
   const files = new Set<string>();
@@ -124,10 +128,17 @@ export function attributeOverlayRefusals(
         const sf = program.getSourceFile(file);
         if (sf === undefined) continue;
         const visit = (n: ts.Node): void => {
-          if (ts.isCallExpression(n) && ts.isIdentifier(n.expression)) {
+          const callee = !ts.isCallExpression(n)
+            ? undefined
+            : ts.isIdentifier(n.expression)
+              ? n.expression
+              : ts.isPropertyAccessExpression(n.expression) && ts.isIdentifier(n.expression.name)
+                ? n.expression.name
+                : undefined;
+          if (callee !== undefined && ts.isCallExpression(n)) {
             let decl: ts.Node | undefined;
             try {
-              decl = checker!.getSymbolAtLocation(n.expression)?.declarations[0];
+              decl = checker!.getSymbolAtLocation(callee)?.declarations[0];
             } catch {
               decl = undefined;
             }
@@ -177,7 +188,7 @@ export function attributeOverlayRefusals(
     }
     // Still nothing: the lowering of the enclosing function changed as a
     // whole (an instantiation the annotations inside it made possible).
-    if (blamed === before) {
+    if (blamed === before && wholeFunctions) {
       const fn = enclosingFunctionLike(node);
       if (fn !== undefined && overlayFiles.has(fn.getSourceFile().fileName)) {
         const visit = (n: ts.Node): void => {
@@ -197,7 +208,7 @@ export function attributeOverlayRefusals(
 
 function enclosingFunction(node: ts.Node): ts.Node | undefined {
   for (let n: ts.Node | undefined = node.parent; n !== undefined; n = n.parent)
-    if (ts.isFunctionDeclaration(n)) return n;
+    if (ts.isFunctionDeclaration(n) || ts.isMethodDeclaration(n)) return n;
     else if (ts.isFunctionLike(n)) return undefined;
   return undefined;
 }

packages/compiler/src/frontend/shape-inference/index.ts

@@ -6,10 +6,17 @@ import type { IrFunction } from "../../ir/ir.js";
 import type { ScrDiagnostic } from "../../diagnostics/diagnostic.js";
 import { npmStaticPackageOfPath } from "../npm-static.js";
 import { isShapelessPackage } from "../package-shims.js";
-import { analyzeShapes, isPackageJsFile } from "./analyze.js";
-import { OVERLAY_MARK, planShapeOverlay, type OverlayExclusions } from "./overlay.js";
+import { analyzeShapes, isPackageJsFile, type ShapeAnalysis } from "./analyze.js";
+import { createHash } from "node:crypto";
+import {
+  OVERLAY_MARK,
+  planShapeOverlay,
+  renderShapeOverlay,
+  type OverlayExclusions,
+  type ShapeOverlayPlan,
+} from "./overlay.js";
 import { attributeOverlayRefusals } from "./fallback.js";
-import { planStaticObjectRewrites } from "./static-objects.js";
+import { planStaticObjectRewrites, type StaticObjectRewrites } from "./static-objects.js";
 import { isJsSourceFile } from "../program.js";
 import { buildShapeReport } from "./report.js";
 import { computeConstantGraphs, constantGraphReport } from "./constant-graphs.js";
@@ -93,6 +100,7 @@ let droppedAnnotations = 0;
 
 /** Starts a build: no annotation is excluded yet. */
 export function beginShapeOverlayBuild(): void {
+  overlayCache = new Map();
   exclusions = { annotations: new Map(), files: new Set() };
   overlayDropped = false;
   droppedAnnotations = 0;
@@ -127,7 +135,12 @@ export function excludeRefusedOverlay(
     droppedAnnotations = overlayAnnotations;
     return true;
   }
-  const blame = attributeOverlayRefusals(program, diagnostics, files);
+  const blame = attributeOverlayRefusals(program, diagnostics, files, level !== "fences");
+  if (process.env["SCRIPTC_DEBUG_SHAPE_FALLBACK"] === "all")
+    for (const [file, ids] of blame.annotations)
+      process.stderr.write(
+        `[shape-fallback] ${level} blames ${file.split("/").slice(-2).join("/")}: ${[...ids].join(",")}\n`,
+      );
   let changed = false;
   for (const file of blame.files) {
     const ids = blame.annotations.get(file);
@@ -158,6 +171,44 @@ function overlayExclusions(): OverlayExclusions {
   return exclusions;
 }
 
+/* Per build, the overlay's analysis-dependent results by program text and
+ * excluded files: a reload that only drops refused annotations renders the
+ * cached plan instead of analyzing again (fallback.ts). */
+interface OverlayCacheEntry {
+  budgetExceeded?: true;
+  rewrites?: StaticObjectRewrites;
+  plan?: ShapeOverlayPlan;
+}
+let overlayCache = new Map<string, OverlayCacheEntry>();
+
+function programTextKey(program: ts.Program): string {
+  const hash = createHash("sha256");
+  for (const sf of program.getSourceFiles()) {
+    if (sf.isDeclarationFile) continue;
+    hash.update(sf.fileName).update("\0").update(sf.text).update("\0");
+  }
+  return hash.digest("hex");
+}
+
+/** Per-call-site clones serve the constant-graph facts (stage 3); the
+ * overlay's rewrites and types do not use them yet, so its analyses run
+ * without them unless SCRIPTC_SHAPE_SITE_CLONES asks. */
+function overlaySiteClones(): number {
+  const env = process.env["SCRIPTC_SHAPE_SITE_CLONES"];
+  return env === undefined ? 0 : Number(env);
+}
+
+function overlayJsFiles(analysis: ShapeAnalysis): ts.SourceFile[] {
+  // Package JavaScript of packages that keep shapes on, and the program's
+  // own JavaScript files.
+  return analysis.files.filter((sf) => {
+    if (sf.isDeclarationFile || !isJsSourceFile(sf)) return false;
+    if (!isPackageJsFile(sf)) return !sf.fileName.includes("/node_modules/");
+    const pkg = npmStaticPackageOfPath(sf.fileName);
+    return pkg === null || !isShapelessPackage(pkg);
+  });
+}
+
 /** loadProgram7's overlayer: analyze, plan, and register the overlay files. */
 export function shapeOverlayer(
   program: ts.Program,
@@ -165,23 +216,34 @@ export function shapeOverlayer(
 ): ReadonlyMap<string, string> | null {
   if (round === 0) resetShapeOverlay();
   if (overlayDropped) return null;
-  const analysis = analyzeShapes(program, program.getTypeChecker());
-  if (analysis.budgetExceeded) return null;
-  // Package JavaScript of packages that keep shapes on, and the program's
-  // own JavaScript files.
-  const jsFiles = analysis.files.filter((sf) => {
-    if (sf.isDeclarationFile || !isJsSourceFile(sf)) return false;
-    if (!isPackageJsFile(sf)) return !sf.fileName.includes("/node_modules/");
-    const pkg = npmStaticPackageOfPath(sf.fileName);
-    return pkg === null || !isShapelessPackage(pkg);
-  });
+  const key = `${programTextKey(program)}|${[...exclusions.files].sort().join(",")}`;
+  const cached = overlayCache.get(key);
+  const entry: OverlayCacheEntry = cached ?? {};
+  if (cached === undefined) overlayCache.set(key, entry);
+  if (entry.budgetExceeded === true) return null;
+  let analysis: ShapeAnalysis | null = null;
+  const analyzed = (): ShapeAnalysis | null => {
+    analysis ??= analyzeShapes(program, program.getTypeChecker(), {
+      siteClones: overlaySiteClones(),
+    });
+    if (analysis.budgetExceeded) {
+      entry.budgetExceeded = true;
+      return null;
+    }
+    return analysis;
+  };
   // Round 0: the structural rewrites (static-objects.ts); the program is
   // rebuilt from them and analyzed again before the typed overlay.
   if (round === 0 && process.env["SCRIPTC_SHAPE_STATIC_OBJECTS"] !== "0") {
-    const rewrites = planStaticObjectRewrites(
-      analysis,
-      jsFiles.filter((sf) => !exclusions.files.has(sf.fileName)),
-    );
+    if (entry.rewrites === undefined) {
+      const a = analyzed();
+      if (a === null) return null;
+      entry.rewrites = planStaticObjectRewrites(
+        a,
+        overlayJsFiles(a).filter((sf) => !exclusions.files.has(sf.fileName)),
+      );
+    }
+    const rewrites = entry.rewrites;
     staticRewriteFiles = new Set(rewrites.files.keys());
     staticObjectStats = {
       classes: rewrites.classes,
@@ -198,8 +260,16 @@ export function shapeOverlayer(
     if (rewrites.files.size > 0) return rewrites.files;
   }
   if (round > 1) return null;
-  const report = buildShapeReport(analysis, "1");
-  const overlay = planShapeOverlay(analysis, report.sites, jsFiles, overlayExclusions());
+  if (entry.plan === undefined) {
+    const a = analyzed();
+    if (a === null) return null;
+    const report = buildShapeReport(a, "1");
+    entry.plan = planShapeOverlay(a, report.sites, overlayJsFiles(a), {
+      annotations: new Map(),
+      files: exclusions.files,
+    });
+  }
+  const overlay = renderShapeOverlay(entry.plan, overlayExclusions());
   overlayFiles = new Set(overlay.files.keys());
   overlayAnnotations = overlay.annotations;
   overlayDeadBranches = overlay.deadBranches;

packages/compiler/src/frontend/shape-inference/overlay.ts

@@ -35,6 +35,43 @@ interface Edit {
   readonly text: string;
   /** Replaces pos..end instead of inserting at pos. */
   readonly end?: number;
+  /** The annotation (or dead branch) the edit belongs to. */
+  readonly id?: number;
+}
+
+/** The overlay before annotation exclusions apply: per file, its text,
+ * edits and appended typedefs. It depends only on the analysis, so a build
+ * that drops refused annotations renders it again instead of analyzing
+ * again (renderShapeOverlay). */
+export interface ShapeOverlayPlan {
+  readonly parts: Map<string, OverlayPart>;
+}
+
+interface OverlayPart {
+  readonly text: string;
+  readonly edits: Edit[];
+  readonly suffix: string;
+  /** The leading trivia of each annotated declaration, by annotation id:
+   * the package's own JSDoc there is neutralized while the annotation is
+   * kept (package JSDoc is a hint; elsewhere it types the code as it does
+   * without the overlay). */
+  readonly docRanges: DocRange[];
+}
+
+interface DocRange {
+  readonly lo: number;
+  readonly hi: number;
+  readonly id: number;
+  /** For a function's annotation, the tags it writes: only the package's
+   * tags for the same result and parameters (and a `@type` or `@overload`
+   * signature) are neutralized, so the parameters it leaves untyped keep
+   * their documented types. Unset: every type tag in the range is. */
+  readonly replaces?: FunctionTags;
+}
+
+interface FunctionTags {
+  readonly returns: boolean;
+  readonly params: readonly string[];
 }
 
 /** Marks the overlay's own comments (an unknown JSDoc tag, ignored). */
@@ -76,7 +113,7 @@ export function planShapeOverlay(
   verdicts: readonly SiteVerdict[],
   packageFiles: readonly ts.SourceFile[],
   exclusions: OverlayExclusions = NO_EXCLUSIONS,
-): ShapeOverlay {
+): ShapeOverlayPlan {
   const pSites = new Map<Atom, SiteVerdict>();
   const siteVerdicts = new Set<Atom>(verdicts.map((v) => v.site.atom));
   // Records and arrays print only for sites whose objects never escape: a
@@ -93,11 +130,9 @@ export function planShapeOverlay(
     siteKeys.set(v.site.atom, keys);
     pSites.set(v.site.atom, v);
   }
-  const files = new Map<string, string>();
+  const parts = new Map<string, OverlayPart>();
   const overlaid = new Set<ts.SourceFile>(packageFiles);
   const exportedClasses = new Map<ts.SourceFile, Set<string>>();
-  let annotations = 0;
-  let deadBranches = 0;
   debugIndex = 0;
   // `if` branches no condition value allowed, per file: the overlay
   // replaces each with a throwing check, so the checker and the lowering
@@ -118,15 +153,30 @@ export function planShapeOverlay(
     const typedefs = new Map<string, string>();
     const typedefText: string[] = [];
     const edits: Edit[] = [];
-    const dropped = exclusions.annotations.get(sf.fileName);
     // One annotation: a JSDoc comment at `pos` carrying MARK and the
     // annotation's id (its position), so a refused lowering can name it and
     // the next load drop it. `close`: the text that closes a type cast.
-    const annotate = (pos: number, tags: string, close?: number): void => {
-      if (dropped?.has(pos) === true) return;
-      edits.push({ pos, text: `/** ${tags} ${MARK} #${pos} */ ${close !== undefined ? "(" : ""}` });
-      if (close !== undefined) edits.push({ pos: close, text: ")" });
-      annotations++;
+    const docRanges: DocRange[] = [];
+    const annotate = (
+      pos: number,
+      tags: string,
+      close?: number,
+      from?: number,
+      replaces?: FunctionTags,
+    ): void => {
+      if (from !== undefined && from < pos)
+        docRanges.push({
+          lo: from,
+          hi: pos,
+          id: pos,
+          ...(replaces !== undefined ? { replaces } : {}),
+        });
+      edits.push({
+        pos,
+        id: pos,
+        text: `/** ${tags} ${MARK} #${pos} */ ${close !== undefined ? "(" : ""}`,
+      });
+      if (close !== undefined) edits.push({ pos: close, id: pos, text: ")" });
     };
 
     const className = (decl: ts.Node): string | null => {
@@ -438,8 +488,12 @@ export function planShapeOverlay(
       return out;
     };
 
-    const paramTags = (fn: ts.FunctionLikeDeclaration): string | null => {
+    const paramTags = (
+      fn: ts.FunctionLikeDeclaration,
+    ): { text: string; replaces: FunctionTags } | null => {
       const tags: string[] = [];
+      const params: string[] = [];
+      let returns = false;
       if (!ts.isConstructorDeclaration(fn) && !constructed.has(fn)) {
         const ret = returnValues(fn);
         const t = ret !== undefined && ret.size > 0 ? typeOf(ret, false, 0) : null;
@@ -455,7 +509,10 @@ export function planShapeOverlay(
           process.stderr.write(
             `[shape-overlay-why] ${fn.name.text}: ${[...ret].map((a) => `${a.kind}:${a.label}${analysis.escaped.has(a) ? "(escaped)" : ""}`).join(" | ")}\n`,
           );
-        if (t !== null) tags.push(`@returns {${t}}`);
+        if (t !== null) {
+          tags.push(`@returns {${t}}`);
+          returns = true;
+        }
       }
       list(fn.parameters).forEach((p, index) => {
         if (
@@ -465,10 +522,18 @@ export function planShapeOverlay(
           p.questionToken !== undefined
         )
           return;
-        const t = typeOf(paramValues(fn, index), false, 0);
-        if (t !== null) tags.push(`@param {${t}} ${p.name.text}`);
+        const values = paramValues(fn, index);
+        const t = typeOf(values, false, 0);
+        // A `typeof p === "…"` test the proven values never pass would
+        // narrow the typed parameter to never inside the branch, which the
+        // lowering refuses: such a parameter stays untyped.
+        if (t !== null && typeofTestsOutside(fn, p.name.text, values)) return;
+        if (t !== null) {
+          tags.push(`@param {${t}} ${p.name.text}`);
+          params.push(p.name.text);
+        }
       });
-      return tags.length > 0 ? tags.join(" ") : null;
+      return tags.length > 0 ? { text: tags.join(" "), replaces: { returns, params } } : null;
     };
 
     const visit = (node: ts.Node): void => {
@@ -486,7 +551,8 @@ export function planShapeOverlay(
       ) {
         if (node.body !== undefined) {
           const tags = paramTags(node);
-          if (tags !== null) annotate(ts.tokenStartOfNode(node), tags);
+          if (tags !== null)
+            annotate(ts.tokenStartOfNode(node), tags.text, undefined, node.pos, tags.replaces);
         }
         // `this.f = …` fields of a constructor's top-level statements.
         if (ts.isConstructorDeclaration(node) && node.body !== undefined) {
@@ -509,7 +575,7 @@ export function planShapeOverlay(
             if (seen.has(name)) continue;
             seen.add(name);
             const t = cls !== undefined ? fieldType(cls, name) : null;
-            if (t !== null) annotate(ts.tokenStartOfNode(st), `@type {${t}}`);
+            if (t !== null) annotate(ts.tokenStartOfNode(st), `@type {${t}}`, undefined, st.pos);
           }
         }
       }
@@ -525,7 +591,7 @@ export function planShapeOverlay(
         const init = d.initializer;
         if (ts.isIdentifier(d.name) && init !== undefined && !literalInitializer(init)) {
           const t = typeOf(analysis.declValues(d), false, 0);
-          if (t !== null) annotate(ts.tokenStartOfNode(node), `@type {${t}}`);
+          if (t !== null) annotate(ts.tokenStartOfNode(node), `@type {${t}}`, undefined, node.pos);
         }
       }
       ts.forEachChild(node, (c) => {
@@ -556,7 +622,6 @@ export function planShapeOverlay(
     const removed = new Set<ts.Node>();
     for (const branch of dead) {
       const start = ts.tokenStartOfNode(branch);
-      if (dropped?.has(start) === true) continue;
       if (ranges.some(([lo, hi]) => start >= lo && start < hi)) continue;
       if (hoistedNames(branch, removed) === null) continue;
       removed.add(branch);
@@ -584,9 +649,9 @@ export function planShapeOverlay(
         edits.push({
           pos: lo,
           end: hi,
+          id: lo,
           text: `{ /** ${MARK} #${lo} */ ${vars}throw new Error(${JSON.stringify(message)}); }`,
         });
-        deadBranches++;
       }
     }
     // Debugging: SCRIPTC_SHAPE_OVERLAY_ONLY=a-b keeps the annotations
@@ -603,7 +668,7 @@ export function planShapeOverlay(
       edits.push(...kept);
     }
     if (edits.length === 0) continue;
-    files.set(sf.fileName, applyEdits(neutralizeDocTags(sf), edits) + typedefText.join(""));
+    parts.set(sf.fileName, { text: sf.text, edits, suffix: typedefText.join(""), docRanges });
   }
   // Type aliases other files name these files' classes through.
   for (const [other, names] of exportedClasses) {
@@ -611,14 +676,169 @@ export function planShapeOverlay(
       .sort()
       .map((n) => `/** @typedef {${n}} __shape_class_${n} ${MARK} */`)
       .join("");
-    files.set(
-      other.fileName,
-      (files.get(other.fileName) ?? neutralizeDocTags(other)) + "\n" + aliases,
-    );
+    const part = parts.get(other.fileName);
+    if (part !== undefined)
+      parts.set(other.fileName, { ...part, suffix: `${part.suffix}\n${aliases}` });
+    else
+      parts.set(other.fileName, {
+        text: other.text,
+        edits: [],
+        suffix: `\n${aliases}`,
+        docRanges: [],
+      });
+  }
+  return { parts };
+}
+
+/** The overlay's files with the annotations a build dropped left out. */
+export function renderShapeOverlay(
+  plan: ShapeOverlayPlan,
+  exclusions: OverlayExclusions = NO_EXCLUSIONS,
+): ShapeOverlay {
+  const files = new Map<string, string>();
+  let annotations = 0;
+  let deadBranches = 0;
+  for (const [file, part] of plan.parts) {
+    const dropped = exclusions.annotations.get(file);
+    const edits = part.edits.filter((e) => e.id === undefined || dropped?.has(e.id) !== true);
+    for (const e of edits) {
+      if (e.id === undefined || e.text === ")") continue;
+      if (e.end !== undefined) deadBranches++;
+      else annotations++;
+    }
+    if (edits.length === 0 && part.suffix === "") continue;
+    files.set(file, applyEdits(neutralizedDocRanges(part, dropped), edits) + part.suffix);
   }
   return { files, annotations, deadBranches };
 }
 
+/** The part's text with the package JSDoc neutralized in the leading trivia
+ * of the annotations a build kept. Each range is one declaration's leading
+ * trivia, so two ranges are equal or disjoint; the text is built once, with
+ * the tags of all the annotations of a range neutralized. */
+function neutralizedDocRanges(part: OverlayPart, dropped: ReadonlySet<number> | undefined): string {
+  const ranges = part.docRanges
+    .filter((r) => dropped?.has(r.id) !== true)
+    .sort((a, b) => a.lo - b.lo || a.hi - b.hi);
+  if (ranges.length === 0) return part.text;
+  let out = "";
+  let at = 0;
+  for (let i = 0; i < ranges.length; i++) {
+    const r = ranges[i]!;
+    let replaces = r.replaces;
+    for (; i + 1 < ranges.length && ranges[i + 1]!.lo === r.lo && ranges[i + 1]!.hi === r.hi; i++) {
+      const next = ranges[i + 1]!.replaces;
+      replaces =
+        replaces === undefined || next === undefined
+          ? undefined
+          : {
+              returns: replaces.returns || next.returns,
+              params: [...replaces.params, ...next.params],
+            };
+    }
+    const lo = Math.max(r.lo, at);
+    if (r.hi <= lo) continue;
+    out += part.text.slice(at, lo) + neutralizeCommentTags(part.text.slice(lo, r.hi), replaces);
+    at = r.hi;
+  }
+  return out + part.text.slice(at);
+}
+
+/** A comment range's JSDoc type tags made plain text (the range is
+ * trivia, so it holds no literal). The overlay's own comments carry MARK
+ * and are kept. With `replaces`, only the tags a function's annotation
+ * replaces are. Lengths are unchanged. */
+export function neutralizeCommentTags(text: string, replaces?: FunctionTags): string {
+  return text.replace(/\/\*\*[\s\S]*?\*\//g, (comment) => {
+    if (comment.includes(MARK)) return comment;
+    if (replaces === undefined)
+      return comment.replace(
+        /@(param|type|typedef|returns?|template|this|callback|property|prop|arg|argument|enum|implements|extends|augments|satisfies|overload|import)\b/g,
+        "-$1",
+      );
+    return comment.replace(
+      /@(param|arg|argument|returns?|type|overload)\b/g,
+      (tag: string, name: string, at: number) => {
+        if (name === "type" || name === "overload") return `-${name}`;
+        if (name === "returns" || name === "return") return replaces.returns ? `-${name}` : tag;
+        const param = paramTagName(comment, at + tag.length);
+        return param !== null && replaces.params.includes(param) ? `-${name}` : tag;
+      },
+    );
+  });
+}
+
+/** The parameter a `@param` tag ending at `from` names (`{type} name`,
+ * `[name=default]`, or `name.member` of a documented object), or null. */
+function paramTagName(comment: string, from: number): string | null {
+  let i = from;
+  while (i < comment.length && /\s/.test(comment[i]!)) i++;
+  if (comment[i] === "{") {
+    let depth = 0;
+    for (; i < comment.length; i++) {
+      if (comment[i] === "{") depth++;
+      else if (comment[i] === "}" && --depth === 0) break;
+    }
+    i++;
+    while (i < comment.length && /\s/.test(comment[i]!)) i++;
+  }
+  if (comment[i] === "[") i++;
+  const name = /^[A-Za-z_$][\w$]*/.exec(comment.slice(i, i + 256));
+  return name !== null ? name[0] : null;
+}
+
+/** The `typeof` results the values can have. */
+function typeofResults(values: ReadonlySet<Atom>): Set<string> {
+  const out = new Set<string>();
+  for (const a of values) {
+    if (a.kind === "prim")
+      out.add(a.label === "null" ? "object" : a.label === "symbol" ? "symbol" : a.label);
+    else if (a.kind === "function" || a.kind === "class") out.add("function");
+    else out.add("object");
+  }
+  return out;
+}
+
+/** Whether the function's body compares `typeof name` with a result the
+ * values never have. */
+function typeofTestsOutside(
+  fn: ts.FunctionLikeDeclaration,
+  name: string,
+  values: ReadonlySet<Atom>,
+): boolean {
+  const results = typeofResults(values);
+  let outside = false;
+  const visit = (n: ts.Node): void => {
+    if (outside) return;
+    if (
+      ts.isBinaryExpression(n) &&
+      (n.operatorToken.kind === ts.SyntaxKind.EqualsEqualsEqualsToken ||
+        n.operatorToken.kind === ts.SyntaxKind.EqualsEqualsToken ||
+        n.operatorToken.kind === ts.SyntaxKind.ExclamationEqualsEqualsToken ||
+        n.operatorToken.kind === ts.SyntaxKind.ExclamationEqualsToken)
+    ) {
+      for (const [op, lit] of [
+        [n.left, n.right],
+        [n.right, n.left],
+      ] as const)
+        if (
+          ts.isTypeOfExpression(op) &&
+          ts.isIdentifier(op.expression) &&
+          op.expression.text === name &&
+          ts.isStringLiteral(lit) &&
+          !results.has(lit.text)
+        )
+          outside = true;
+    }
+    ts.forEachChild(n, (c) => {
+      visit(c);
+      return undefined;
+    });
+  };
+  if (fn.body !== undefined) visit(fn.body);
+  return outside;
+}
+
 function packageNameOf(fileName: string): string {
   const parts = (fileName.split("/node_modules/").pop() ?? "").split("/");
   return parts[0]!.startsWith("@") ? `${parts[0]}/${parts[1]}` : parts[0]!;

packages/compiler/test/shape-overlay-render.test.ts

@@ -0,0 +1,83 @@
+/* renderShapeOverlay neutralizes the package JSDoc in the leading trivia of
+ * the annotations a build keeps, and leaves the trivia of dropped ones. */
+import { expect, test } from "vitest";
+import {
+  neutralizeCommentTags,
+  OVERLAY_MARK,
+  renderShapeOverlay,
+} from "../src/frontend/shape-inference/overlay.js";
+import type { ShapeOverlayPlan } from "../src/frontend/shape-inference/overlay.js";
+
+test("each kept annotation's leading JSDoc is neutralized, once per range", () => {
+  const decls = [
+    "/** @param {string} a */\nfunction f(a) {}\n",
+    "/** @type {number} */\nvar x = 1;\n",
+    "/** @returns {void} */\nfunction g() {}\n",
+  ];
+  const text = decls.join("");
+  const docRanges: { lo: number; hi: number; id: number }[] = [];
+  const edits: { pos: number; id: number; text: string }[] = [];
+  let lo = 0;
+  for (const decl of decls) {
+    const hi = lo + decl.indexOf("\n") + 1;
+    docRanges.push({ lo, hi, id: hi });
+    // A declaration's first child starts at the same position: its range is
+    // the same trivia.
+    docRanges.push({ lo, hi, id: hi });
+    edits.push({ pos: hi, id: hi, text: `/** @type {any} ${OVERLAY_MARK} #${hi} */ ` });
+    lo += decl.length;
+  }
+  const plan: ShapeOverlayPlan = {
+    parts: new Map([["a.js", { text, edits, suffix: "", docRanges }]]),
+  };
+  const dropped = docRanges[2]!.id;
+  const overlay = renderShapeOverlay(plan, {
+    annotations: new Map([["a.js", new Set([dropped])]]),
+    files: new Set(),
+  });
+  const annotated = (decl: string, keep: boolean, at: number): string => {
+    const head = decl.slice(0, decl.indexOf("\n") + 1);
+    const rest = decl.slice(head.length);
+    return keep
+      ? `${neutralizeCommentTags(head)}/** @type {any} ${OVERLAY_MARK} #${at} */ ${rest}`
+      : decl;
+  };
+  expect(overlay.files.get("a.js")).toBe(
+    decls
+      .map((decl, i) => annotated(decl, docRanges[2 * i]!.id !== dropped, docRanges[2 * i]!.id))
+      .join(""),
+  );
+  expect(overlay.files.get("a.js")).toContain("/** @type {number} */");
+  expect(overlay.files.get("a.js")).toContain("/** -param {string} a */");
+  expect(overlay.annotations).toBe(2);
+});
+
+test("a function's annotation neutralizes only the tags it replaces", () => {
+  const head =
+    "/**\n * @template T\n * @param {{ a?: boolean }} options\n * @param {T} [b]\n * @param {string} options.c\n * @param {number} n\n * @returns {T}\n */\n";
+  expect(neutralizeCommentTags(head, { returns: true, params: ["n"] })).toBe(
+    "/**\n * @template T\n * @param {{ a?: boolean }} options\n * @param {T} [b]\n * @param {string} options.c\n * -param {number} n\n * -returns {T}\n */\n",
+  );
+  expect(neutralizeCommentTags(head, { returns: false, params: ["options", "b"] })).toBe(
+    "/**\n * @template T\n * -param {{ a?: boolean }} options\n * -param {T} [b]\n * -param {string} options.c\n * @param {number} n\n * @returns {T}\n */\n",
+  );
+  const text = `${head}function f(options, b, n) {}\n`;
+  const plan: ShapeOverlayPlan = {
+    parts: new Map([
+      [
+        "a.js",
+        {
+          text,
+          edits: [{ pos: head.length, id: head.length, text: "/** @returns {undefined} */ " }],
+          suffix: "",
+          docRanges: [
+            { lo: 0, hi: head.length, id: head.length, replaces: { returns: true, params: [] } },
+          ],
+        },
+      ],
+    ]),
+  };
+  const out = renderShapeOverlay(plan).files.get("a.js")!;
+  expect(out).toContain("@param {{ a?: boolean }} options");
+  expect(out).toContain("-returns {T}");
+});

tests/harness/npm-suite.ts

@@ -113,15 +113,8 @@ const KNOWN: Record<string, Known> = {
   "workspace-unclean": { build: "SC2013" },
 };
 
-/** Where the overlay changes how a case KNOWN refuses: the operation it
- * fails at on an argv list that fails in both builds. */
-const OVERLAY_KNOWN: Record<string, Known> = {
-  // `--help` reaches an unsupported operation in both builds; with the
-  // overlay it fails at a checked conversion of an option value first
-  // (results/design-shape-inference/NOTES.md, open item).
-  "commander-calc": { run: "expected boolean at $, got undefined", argvs: [["--help"]] },
-  "commander-calc-async": { run: "expected boolean at $, got undefined", argvs: [["--help"]] },
-};
+/** Where the overlay changes how a case KNOWN refuses (none today). */
+const OVERLAY_KNOWN: Record<string, Known> = {};
 
 /** Per-case timeouts past the default 120 s: the undici cases compile the
  * built-in undici implementation's module graph, the zod 3 cases (zod3,