diff --git a/packages/compiler/src/frontend/shape-inference/analyze.ts b/packages/compiler/src/frontend/shape-inference/analyze.ts index c0c2b5283..4c4513929 100644 --- a/packages/compiler/src/frontend/shape-inference/analyze.ts +++ b/packages/compiler/src/frontend/shape-inference/analyze.ts @@ -24,7 +24,13 @@ import * as ts from "../ts7/adapter.js"; import { isJsSourceFile } from "../program.js"; import { npmStaticPackageOfPath } from "../npm-static.js"; -import { AtomTable, MODELED_GLOBALS, STRING_METHOD_RESULTS, type Atom } from "./domain.js"; +import { + AtomTable, + MODELED_GLOBALS, + OBJECT_PROTOTYPE_MEMBERS, + STRING_METHOD_RESULTS, + type Atom, +} from "./domain.js"; type Node = ts.Node; @@ -36,6 +42,9 @@ export interface ValueNode { readonly onAtom: ((atom: Atom) => void)[]; /** Every atom that arrives escapes. */ escapeSink: string | null; + /** Widened (WIDEN_AT): the node holds ⊤, and every further atom that + * arrives escapes instead of flowing on. */ + wide?: true; } export interface FunctionInfo { @@ -185,6 +194,79 @@ const ARRAY_METHODS: ReadonlySet = new Set([ ]); const MAX_STEPS = 20_000_000; +/** The members String.prototype, Number.prototype and Boolean.prototype + * define (Node 24); Object.prototype's come from OBJECT_PROTOTYPE_MEMBERS. */ +const PRIMITIVE_MEMBERS: Readonly>> = { + string: new Set([ + "at", + "charAt", + "charCodeAt", + "codePointAt", + "concat", + "endsWith", + "includes", + "indexOf", + "isWellFormed", + "lastIndexOf", + "localeCompare", + "match", + "matchAll", + "normalize", + "padEnd", + "padStart", + "repeat", + "replace", + "replaceAll", + "search", + "slice", + "split", + "startsWith", + "substr", + "substring", + "toLocaleLowerCase", + "toLocaleUpperCase", + "toLowerCase", + "toString", + "toUpperCase", + "toWellFormed", + "trim", + "trimEnd", + "trimStart", + "trimLeft", + "trimRight", + "valueOf", + "anchor", + "big", + "blink", + "bold", + "fixed", + "fontcolor", + "fontsize", + "italics", + "link", + "small", + "strike", + "sub", + "sup", + "length", + "constructor", + ]), + number: new Set([ + "toExponential", + "toFixed", + "toLocaleString", + "toPrecision", + "toString", + "valueOf", + "constructor", + ]), + boolean: new Set(["toString", "valueOf", "constructor"]), +}; + +/** The most atoms a value node tracks before it widens to ⊤ + * (SCRIPTC_SHAPE_WIDEN overrides). */ +const WIDEN_AT = Number(process.env["SCRIPTC_SHAPE_WIDEN"] ?? 32); + function list(arr: { readonly nodes: readonly T[] } | undefined): readonly T[] { return arr?.nodes ?? []; } @@ -335,6 +417,49 @@ class Analyzer { this.top = this.atoms.intern("top", () => ({ kind: "top", label: "⊤" })); } + /** SCRIPTC_DEBUG_SHAPES_PROFILE: where the solver's steps go. */ + private profileReport(): void { + const byNodeLabel = new Map(); + let atomsTotal = 0; + let maxAtoms = 0; + let onAtoms = 0; + let succ = 0; + for (const n of this.nodes) { + atomsTotal += n.atoms.size; + maxAtoms = Math.max(maxAtoms, n.atoms.size); + onAtoms += n.onAtom.length; + succ += n.succ.size; + } + const kinds = new Map(); + for (const a of this.atoms.atoms) kinds.set(a.kind, (kinds.get(a.kind) ?? 0) + 1); + const ctxPerFn = new Map(); + for (const info of this.functions.values()) + ctxPerFn.set(info.node, (ctxPerFn.get(info.node) ?? 0) + 1); + const topFns = [...ctxPerFn.entries()].sort((a, b) => b[1] - a[1]).slice(0, 15); + const lines = [ + `nodes ${this.nodes.length} atoms ${this.atoms.atoms.length} (${[...kinds].map(([k, v]) => `${k} ${v}`).join(", ")})`, + `solve ${this.solveMs.toFixed(0)} ms (nested walks included) checker ${this.checkerMs.toFixed(0)} ms in ${this.checkerCalls} calls`, + `node-atom pairs ${atomsTotal} max ${maxAtoms} edges ${succ} listeners ${onAtoms} functions ${this.functions.size} steps ${this.steps}`, + ...topFns.map(([n, c]) => ` ${c} contexts: ${locLabel(n)}`), + ...[...this.popsByAtom.entries()] + .sort((a, b) => b[1] - a[1]) + .slice(0, 15) + .map(([a, c]) => ` ${c} pops: atom ${a.kind} ${a.label}`), + ...[...this.popsByNode.entries()] + .sort((a, b) => b[1] - a[1]) + .slice(0, 10) + .map( + ([id, c]) => + ` ${c} pops: node ${id} ${this.nodeLabel.get(id) ?? ""} atoms ${this.nodes[id]!.atoms.size} succ ${this.nodes[id]!.succ.size} listeners ${this.nodes[id]!.onAtom.length}`, + ), + ]; + void byNodeLabel; + process.stderr.write(lines.map((l) => `[shapes-profile] ${l}\n`).join("")); + } + private readonly profiling = process.env["SCRIPTC_DEBUG_SHAPES_PROFILE"] !== undefined; + private readonly popsByAtom = new Map(); + private readonly popsByNode = new Map(); + run(): ShapeAnalysis { for (const sf of this.program.getSourceFiles()) { if (sf.isDeclarationFile) continue; @@ -347,6 +472,7 @@ class Analyzer { for (const sf of this.files) this.checker.prefetchSourceFile(sf); for (const sf of this.files) this.moduleBody(sf); this.finish(); + if (this.profiling) this.profileReport(); return { atoms: this.atoms, nodes: this.nodes, @@ -393,7 +519,22 @@ class Analyzer { add(node: number, atom: Atom): void { const n = this.nodes[node]!; if (n.atoms.has(atom)) return; + if (n.wide === true) { + this.escape(atom, "a value set too large to track"); + return; + } n.atoms.add(atom); + // Widening: a value set this large is merged code (a utility every + // value flows through). It becomes ⊤: its values escape, so the facts + // stay sound, and the solver stops multiplying constraints per atom. + if (n.atoms.size > WIDEN_AT) { + n.wide = true; + for (const a of n.atoms) this.escape(a, "a value set too large to track"); + if (!n.atoms.has(this.top)) { + n.atoms.add(this.top); + this.worklist.push([node, this.top]); + } + } if (this.traceTop && atom === this.top && !this.topCause.has(node)) this.topCause.set( node, @@ -542,7 +683,17 @@ class Analyzer { return this.atoms.intern(key, () => ({ kind: "builtin", label: path, path, receiver })); } + private solveMs = 0; solve(): void { + const started = this.profiling ? performance.now() : 0; + try { + this.solveLoop(); + } finally { + if (this.profiling) this.solveMs += performance.now() - started; + } + } + + private solveLoop(): void { while (this.worklist.length > 0) { if (++this.steps > MAX_STEPS) { this.budgetExceeded = true; @@ -550,6 +701,10 @@ class Analyzer { } const [id, atom] = this.worklist.pop()!; const n = this.nodes[id]!; + if (this.profiling) { + this.popsByAtom.set(atom, (this.popsByAtom.get(atom) ?? 0) + 1); + this.popsByNode.set(id, (this.popsByNode.get(id) ?? 0) + 1); + } this.cause = id; for (const s of n.succ) this.add(s, atom); this.cause = null; @@ -765,7 +920,22 @@ class Analyzer { // ---- symbols ----------------------------------------------------------------- + private checkerMs = 0; + private checkerCalls = 0; + private readonly resolved = new Map(); resolve(id: Node): Node | undefined { + if (this.resolved.has(id)) return this.resolved.get(id); + const started = this.profiling ? performance.now() : 0; + const out = this.resolveUncached(id); + if (this.profiling) { + this.checkerMs += performance.now() - started; + this.checkerCalls++; + } + this.resolved.set(id, out); + return out; + } + + private resolveUncached(id: Node): Node | undefined { let sym = this.checker.getSymbolAtLocation(id); if (sym === undefined) return undefined; if (sym.flags & ts.SymbolFlags.Alias) { @@ -2198,8 +2368,15 @@ class Analyzer { return; case "prim": if (name === "length" && a.label === "string") this.add(dst, this.prim("number")); - else if (a.label === "string" || a.label === "number" || a.label === "boolean") - this.add(dst, this.builtin(`${a.label}.${name}`, a)); + else if (a.label === "string" && /^\d+$/.test(name)) { + this.add(dst, this.prim("string")); + this.add(dst, this.prim("undefined")); + } else if (a.label === "string" || a.label === "number" || a.label === "boolean") { + // A name the primitive's prototype chain lacks reads undefined. + if (PRIMITIVE_MEMBERS[a.label].has(name) || OBJECT_PROTOTYPE_MEMBERS.has(name)) + this.add(dst, this.builtin(`${a.label}.${name}`, a)); + else this.add(dst, this.prim("undefined")); + } return; case "instance": { const decl = a.node!; @@ -2358,6 +2535,7 @@ class Analyzer { } else if ( a.receiver === undefined && MODELED_GLOBALS.has(path.split(".")[0]!) && + !path.endsWith(".instance") && path.split(".").length < 4 ) this.add(dst, this.builtin(`${path}.${name}`)); diff --git a/packages/compiler/test/shape-inference.test.ts b/packages/compiler/test/shape-inference.test.ts index 911cef79a..8bf97de10 100644 --- a/packages/compiler/test/shape-inference.test.ts +++ b/packages/compiler/test/shape-inference.test.ts @@ -102,6 +102,10 @@ export function jsonCallbacks(k) { export function revivedResult(text) { const r = { fromReviver: 1 }; return leak(JSON.parse(text, function () { return r; })); } export function constructedProxy() { return Reflect.construct(Proxy, [{ constructedProxy: 1 }, {}]); } export function globalProxy() { return new globalThis.Proxy({ globalProxy: 1 }, {}); } +function same(value) { return value; } +export function manyShapes() { + return [same({ w0: 0 }), same({ w1: 1 }), same({ w2: 2 }), same({ w3: 3 }), same({ w4: 4 }), same({ w5: 5 }), same({ w6: 6 }), same({ w7: 7 }), same({ w8: 8 }), same({ w9: 9 }), same({ w10: 10 }), same({ w11: 11 }), same({ w12: 12 }), same({ w13: 13 }), same({ w14: 14 }), same({ w15: 15 }), same({ w16: 16 }), same({ w17: 17 }), same({ w18: 18 }), same({ w19: 19 }), same({ w20: 20 }), same({ w21: 21 }), same({ w22: 22 }), same({ w23: 23 }), same({ w24: 24 }), same({ w25: 25 }), same({ w26: 26 }), same({ w27: 27 }), same({ w28: 28 }), same({ w29: 29 }), same({ w30: 30 }), same({ w31: 31 }), same({ w32: 32 }), same({ w33: 33 }), same({ w34: 34 }), same({ w35: 35 }), same({ w36: 36 }), same({ w37: 37 }), same({ w38: 38 }), same({ w39: 39 })].length; +} `; const packageTypes = ` @@ -125,6 +129,7 @@ export declare function jsonCallbacks(k: string): unknown; export declare function revivedResult(text: string): unknown; export declare function constructedProxy(): unknown; export declare function globalProxy(): unknown; +export declare function manyShapes(): number; `; const mainSource = ` @@ -134,6 +139,7 @@ import { computedField, constructedProxy, globalProxy, + manyShapes, jsonCallbacks, leak, obj, @@ -165,6 +171,7 @@ const more = [ revivedResult("{}"), constructedProxy(), globalProxy(), + manyShapes(), ]; console.log(more.length); `; @@ -284,6 +291,8 @@ test("S1 report follows values through array properties, unknown keys, callbacks // a proxy constructed through Reflect.construct and globalThis expect(site(lines, /^site \{constructedProxy\}@/)).toMatch(/: G — proxy at/); expect(site(lines, /^site \{globalProxy\}@/)).toMatch(/: G — proxy at/); + // a value set past the widening limit: its values escape + expect(site(lines, /^site \{w0\}@/)).toMatch(/: P-escaping — escapes: a value set too large/); // the existing verdicts hold expect(site(lines, /^site \{kept\}@main\.ts/)).toMatch(/: P-escaping — escapes: stored into ⊤/); }); diff --git a/tests/harness/npm-suite.ts b/tests/harness/npm-suite.ts index 85785364f..b0f9bc536 100644 --- a/tests/harness/npm-suite.ts +++ b/tests/harness/npm-suite.ts @@ -112,6 +112,16 @@ const KNOWN: Record = { "workspace-unclean": { build: "SC2013" }, }; +/** Where the overlay changes how a case KNOWN refuses: the operation it + * fails at on an argv list that fails in both builds. */ +const OVERLAY_KNOWN: Record = { + // `--help` reaches an unsupported operation in both builds; with the + // overlay it fails at a checked conversion of an option value first + // (results/design-shape-inference/NOTES.md, open item). + "commander-calc": { run: "expected number at $, got undefined", argvs: [["--help"]] }, + "commander-calc-async": { run: "expected number at $, got undefined", argvs: [["--help"]] }, +}; + /** Per-case timeouts past the default 120 s: the undici cases compile the * built-in undici implementation's module graph, the zod 3 cases (zod3, * zod3-adoption, zod3-boundary, zod3-infer) compile zod 3's 3,700-line @@ -234,7 +244,7 @@ export function defineNpmSuite(options: { label: string; shapeOverlay?: boolean describe(`${label} (${cases.length} programs${sanitize ? ", sanitized" : ""}${shardSuffix()})`, () => { for (const c of cases) { - const known = KNOWN[c.name]; + const known = (shapeOverlay === true ? OVERLAY_KNOWN[c.name] : undefined) ?? KNOWN[c.name]; if (known !== undefined && "divergence" in known) { test.fails(`${c.name} (diverges: ${known.divergence})`, async () => { const built = await build(c.entry, shapeOverlay);