From 34eef83ef4c5364995e4d47174ce65acc1fc24b8 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Sun, 25 Oct 2026 09:19:53 -0700 Subject: [PATCH 01/21] Decide concat spreading of dynamic values at run time; keep JS lets assigned from index reads unchecked Three JavaScript lowering bugs threw a checked conversion where Node answers: - concat with a checked-dynamic argument lowered it as an element, so an array argument threw. The argument now spreads when it is an array at run time and is appended otherwise. - `let c; while ((c = str[i++]) !== "]" && c)`: the checker types c from the index read, without the undefined a read past the end answers. Evolving-let inference ignores index-read writes, and reads of a JS `let x;` statement binding keep the actual value. - Strict equality on a flow-narrowed dynamic JS name compares unchecked. npm cases concat-dynamic-args and evolving-let-index match Node. Co-Authored-By: Claude Opus 5.5 --- .../lowering/containers/array-concat.ts | 59 +++++++++++++++++-- .../lowering/containers/array-methods.ts | 12 +++- .../frontend/lowering/expressions/binary.ts | 27 +++++++++ .../lowering/expressions/narrowing.ts | 21 +++++++ .../src/frontend/lowering/lower-stmts.ts | 8 +++ .../npm/cases/concat-dynamic-args/main.ts | 3 + .../node_modules/concat-dynamic/index.d.ts | 1 + .../node_modules/concat-dynamic/index.js | 12 ++++ .../node_modules/concat-dynamic/package.json | 1 + .../cases/concat-dynamic-args/package.json | 1 + .../npm/cases/evolving-let-index/main.ts | 4 ++ .../node_modules/evolving-let/index.d.ts | 1 + .../node_modules/evolving-let/index.js | 24 ++++++++ .../node_modules/evolving-let/package.json | 1 + .../npm/cases/evolving-let-index/package.json | 1 + 15 files changed, 169 insertions(+), 7 deletions(-) create mode 100644 tests/fixtures/npm/cases/concat-dynamic-args/main.ts create mode 100644 tests/fixtures/npm/cases/concat-dynamic-args/node_modules/concat-dynamic/index.d.ts create mode 100644 tests/fixtures/npm/cases/concat-dynamic-args/node_modules/concat-dynamic/index.js create mode 100644 tests/fixtures/npm/cases/concat-dynamic-args/node_modules/concat-dynamic/package.json create mode 100644 tests/fixtures/npm/cases/concat-dynamic-args/package.json create mode 100644 tests/fixtures/npm/cases/evolving-let-index/main.ts create mode 100644 tests/fixtures/npm/cases/evolving-let-index/node_modules/evolving-let/index.d.ts create mode 100644 tests/fixtures/npm/cases/evolving-let-index/node_modules/evolving-let/index.js create mode 100644 tests/fixtures/npm/cases/evolving-let-index/node_modules/evolving-let/package.json create mode 100644 tests/fixtures/npm/cases/evolving-let-index/package.json diff --git a/packages/compiler/src/frontend/lowering/containers/array-concat.ts b/packages/compiler/src/frontend/lowering/containers/array-concat.ts index a7ce0a106..e3164689e 100644 --- a/packages/compiler/src/frontend/lowering/containers/array-concat.ts +++ b/packages/compiler/src/frontend/lowering/containers/array-concat.ts @@ -3,6 +3,7 @@ import { identityPreservingWidening } from "../coercions/identity.js"; import { countedFor, numLit, varRef } from "../../../ir/build.js"; import { BOOL, + DYN, F64, arrayOf, typeEquals, @@ -18,15 +19,18 @@ import type { Lowerer } from "../lowerer.js"; /** Copy into fresh storage after every argument has evaluated. Array * arguments retain their own element ABI; only copied values are widened. - * Missing slots stay holes and present undefined stays present. */ + * Missing slots stay holes and present undefined stays present. A shape + * entry is an array argument's element type, null for an element value, or + * "dyn" for a checked-dynamic argument, which JS spreads when it is an + * array at run time (IsConcatSpreadable) and appends otherwise. */ export function arrayConcatHelper( lowerer: Lowerer, elem: IrType, - shape: (IrType | null)[], + shape: (IrType | null | "dyn")[], node: ts.CallExpression, loc: SrcLoc, ): string { - const key = `concat:${typeKey(elem)}:${shape.map((type) => (type ? `a:${typeKey(type)}` : "e")).join(";")}`; + const key = `concat:${typeKey(elem)}:${shape.map((type) => (type === "dyn" ? "d" : type ? `a:${typeKey(type)}` : "e")).join(";")}`; const existing = lowerer.arrHofHelpers.get(key); if (existing) return existing; const name = keyedHelperName(lowerer, "%arr.concat", key); @@ -36,7 +40,7 @@ export function arrayConcatHelper( ...shape.map((type, i) => ({ localId: `x.${i}`, name: `x${i}`, - type: type ? arrayOf(type) : elem, + type: type === "dyn" ? DYN : type ? arrayOf(type) : elem, })), ]; const locals: IrLocal[] = params.map((p) => ({ @@ -146,6 +150,43 @@ export function arrayConcatHelper( }, ]; }; + // A checked-dynamic argument: an array spreads its elements, converted + // to the element type; any other value is appended as one element. + const appendDyn = (source: IrExpr, site: ts.Node): IrStmt[] => [ + { + kind: "if", + cond: { kind: "dynTest", test: "array", value: source, type: BOOL, loc }, + then: [ + { + kind: "exprStmt", + expr: { + kind: "arrIntrinsic", + method: "concatSpread", + receiver: out, + args: [lowerer.coerceInto(site, source, arrT)], + type: F64, + loc, + }, + loc, + }, + ], + else_: [ + { + kind: "exprStmt", + expr: { + kind: "arrIntrinsic", + method: "push", + receiver: out, + args: [lowerer.coerceInto(site, source, elem)], + type: F64, + loc, + }, + loc, + }, + ], + loc, + }, + ]; const body: IrStmt[] = [ { kind: "varDecl", @@ -155,11 +196,17 @@ export function arrayConcatHelper( }, ...append(varRef("a.0", arrT, loc), elem, node.expression), ...shape.flatMap((type, i) => - append(varRef(`x.${i}`, type ? arrayOf(type) : elem, loc), type, node.arguments.nodes[i]!), + type === "dyn" + ? appendDyn(varRef(`x.${i}`, DYN, loc), node.arguments.nodes[i]!) + : append( + varRef(`x.${i}`, type ? arrayOf(type) : elem, loc), + type, + node.arguments.nodes[i]!, + ), ), { kind: "return", value: out, loc }, ]; - if (shape.some((type) => type && !typeEquals(type, elem))) { + if (shape.some((type) => type && type !== "dyn" && !typeEquals(type, elem))) { for (const id of ["i.0", "offset.0", "count.0"]) locals.push({ id, name: id, type: F64, mutable: true }); } diff --git a/packages/compiler/src/frontend/lowering/containers/array-methods.ts b/packages/compiler/src/frontend/lowering/containers/array-methods.ts index 4ef203edd..7f0cd6396 100644 --- a/packages/compiler/src/frontend/lowering/containers/array-methods.ts +++ b/packages/compiler/src/frontend/lowering/containers/array-methods.ts @@ -924,7 +924,7 @@ export function lowerArrayMethodCall( // an array-of-arrays receiver given a bare inner array, where TS // types it as an element but JS would SPREAD it — is fenced. const receiver = lowerer.lowerExpr(access.expression); - const shape: (IrType | null)[] = []; + const shape: (IrType | null | "dyn")[] = []; const args: IrExpr[] = []; for (const argNode of call.arguments.nodes) { if (ts.isSpreadElement(argNode)) { @@ -958,6 +958,16 @@ export function lowerArrayMethodCall( args.push(arg); continue; } + // A checked-dynamic value: whether JS spreads it is decided at run + // time (an array spreads one level, anything else is one element). + if ( + (argIr === null || argIr.kind === "dyn") && + (probedArg ?? tryLowerExpression(lowerer, argNode))?.type.kind === "dyn" + ) { + shape.push("dyn"); + args.push(lowerer.lowerExpr(argNode)); + continue; + } // An element value — union elements wrap exactly like a push. shape.push(null); args.push(lowerer.lowerExprExpecting(argNode, elem)); diff --git a/packages/compiler/src/frontend/lowering/expressions/binary.ts b/packages/compiler/src/frontend/lowering/expressions/binary.ts index 00a23caf5..8d6181a6e 100644 --- a/packages/compiler/src/frontend/lowering/expressions/binary.ts +++ b/packages/compiler/src/frontend/lowering/expressions/binary.ts @@ -2280,6 +2280,33 @@ export function lowerStrictEquality( negated: boolean, loc: SrcLoc, ): IrExpr { + // JavaScript: a checked-dynamic name the checker's flow analysis narrowed + // (`let c; ... (c = s[i]) === " "`) is compared unchecked. Strict + // equality never throws, and the flow type is the checker's inference + // (an indexed read past the end is undefined whatever it says). + const flowNarrowed = (n: ts.Node, e: IrExpr): IrExpr | null => + ts.isIdentifier(n) && + e.kind === "dynCheck" && + e.value.type.kind === "dyn" && + isJsSourceFile(ts.getSourceFileOfNode(n)) + ? e.value + : null; + const scalar = (e: IrExpr): boolean => + e.type.kind === "string" || e.type.kind === "f64" || e.type.kind === "bool"; + { + const l = flowNarrowed(leftNode, left); + const r = flowNarrowed(rightNode, right); + if ((l !== null && (scalar(right) || r !== null)) || (r !== null && scalar(left))) { + const cmp: IrExpr = { + kind: "dynScalarEq", + left: l ?? lowerer.coerceInto(leftNode, left, DYN), + right: r ?? lowerer.coerceInto(rightNode, right, DYN), + type: BOOL, + loc, + }; + return negated ? { kind: "unary", op: "!", operand: cmp, type: BOOL, loc } : cmp; + } + } const plainBothNum = left.type.kind === "f64" && right.type.kind === "f64"; const bothStr = left.type.kind === "string" && right.type.kind === "string"; const indexed = tryLowerIndexedComparison(lowerer, left, right, negated, loc); diff --git a/packages/compiler/src/frontend/lowering/expressions/narrowing.ts b/packages/compiler/src/frontend/lowering/expressions/narrowing.ts index 01cab9901..3bedb7acb 100644 --- a/packages/compiler/src/frontend/lowering/expressions/narrowing.ts +++ b/packages/compiler/src/frontend/lowering/expressions/narrowing.ts @@ -513,6 +513,27 @@ export function maybeNarrow(lowerer: Lowerer, expr: IrExpr, node: ts.Node): IrEx ) ) return expr; + // A `let c;` binding without a type takes its flow types from what + // the code assigns, and the checker's types for those values can omit + // undefined (`c = s[i]` past the end): reads keep the actual value. + if ( + symbol && + symbol.declarations.some( + (decl) => + ts.isVariableDeclaration(decl) && + decl.initializer === undefined && + decl.type === undefined && + ts.isIdentifier(decl.name) && + decl.parent?.parent !== undefined && + ts.isVariableStatement(decl.parent.parent) && + !/@type\b/.test( + ts + .getSourceFileOfNode(decl) + .text.slice(decl.parent?.parent?.pos ?? decl.pos, ts.tokenStartOfNode(decl)), + ), + ) + ) + return expr; if ( symbol && symbol.declarations.some( diff --git a/packages/compiler/src/frontend/lowering/lower-stmts.ts b/packages/compiler/src/frontend/lowering/lower-stmts.ts index 96d185b44..7feb2d351 100644 --- a/packages/compiler/src/frontend/lowering/lower-stmts.ts +++ b/packages/compiler/src/frontend/lowering/lower-stmts.ts @@ -5367,6 +5367,14 @@ function inferredEvolvingLetType(lowerer: Lowerer, decl: ts.VariableDeclaration) invalid = true; return "stop"; } + // An index read's checker type omits the undefined a read past the + // end (or of an absent key) answers. + let right = node.right; + while (ts.isParenthesizedExpression(right)) right = right.expression; + if (ts.isElementAccessExpression(right)) { + invalid = true; + return "stop"; + } const mapped = lowerer.mapTypeOf(lowerer.typeOf(node.right)); if (!mapped || mapped.kind === "dyn" || mapped.kind === "void" || isUnitType(mapped)) { invalid = true; diff --git a/tests/fixtures/npm/cases/concat-dynamic-args/main.ts b/tests/fixtures/npm/cases/concat-dynamic-args/main.ts new file mode 100644 index 000000000..35d6e0268 --- /dev/null +++ b/tests/fixtures/npm/cases/concat-dynamic-args/main.ts @@ -0,0 +1,3 @@ +// concat of checked-dynamic values decides spreading at run time. +import { run } from "concat-dynamic"; +console.log(run(JSON.stringify({ text: "desc", list: ["a", "b"] }))); diff --git a/tests/fixtures/npm/cases/concat-dynamic-args/node_modules/concat-dynamic/index.d.ts b/tests/fixtures/npm/cases/concat-dynamic-args/node_modules/concat-dynamic/index.d.ts new file mode 100644 index 000000000..615876884 --- /dev/null +++ b/tests/fixtures/npm/cases/concat-dynamic-args/node_modules/concat-dynamic/index.d.ts @@ -0,0 +1 @@ +export declare function run(raw: string): string; diff --git a/tests/fixtures/npm/cases/concat-dynamic-args/node_modules/concat-dynamic/index.js b/tests/fixtures/npm/cases/concat-dynamic-args/node_modules/concat-dynamic/index.js new file mode 100644 index 000000000..e2bc8066f --- /dev/null +++ b/tests/fixtures/npm/cases/concat-dynamic-args/node_modules/concat-dynamic/index.js @@ -0,0 +1,12 @@ +'use strict'; +// Array.prototype.concat spreads an argument that is an array at run time and +// appends any other value, whatever the static type says. +function run(raw) { + const parsed = JSON.parse(raw); + let output = ['Usage:', '']; + output = output.concat([parsed.text, '']); + output = output.concat(parsed.list); + output = output.concat(parsed.text); + return output.join('|') + ' ' + output.length; +} +module.exports = { run }; diff --git a/tests/fixtures/npm/cases/concat-dynamic-args/node_modules/concat-dynamic/package.json b/tests/fixtures/npm/cases/concat-dynamic-args/node_modules/concat-dynamic/package.json new file mode 100644 index 000000000..afd15f22b --- /dev/null +++ b/tests/fixtures/npm/cases/concat-dynamic-args/node_modules/concat-dynamic/package.json @@ -0,0 +1 @@ +{ "name": "concat-dynamic", "version": "1.0.0", "main": "index.js", "types": "index.d.ts" } diff --git a/tests/fixtures/npm/cases/concat-dynamic-args/package.json b/tests/fixtures/npm/cases/concat-dynamic-args/package.json new file mode 100644 index 000000000..55b029ac8 --- /dev/null +++ b/tests/fixtures/npm/cases/concat-dynamic-args/package.json @@ -0,0 +1 @@ +{ "name": "concat-dynamic-args-case", "private": true, "type": "module" } diff --git a/tests/fixtures/npm/cases/evolving-let-index/main.ts b/tests/fixtures/npm/cases/evolving-let-index/main.ts new file mode 100644 index 000000000..f34197343 --- /dev/null +++ b/tests/fixtures/npm/cases/evolving-let-index/main.ts @@ -0,0 +1,4 @@ +// Reads of a `let` binding whose checker type comes from string index reads +// see undefined past the end. +import { countUntilBracket, skipSpaces } from "evolving-let"; +console.log(skipSpaces("ab ", 2), skipSpaces("a b", 1), countUntilBracket("abc"), countUntilBracket("ab]c")); diff --git a/tests/fixtures/npm/cases/evolving-let-index/node_modules/evolving-let/index.d.ts b/tests/fixtures/npm/cases/evolving-let-index/node_modules/evolving-let/index.d.ts new file mode 100644 index 000000000..51158faab --- /dev/null +++ b/tests/fixtures/npm/cases/evolving-let-index/node_modules/evolving-let/index.d.ts @@ -0,0 +1 @@ +export declare function skipSpaces(str: string, ptr: number): number; export declare function countUntilBracket(str: string): number; diff --git a/tests/fixtures/npm/cases/evolving-let-index/node_modules/evolving-let/index.js b/tests/fixtures/npm/cases/evolving-let-index/node_modules/evolving-let/index.js new file mode 100644 index 000000000..37cbcac70 --- /dev/null +++ b/tests/fixtures/npm/cases/evolving-let-index/node_modules/evolving-let/index.js @@ -0,0 +1,24 @@ +'use strict'; +// `let c;` takes what an index read past the end answers: undefined. +/** + * @param {string} str + * @param {number} ptr + * @returns {number} + */ +function skipSpaces(str, ptr) { + let c; + while ((c = str[ptr]) === ' ' || c === '\t') ptr++; + return ptr; +} +/** + * @param {string} str + * @returns {number} + */ +function countUntilBracket(str) { + let c; + let ptr = 0; + let n = 0; + while ((c = str[ptr++]) !== ']' && c) n++; + return n; +} +module.exports = { skipSpaces, countUntilBracket }; diff --git a/tests/fixtures/npm/cases/evolving-let-index/node_modules/evolving-let/package.json b/tests/fixtures/npm/cases/evolving-let-index/node_modules/evolving-let/package.json new file mode 100644 index 000000000..02f3315c7 --- /dev/null +++ b/tests/fixtures/npm/cases/evolving-let-index/node_modules/evolving-let/package.json @@ -0,0 +1 @@ +{ "name": "evolving-let", "version": "1.0.0", "main": "index.js", "types": "index.d.ts" } diff --git a/tests/fixtures/npm/cases/evolving-let-index/package.json b/tests/fixtures/npm/cases/evolving-let-index/package.json new file mode 100644 index 000000000..ec244a62b --- /dev/null +++ b/tests/fixtures/npm/cases/evolving-let-index/package.json @@ -0,0 +1 @@ +{ "name": "evolving-let-index-case", "private": true, "type": "module" } From 5455a4d81785f592e2e6902900388442fc8edcc9 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Sun, 25 Oct 2026 09:20:03 -0700 Subject: [PATCH 02/21] Make the shape overlay's facts sound on the real-package suites The overlay validation suites (npm, npm-static, three, Effect 3 and 4) now pass with the overlay on. Each cause had an npm repro or suite case that failed with the overlay and passed without it: Analysis facts: - Symbol members: Symbol.for/keyFor are functions, well-known symbols are symbols, other members unknown. - Reads that may answer undefined: index reads past the end (unless the enclosing for loop proves them in bounds of an array that never shrinks or has holes), computed reads of objects, keys an object literal lacks at creation, optional-chain short circuits, `let x;` before assignment. - Object.create(proto) inherits proto members; arguments past the declared parameters are unknown; `new` of a primitive answers nothing. - Arithmetic on bigints answers bigints (number|bigint is never printed). Overlay: - Package JSDoc is a declaration: it stays as written and the overlay adds only types the JSDoc does not declare. - Objects held by module-level bindings (and what they hold) are never typed as records: a typed return would copy them. - Only checks of overlay facts carry the inferred-fact message. - static-objects ignores nullish receivers outside optional chains; folded literal keys are unparenthesized. Fallback (the overlay never adds a failure the default build lacks): - the overlay state is kept per load (a later load reset it, so the fallback never ran); - deferred refusals in unreached code count; - a preflight, deferred refusals or IR the program without the overlay does not have make the build compile without the overlay. CI: the shape_overlay job builds its own native packages; a manual run with scope shape_overlay runs it alone. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 85 +++- packages/compiler/src/executable/prepare.ts | 13 +- packages/compiler/src/frontend/pipeline.ts | 135 ++++++- packages/compiler/src/frontend/program.ts | 6 + .../src/frontend/shape-inference/analyze.ts | 372 ++++++++++++++++-- .../shape-inference/constant-graphs.ts | 10 +- .../src/frontend/shape-inference/fallback.ts | 2 +- .../src/frontend/shape-inference/index.ts | 67 +++- .../shape-inference/inferred-checks.ts | 88 +++++ .../src/frontend/shape-inference/overlay.ts | 193 ++++++--- .../shape-inference/static-objects.ts | 21 +- .../test/shape-overlay-render.test.ts | 53 +-- .../npm/cases/shape-absent-reads/main.ts | 3 + .../node_modules/absent-reads/index.d.ts | 1 + .../node_modules/absent-reads/index.js | 48 +++ .../node_modules/absent-reads/package.json | 1 + .../npm/cases/shape-absent-reads/package.json | 1 + .../npm/cases/shape-bigint-arith/main.ts | 3 + .../node_modules/bigint-arith/index.d.ts | 1 + .../node_modules/bigint-arith/index.js | 15 + .../node_modules/bigint-arith/package.json | 1 + .../npm/cases/shape-bigint-arith/package.json | 1 + .../npm/cases/shape-module-identity/main.ts | 4 + .../node_modules/module-identity/index.d.ts | 1 + .../node_modules/module-identity/index.js | 21 + .../node_modules/module-identity/package.json | 1 + .../cases/shape-module-identity/package.json | 1 + tests/harness/ci-stack-scope.test.ts | 15 +- tests/harness/npm-static.test.ts | 5 +- 29 files changed, 1024 insertions(+), 144 deletions(-) create mode 100644 packages/compiler/src/frontend/shape-inference/inferred-checks.ts create mode 100644 tests/fixtures/npm/cases/shape-absent-reads/main.ts create mode 100644 tests/fixtures/npm/cases/shape-absent-reads/node_modules/absent-reads/index.d.ts create mode 100644 tests/fixtures/npm/cases/shape-absent-reads/node_modules/absent-reads/index.js create mode 100644 tests/fixtures/npm/cases/shape-absent-reads/node_modules/absent-reads/package.json create mode 100644 tests/fixtures/npm/cases/shape-absent-reads/package.json create mode 100644 tests/fixtures/npm/cases/shape-bigint-arith/main.ts create mode 100644 tests/fixtures/npm/cases/shape-bigint-arith/node_modules/bigint-arith/index.d.ts create mode 100644 tests/fixtures/npm/cases/shape-bigint-arith/node_modules/bigint-arith/index.js create mode 100644 tests/fixtures/npm/cases/shape-bigint-arith/node_modules/bigint-arith/package.json create mode 100644 tests/fixtures/npm/cases/shape-bigint-arith/package.json create mode 100644 tests/fixtures/npm/cases/shape-module-identity/main.ts create mode 100644 tests/fixtures/npm/cases/shape-module-identity/node_modules/module-identity/index.d.ts create mode 100644 tests/fixtures/npm/cases/shape-module-identity/node_modules/module-identity/index.js create mode 100644 tests/fixtures/npm/cases/shape-module-identity/node_modules/module-identity/package.json create mode 100644 tests/fixtures/npm/cases/shape-module-identity/package.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index edb2b5731..95137c5f1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,6 +17,11 @@ on: type: choice options: [full, darwin] default: full + scope: + description: Every lane, or only the shape overlay validation (shape_overlay) + type: choice + options: [all, shape_overlay] + default: all concurrency: # A new push to a pull request cancels its previous run. Every push to main runs to completion in a group of @@ -55,7 +60,10 @@ jobs: const upperLayer = process.env.GITHUB_EVENT_NAME === 'pull_request' && sameRepository && trustedAuthor && typeof stackBase === 'string' && stackBase.length > 0 && typeof prBase === 'string' && prBase.length > 0 && stackBase !== prBase; - const full = !upperLayer; + // A manual run of the shape overlay validation alone skips the other lanes. + const shapeOnly = process.env.GITHUB_EVENT_NAME === 'workflow_dispatch' && + event.inputs?.scope === 'shape_overlay'; + const full = !upperLayer && !shapeOnly; appendFileSync(process.env.GITHUB_OUTPUT, `full=${full}\n`); console.log(full ? 'Run full CI' : 'Defer full CI until this PR becomes the lowest unmerged layer'); NODE @@ -346,6 +354,81 @@ jobs: - run: tar -xzf "$RUNNER_TEMP/native-linux-x64-gnu.tar.gz" - run: pnpm test tests/harness/effect.test.ts tests/harness/effect4.test.ts + # Shape inference's overlay (opt-in, SCRIPTC_SHAPE_OVERLAY=1) over the + # suites of real packages: nightly and on manual runs only (scope + # shape_overlay runs it alone), outside the `test` roll-up. Every case + # must build and run as in the default build. It builds its own native + # packages, so it does not depend on the other lanes. + shape_overlay_native: + if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} + name: build (Linux x64 native packages, shape overlay) + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + with: + version: 11.1.3 + - uses: actions/setup-node@v4 + with: + node-version-file: .node-version + cache: pnpm + - uses: ./.github/actions/llvm-linux + - run: pnpm install --frozen-lockfile + - uses: ./.github/actions/zig + - uses: ./.github/actions/native-packages + with: + target: linux-x64-gnu + - name: Preserve executable permissions in shared packages + run: >- + tar -czf "$RUNNER_TEMP/native-linux-x64-gnu.tar.gz" + packages/llvm-linux-x64-gnu/bin + packages/runtime-linux-x64-gnu/artifacts + packages/runtime-linux-x64-gnu/runtime-pack.json + - uses: actions/upload-artifact@v4 + with: + name: native-linux-x64-gnu-shape-overlay + path: ${{ runner.temp }}/native-linux-x64-gnu.tar.gz + retention-days: 1 + + shape_overlay: + needs: [shape_overlay_native] + if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} + name: test (Linux, shape overlay, ${{ matrix.suite }} ${{ matrix.shard }}/8) + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + suite: [npm, npm-static, three, effect] + shard: [1, 2, 3, 4, 5, 6, 7, 8] + env: + SCRIPTC_SHAPE_OVERLAY: "1" + SCRIPTC_TEST_WORKERS: "1" + SCRIPTC_EFFECT_TEST_CONCURRENCY: "1" + SCRIPTC_TEST_SHARD: ${{ matrix.shard }}/8 + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + with: + version: 11.1.3 + - uses: actions/setup-node@v4 + with: + node-version-file: .node-version + cache: pnpm + - run: pnpm install --frozen-lockfile + - uses: actions/download-artifact@v4 + with: + name: native-linux-x64-gnu-shape-overlay + path: ${{ runner.temp }} + - run: tar -xzf "$RUNNER_TEMP/native-linux-x64-gnu.tar.gz" + - name: Run the suite with the overlay on + run: | + case "${{ matrix.suite }}" in + npm) pnpm test tests/harness/npm-shape-overlay.test.ts ;; + npm-static) pnpm test tests/harness/npm-static.test.ts ;; + three) pnpm test tests/harness/three.test.ts ;; + effect) pnpm test tests/harness/effect.test.ts tests/harness/effect4.test.ts ;; + esac + # Build each instrumented seed once. Downstream contracts retain the same # compiler and source artifacts without repeating seed construction. bootstrap_seed: diff --git a/packages/compiler/src/executable/prepare.ts b/packages/compiler/src/executable/prepare.ts index 1771f98f5..3381dc4a3 100644 --- a/packages/compiler/src/executable/prepare.ts +++ b/packages/compiler/src/executable/prepare.ts @@ -68,11 +68,14 @@ export function prepareExecutableModule( if (fe.preflight.length > 0) return fail(fe.preflight); try { - lowered = fe.lower({ - targetPlatform: buildPlatform, - ...(ffi !== null ? { ffiImports: ffi.functions } : {}), - ...(opts.tier !== undefined ? { tierMode: opts.tier } : {}), - }); + lowered = fe.lower( + { + targetPlatform: buildPlatform, + ...(ffi !== null ? { ffiImports: ffi.functions } : {}), + ...(opts.tier !== undefined ? { tierMode: opts.tier } : {}), + }, + (module) => validateModule(module).length === 0, + ); timing("lower"); } catch (e) { // The last-resort panic fence: an upstream tsgo panic that crossed a diff --git a/packages/compiler/src/frontend/pipeline.ts b/packages/compiler/src/frontend/pipeline.ts index bb1053cb5..e1ea856c5 100644 --- a/packages/compiler/src/frontend/pipeline.ts +++ b/packages/compiler/src/frontend/pipeline.ts @@ -3,10 +3,13 @@ import { beginShapeOverlayBuild, excludeRefusedOverlay, + restoreShapeOverlayState, + setShapeOverlayDropped, shapeOverlayActive, shapeOverlayReport, } from "./shape-inference/index.js"; import type { ScrDiagnostic } from "../diagnostics/diagnostic.js"; +import type { IrModule } from "../ir/ir.js"; import type { SrcLoc } from "../ir/ir.js"; import type { NpmStaticStatus } from "../coverage/report.js"; import { entryFunctionExports, type EntryExportInfo } from "./lib-exports.js"; @@ -104,7 +107,9 @@ export interface Frontend { * dispose — it reads the ts7 AST). */ entryContract: () => ContractFacts; sourceTexts: () => Map; - lower: (opts: LowerOptions) => LowerResult; + /** `valid`: the caller's IR validation; a module it rejects with the + * shape overlay on is lowered again without the overlay. */ + lower: (opts: LowerOptions, valid?: (module: IrModule) => boolean) => LowerResult; /** Each requested or detected package's native compilation outcome, * including the first refusal reason for ineligible packages. */ npmStatic: NpmStaticStatus[]; @@ -535,6 +540,32 @@ function loadFrontend( // and library builds with a package-specific diagnostic. let load = reusableScout ?? loadProgram(entryPath, { npmStatic: requested, externalTypes }); let preflight = reusablePreflight ?? checkPreflight(load); + // Shape inference never makes the preflight fail: when the overlay's + // program reports diagnostics the program without it does not (its type + // references can load package files the program would not), the build + // continues without the overlay. + if (preflight.length > 0 && load.shapeOverlayState !== undefined) { + restoreShapeOverlayState(load.shapeOverlayState); + if (shapeOverlayActive()) { + const key = (d: ScrDiagnostic): string => `${d.code}|${d.loc.file}|${d.loc.start}`; + setShapeOverlayDropped(true); + const plain = loadProgram(entryPath, { npmStatic: requested, externalTypes }); + const plainPreflight = checkPreflight(plain); + const known = new Set(plainPreflight.map(key)); + if (preflight.some((d) => !known.has(key(d)))) { + load.dispose(); + load = plain; + preflight = plainPreflight; + if (process.env["SCRIPTC_DEBUG_SHAPE_FALLBACK"] !== undefined) + process.stderr.write( + `[shape-fallback] preflight: the overlay's program failed; overlay off\n`, + ); + } else { + plain.dispose(); + setShapeOverlayDropped(false); + } + } + } // Library mode's fixpoint: the opted-in packages' files joined the // program just now, and THEIR bare edges (import statements and // top-level requires) name packages the scout could not see. Judge each @@ -699,18 +730,21 @@ function loadFrontend( ...finalLoad.program.getImplementationSourceFiles().filter((sf) => !sf.isDeclarationFile), ].map((sf) => [sf.fileName, sf.text]), ), - lower: (opts) => { - const lowerOnce = (): LowerResult => - lowerToIr(finalLoad.program, finalLoad.entry, finalLoad.moduleOrder, { + lower: (opts, valid) => { + const lowerLoad = (l: typeof finalLoad): LowerResult => { + if (l.shapeOverlayState !== undefined) restoreShapeOverlayState(l.shapeOverlayState); + return lowerToIr(l.program, l.entry, l.moduleOrder, { ...opts, - frontendServices: finalLoad.services, - startupCrash: finalLoad.startupCrash ?? null, - inlineRequireCycleCalls: finalLoad.inlineRequireCycleCalls, - moduleEdges: finalLoad.moduleEdges, - projectCheckJs: finalLoad.typeWorld?.projectCheckJs ?? false, - externalTypes: finalLoad.externalTypes, - externalTypeSpecifiersByFile: finalLoad.externalTypeSpecifiersByFile, + frontendServices: l.services, + startupCrash: l.startupCrash ?? null, + inlineRequireCycleCalls: l.inlineRequireCycleCalls, + moduleEdges: l.moduleEdges, + projectCheckJs: l.typeWorld?.projectCheckJs ?? false, + externalTypes: l.externalTypes, + externalTypeSpecifiersByFile: l.externalTypeSpecifiersByFile, }); + }; + const lowerOnce = (): LowerResult => lowerLoad(finalLoad); let lowered = lowerOnce(); // Shape inference never breaks compilation (shape-inference/ // fallback.ts): refusals drop the overlay annotations they depend on @@ -720,9 +754,19 @@ function loadFrontend( // annotations they are blamed on: a fence the overlay did not cause // stays as it is without the overlay. let level: "annotations" | "files" | "all" | "fences" = "annotations"; + // Deferred refusals the coverage pass found in code the lowering + // judged unreached count too: that code can still run. + const fencesOf = (l: LowerResult): ScrDiagnostic[] => [ + ...l.runtimeFences, + ...(l.unreached?.runtimeFences ?? []), + ]; + if (process.env["SCRIPTC_DEBUG_SHAPE_FALLBACK"] !== undefined) + process.stderr.write( + `[shape-fallback] lowered: overlay ${shapeOverlayActive()}, ${lowered.diagnostics.length} refusals, ${fencesOf(lowered).length} deferred\n`, + ); for (let round = 0; shapeOverlayActive(); round++) { const failed = lowered.diagnostics.length > 0; - const refusals = failed ? lowered.diagnostics : lowered.runtimeFences; + const refusals = failed ? lowered.diagnostics : fencesOf(lowered); if (refusals.length === 0) break; let changed: boolean; if (!failed) { @@ -741,15 +785,18 @@ function loadFrontend( } // The same round also drops what the deferred refusals blame, // saving a reload. - if (level !== "all" && lowered.runtimeFences.length > 0) + if (level !== "all" && fencesOf(lowered).length > 0) changed = - excludeRefusedOverlay(finalLoad.program, lowered.runtimeFences, "fences") || changed; + excludeRefusedOverlay(finalLoad.program, fencesOf(lowered), "fences") || changed; } if (process.env["SCRIPTC_DEBUG_SHAPE_FALLBACK"] !== undefined) process.stderr.write( `[shape-fallback] round ${round} level ${level}: ${refusals.length} refusals (${refusals .slice(0, process.env["SCRIPTC_DEBUG_SHAPE_FALLBACK"] === "all" ? undefined : 3) - .map((d) => `${d.loc.file.split("/").pop()}:${d.loc.start} ${d.message.slice(0, 60)}`) + .map( + (d) => + `${d.loc.file.split("/").pop()}:${d.loc.start} ${process.env["SCRIPTC_DEBUG_SHAPE_FALLBACK"] === "all" ? d.message : d.message.slice(0, 60)}`, + ) .join("; ")}), changed ${changed}\n`, ); if (!changed) break; @@ -769,6 +816,64 @@ function loadFrontend( `[shape-fallback] reload ${(t1 - t0).toFixed(0)} ms, preflight ${(t2 - t1).toFixed(0)} ms, lower ${(performance.now() - t2).toFixed(0)} ms\n`, ); } + // Deferred refusals the program without the overlay does not have + // mean the overlay changed what runs: the build compiles without it + // (the overlay must not add failures the default build lacks). + if (shapeOverlayActive() && lowered.diagnostics.length === 0) { + const fences = fencesOf(lowered); + if (fences.length > 0) { + const keyOf = (l: typeof finalLoad, d: ScrDiagnostic): string => { + const sf = l.program.getSourceFile(d.loc.file); + const line = sf === undefined ? -1 : sf.getLineAndCharacterOfPosition(d.loc.start).line; + return `${d.code}|${d.loc.file}|${line}`; + }; + const overlayState = finalLoad.shapeOverlayState; + setShapeOverlayDropped(true); + const plainLoad = loadProgram(entryPath, { npmStatic: finalNpmStatic, externalTypes }); + const plainPreflight = checkPreflight(plainLoad); + const plain = plainPreflight.length === 0 ? lowerLoad(plainLoad) : null; + const known = new Set( + plain === null ? [] : fencesOf(plain).map((d) => keyOf(plainLoad, d)), + ); + const added = fences.filter((d) => !known.has(keyOf(finalLoad, d))); + if (plain !== null && plain.diagnostics.length === 0 && added.length > 0) { + if (process.env["SCRIPTC_DEBUG_SHAPE_FALLBACK"] !== undefined) + process.stderr.write( + `[shape-fallback] ${added.length} deferred refusals the default build lacks (${added + .slice(0, 3) + .map((d) => `${d.loc.file.split("/").pop()}: ${d.message.slice(0, 60)}`) + .join("; ")}); overlay off\n`, + ); + finalLoad.dispose(); + finalLoad = plainLoad; + lowered = plain; + } else { + plainLoad.dispose(); + setShapeOverlayDropped(false); + if (overlayState !== undefined) restoreShapeOverlayState(overlayState); + } + } + } + // A module the caller's IR validator rejects (a lowering bug the + // overlay's types reach) compiles without the overlay. + if ( + shapeOverlayActive() && + lowered.module !== null && + valid !== undefined && + !valid(lowered.module) + ) { + if (process.env["SCRIPTC_DEBUG_SHAPE_FALLBACK"] !== undefined) + process.stderr.write( + `[shape-fallback] the IR validator rejected the module; overlay off\n`, + ); + setShapeOverlayDropped(true); + finalLoad.dispose(); + finalLoad = loadProgram(entryPath, { npmStatic: finalNpmStatic, externalTypes }); + const reloadPreflight = checkPreflight(finalLoad); + if (reloadPreflight.length > 0) + return { ...lowered, module: null, diagnostics: reloadPreflight }; + lowered = lowerOnce(); + } // The declaration-typed boundary's notes (SC6102) join the // performance warnings; they never fail a build. const notes = [ diff --git a/packages/compiler/src/frontend/program.ts b/packages/compiler/src/frontend/program.ts index 2dfc46bdc..b54976013 100644 --- a/packages/compiler/src/frontend/program.ts +++ b/packages/compiler/src/frontend/program.ts @@ -146,6 +146,8 @@ import { shapeOverlayDone, shapeOverlayEnabled, shapeOverlayer, + captureShapeOverlayState, + type ShapeOverlayLoadState, } from "./shape-inference/index.js"; import { inferredJsDiagnosticSuppressed } from "./inferred-js-diagnostics.js"; import { @@ -364,6 +366,9 @@ export interface TypeWorldDifferences { export interface LoadResult { program: ts.Program; entry: ts.SourceFile; + /** Shape inference's overlay state for this load (restored before its + * lowering: a later load resets the per-process registry). */ + shapeOverlayState?: ShapeOverlayLoadState; /** User source files in Node's evaluation order: depth-first postorder * of the import graph from the entry, each module once. The entry is * last. Empty until checkPreflight runs (and empty when preflight fails @@ -1047,6 +1052,7 @@ function loadProgram7( return { program, entry, + shapeOverlayState: captureShapeOverlayState(), moduleOrder: [], configDiags: config.diags, ...(declBoundaryNotes.length > 0 ? { declBoundaryNotes } : {}), diff --git a/packages/compiler/src/frontend/shape-inference/analyze.ts b/packages/compiler/src/frontend/shape-inference/analyze.ts index 2dd53079c..16d982d9e 100644 --- a/packages/compiler/src/frontend/shape-inference/analyze.ts +++ b/packages/compiler/src/frontend/shape-inference/analyze.ts @@ -196,6 +196,25 @@ const ARRAY_METHODS: ReadonlySet = new Set([ ]); const MAX_STEPS = 20_000_000; +/** The well-known symbols, static members of `Symbol` (Node 24). */ +const WELL_KNOWN_SYMBOLS: ReadonlySet = new Set([ + "asyncDispose", + "asyncIterator", + "dispose", + "hasInstance", + "isConcatSpreadable", + "iterator", + "match", + "matchAll", + "replace", + "search", + "species", + "split", + "toPrimitive", + "toStringTag", + "unscopables", +]); + /** The members String.prototype, Number.prototype and Boolean.prototype * define (Node 24); Object.prototype's come from OBJECT_PROTOTYPE_MEMBERS. */ const PRIMITIVE_MEMBERS: Readonly>> = { @@ -609,6 +628,26 @@ class Analyzer { * and which function each `new F()` instance atom belongs to. */ readonly protoOf = new Map(); readonly fnOfInstance = new Map(); + /** Per dictionary atom, the prototypes it was created with (Object.create). */ + private readonly dictProtos = new Map(); + /** Per object literal atom, the keys it has from its creation: another + * key may be absent when read (flow-insensitively, before its write). */ + private readonly literalOwnKeys = new Map>(); + private readonly protoReads = new Set(); + /** Arrays whose elements an index read inside its bounds may still miss: + * shrunk (pop, shift, splice, a length write), holey (`new Array(n)`, a + * delete, a store at an index that is not proven in bounds or the end). */ + private readonly unsafeArrays = new Set(); + /** Index reads proven in bounds by their loop (`for (let i = 0; i < + * a.length; i++) a[i]`): undefined joins them only if the array turns + * out unsafe (finish). */ + private readonly guardedIndexReads: { recv: Atom; dst: number; done: boolean }[] = []; + private readonly boundsMemo = new Map(); + dictProto(d: Atom): number { + let id = this.dictProtos.get(d); + if (id === undefined) this.dictProtos.set(d, (id = this.newNode())); + return id; + } protoAtom(fn: Atom): Atom { let p = this.protoOf.get(fn); if (p === undefined) { @@ -1355,6 +1394,15 @@ class Analyzer { this.edgeFor(site, info, args); } + /** The prototype object of a class declaration (`C.prototype`). */ + classProto(decl: Node): Atom { + return this.atoms.intern(`cproto:${this.nodeKey(decl)}`, () => ({ + kind: "object", + label: `${this.className(decl)}.prototype`, + node: decl, + })); + } + /** Per-call-site clones of init factories: a function called from * module initialization (top-level code outside loops, or another such * clone) is analyzed once per call site, so the objects it creates are @@ -1574,6 +1622,13 @@ class Analyzer { varDecl(d: ts.VariableDeclaration): void { const b = this.binding(d); if (d.initializer !== undefined) this.edge(this.expr(d.initializer), b); + else { + // `let x;` holds undefined until it is assigned (a for-in/for-of + // binding gets its values from the loop). + const loop = d.parent?.parent; + if (loop === undefined || !(ts.isForOfStatement(loop) || ts.isForInStatement(loop))) + this.add(b, this.prim("undefined")); + } this.bindPattern(d.name, b); } @@ -1632,7 +1687,15 @@ class Analyzer { const key = this.ctxKey(e); const memo = this.exprMemo.get(key); if (memo !== undefined) return memo; - const id = this.exprUncached(e); + let id = this.exprUncached(e); + // A member of an optional chain (`a?.b.c`, `f?.()`) is undefined when + // the chain short-circuits. + if ((e.flags & ts.NodeFlags.OptionalChain) !== 0) { + const out = this.newNode(); + this.edge(id, out); + this.add(out, this.prim("undefined")); + id = out; + } this.exprMemo.set(key, id); if (this.traceTop && !this.nodeLabel.has(id)) this.nodeLabel.set( @@ -1756,15 +1819,19 @@ class Analyzer { } if (ts.isPrefixUnaryExpression(e) || ts.isPostfixUnaryExpression(e)) { const operand = e.operand; - this.expr(operand); + const value = this.expr(operand); + if (ts.isPrefixUnaryExpression(e) && e.operator === ts.SyntaxKind.ExclamationToken) + return this.constNode(this.prim("boolean")); + // `+x` is a number (a bigint throws); `-x`, `~x`, `++`/`--` keep a bigint. + if (ts.isPrefixUnaryExpression(e) && e.operator === ts.SyntaxKind.PlusToken) + return this.constNode(this.prim("number")); + const result = this.numericResult([value]); if ( e.operator === ts.SyntaxKind.PlusPlusToken || e.operator === ts.SyntaxKind.MinusMinusToken ) - this.assignTo(operand, this.constNode(this.prim("number"))); - if (ts.isPrefixUnaryExpression(e) && e.operator === ts.SyntaxKind.ExclamationToken) - return this.constNode(this.prim("boolean")); - return this.constNode(this.prim("number")); + this.assignTo(operand, result); + return result; } if (ts.isSpreadElement(e)) return this.expr(e.expression); if (ts.isTaggedTemplateExpression(e)) return this.taggedTemplate(e); @@ -1792,6 +1859,8 @@ class Analyzer { if (!forwarding) fn.usesArguments = true; const arr = this.syntheticArray(id, "arguments"); for (const p of fn.params) this.edge(p, this.element(arr)); + // Arguments past the declared parameters are not tracked. + this.add(this.element(arr), this.top); return this.constNode(arr); } } @@ -2085,6 +2154,7 @@ class Analyzer { if (key !== null) this.edge(value, this.field(atom, key)); else this.edge(value, this.anyField(atom)); } + if (!this.literalOwnKeys.has(atom)) this.literalOwnKeys.set(atom, new Set(keys)); if (!this.siteAtoms.has(atom) && this.siteAtoms.add(atom)) this.sites.push({ atom, @@ -2295,16 +2365,11 @@ class Analyzer { return n; } if (op >= ts.SyntaxKind.FirstCompoundAssignment && op <= ts.SyntaxKind.LastCompoundAssignment) { - this.expr(e.left); - this.expr(e.right); - const result = this.constNode( - op === ts.SyntaxKind.PlusEqualsToken ? this.top : this.prim("number"), - ); - if (op === ts.SyntaxKind.PlusEqualsToken) { - this.nodes[result]!.atoms.clear(); - this.add(result, this.prim("string")); - this.add(result, this.prim("number")); - } + const lv = this.expr(e.left); + const rv = this.expr(e.right); + // Arithmetic on bigints answers bigints; `+=` may also concatenate. + const result = this.numericResult([lv, rv]); + if (op === ts.SyntaxKind.PlusEqualsToken) this.add(result, this.prim("string")); this.assignTo(e.left, result); return result; } @@ -2351,13 +2416,15 @@ class Analyzer { ) this.add(n, this.prim("number")); if (a.kind === "top") this.add(n, this.prim("number")); + if (a.kind === "top" || (a.kind === "prim" && a.label === "bigint")) + this.add(n, this.prim("bigint")); }; this.on(l, strOrNum); this.on(r, strOrNum); return n; } default: - return this.constNode(this.prim("number")); + return this.numericResult([l, r]); } } @@ -2455,6 +2522,8 @@ class Analyzer { this.unknownKeysInto(a, dst); return; } + // Members stored on a class's prototype object after the fact. + for (const c of this.classChain(decl)) this.edge(this.field(this.classProto(c), name), dst); const member = this.lookupMember(decl, name, false); if (member?.kind === "method") this.add( @@ -2478,7 +2547,9 @@ class Analyzer { case "class": { const decl = a.node!; if (name === "prototype" || name === "name") { - this.add(dst, name === "prototype" ? this.top : this.prim("string")); + // A class's prototype object: members stored on it later + // (`C.prototype.isVector3 = true`) are members of its instances. + this.add(dst, name === "prototype" ? this.classProto(decl) : this.prim("string")); this.fieldInto(a, name, dst); this.unknownKeysInto(a, dst); return; @@ -2524,8 +2595,10 @@ class Analyzer { return; case "array": if (name === "length") this.add(dst, this.prim("number")); - else if (/^\d+$/.test(name)) this.edge(this.element(a), dst); - else { + else if (/^\d+$/.test(name)) { + this.edge(this.element(a), dst); + this.add(dst, this.prim("undefined")); + } else { // A named property, written by name or under an unknown key; an own field shadows a method. if (ARRAY_METHODS.has(name)) { this.add(dst, this.builtin(`array.${name}`, a)); @@ -2535,8 +2608,19 @@ class Analyzer { } return; case "object": { + // A read through a class's prototype object (`C.prototype.m`) may + // be any member: unknown, as before prototype objects were tracked. + if ( + a.label.endsWith(".prototype") && + a.node !== undefined && + ts.isClassLikeDeclaration(a.node) + ) + this.add(dst, this.top); this.edge(this.field(a, name), dst); this.edge(this.anyField(a), dst); + const own = this.literalOwnKeys.get(a); + if (own !== undefined && !own.has(name) && !OBJECT_PROTOTYPE_MEMBERS.has(name)) + this.add(dst, this.prim("undefined")); if (this.namespaceAtoms.has(a) && name !== "exports") { this.loadConstraint(this.field(a, "exports"), name, dst, site); } @@ -2556,10 +2640,23 @@ class Analyzer { this.add(dst, this.builtin(`object.${name}`, a)); return; } - case "dictionary": + case "dictionary": { this.edge(this.anyField(a), dst); this.edge(this.field(a, name), dst); + // An absent key reads undefined, or what the prototype has. + this.add(dst, this.prim("undefined")); + const proto = this.dictProtos.get(a); + // Once per dictionary, name and destination: prototype chains can + // be cyclic in the analysis (`Object.create(this)`). + const key = `${a.id}:${name}:${dst}`; + if (proto !== undefined && !this.protoReads.has(key)) { + this.protoReads.add(key); + this.on(proto, (p) => { + if (p.kind !== "prim") this.loadAtom(p, name, dst, site); + }); + } return; + } case "promise": if (name === "then" || name === "catch" || name === "finally") this.add(dst, this.builtin(`promise.${name}`, a)); @@ -2595,8 +2692,13 @@ class Analyzer { ) this.add(dst, this.prim("number")); else if (path === "Math" && /^[A-Z0-9_]+$/.test(name)) this.add(dst, this.prim("number")); - else if (path === "Symbol") this.add(dst, this.prim("symbol")); - else if (path === "Error.instance" || path === "error") { + else if (path === "Symbol") { + // The well-known symbols are symbols; `Symbol.for` and `Symbol.keyFor` + // are functions (a call of a symbol would answer nothing). + if (WELL_KNOWN_SYMBOLS.has(name)) this.add(dst, this.prim("symbol")); + else if (name === "for" || name === "keyFor") this.add(dst, this.builtin(`Symbol.${name}`)); + else this.add(dst, this.top); + } else if (path === "Error.instance" || path === "error") { if (name === "message" || name === "stack" || name === "name") this.add(dst, this.prim("string")); else this.add(dst, this.top); @@ -2637,6 +2739,7 @@ class Analyzer { this.edge(value, this.field(a, name)); return; case "array": + if (/^\d+$/.test(name) || name === "length") this.unsafeArrays.add(a); if (/^\d+$/.test(name)) this.edge(value, this.element(a)); else if (name !== "length") this.edge(value, this.field(a, name)); return; @@ -2655,11 +2758,20 @@ class Analyzer { this.add(dst, this.top); return; case "array": + // An index past the end (or a hole) reads undefined, unless the + // loop proves the index in bounds of an array without holes. + this.edge(this.element(a), dst); + if (this.indexInBounds(site)) this.guardedIndexReads.push({ recv: a, dst, done: false }); + else this.add(dst, this.prim("undefined")); + return; case "collection": this.edge(this.element(a), dst); return; case "prim": - if (a.label === "string") this.add(dst, this.prim("string")); + if (a.label === "string") { + this.add(dst, this.prim("string")); + if (!this.indexInBounds(site)) this.add(dst, this.prim("undefined")); + } return; case "object": case "dictionary": @@ -2678,6 +2790,8 @@ class Analyzer { copy(); this.lateCopies.push(copy); if (a.kind === "instance" || a.kind === "class") this.add(dst, this.top); + // A key the object does not have reads undefined. + else this.add(dst, this.prim("undefined")); return; } default: @@ -2698,6 +2812,7 @@ class Analyzer { // An index, or a named property under an unknown key. this.edge(value, this.element(a)); this.edge(value, this.anyField(a)); + if (!this.indexInBounds(site) && !isAppendIndex(site)) this.unsafeArrays.add(a); return; case "prim": return; @@ -2937,6 +3052,8 @@ class Analyzer { case "builtin": this.constructBuiltin(a.path ?? "", args, dst, e); return; + case "prim": + return; // `new` of a primitive throws default: for (const x of args) this.sink(x, `argument of ⊤ constructor at ${locLabel(e)}`); this.add(dst, this.top); @@ -2993,7 +3110,10 @@ class Analyzer { this.add(dst, this.builtin("RegExp.instance")); return; case "Array": { - this.add(dst, this.syntheticArray(site, "Array")); + const arr = this.syntheticArray(site, "Array"); + // `new Array(n)` has holes. + this.unsafeArrays.add(arr); + this.add(dst, arr); return; } case "Proxy": @@ -3059,6 +3179,7 @@ class Analyzer { case "pop": case "shift": case "at": + if (method !== "at") this.unsafeArrays.add(recv); this.edge(el, dst); this.add(dst, this.prim("undefined")); return; @@ -3111,6 +3232,7 @@ class Analyzer { case "toSorted": case "toReversed": case "flat": { + if (method === "splice") this.unsafeArrays.add(recv); if (method === "filter") callback(0, [el, num, recvN], null); if (method === "sort" || method === "toSorted") callback(0, [el, el], null); if (method === "reverse" || method === "sort") { @@ -3377,6 +3499,9 @@ class Analyzer { label: `Object.create@${locLabel(site)}`, node: site, })); + // Its members are inherited from the prototype argument. + this.edge(arg(0), this.dictProto(d)); + if (args[1] !== undefined) this.sink(args[1], `property descriptors at ${locLabel(site)}`); this.add(dst, d); return; } @@ -3570,12 +3695,185 @@ class Analyzer { escapeAll(); } + /** Whether an index access `x[i]` reads inside the bounds of `x` on + * every evaluation: an enclosing `for (let i = ; i < x.length; + * i++ | ++i | i += 0>)` whose body assigns neither `i` nor `x`. + * (The array's own length may still shrink or have holes: unsafeArrays.) */ + indexInBounds(site: Node): boolean { + if (!ts.isElementAccessExpression(site) || site.questionDotToken !== undefined) return false; + const memo = this.boundsMemo.get(site); + if (memo !== undefined) return memo; + const out = this.indexInBoundsUncached(site); + this.boundsMemo.set(site, out); + return out; + } + + private indexInBoundsUncached(site: ts.ElementAccessExpression): boolean { + const obj = unparen(site.expression); + const key = unparen(site.argumentExpression); + if (!ts.isIdentifier(obj) || !ts.isIdentifier(key)) return false; + const objDecl = this.resolve(obj); + const keyDecl = this.resolve(key); + if (objDecl === undefined || keyDecl === undefined) return false; + const names = (id: Node, decl: Node): boolean => + ts.isIdentifier(id) && this.resolve(id) === decl; + for (let n: Node | undefined = site.parent; n !== undefined; n = n.parent) { + if (ts.isFunctionLike(n) || ts.isSourceFile(n)) return false; + if (!ts.isForStatement(n) || n.statement === undefined) continue; + if (!(site.pos >= n.statement.pos && site.end <= n.statement.end)) continue; + const cond = n.condition !== undefined ? unparen(n.condition) : undefined; + if ( + cond === undefined || + !ts.isBinaryExpression(cond) || + cond.operatorToken.kind !== ts.SyntaxKind.LessThanToken || + !names(unparen(cond.left), keyDecl) + ) + continue; + const bound = unparen(cond.right); + if ( + !ts.isPropertyAccessExpression(bound) || + bound.name.text !== "length" || + !names(unparen(bound.expression), objDecl) + ) + continue; + // The index starts at a non-negative integer literal ... + const init = n.initializer; + let start: ts.Expression | undefined; + // (declared by the loop, so only the loop's own code can write it) + if ( + init !== undefined && + ts.isVariableDeclarationList(init) && + (init.flags & (ts.NodeFlags.Let | ts.NodeFlags.Const)) !== 0 + ) { + const d = init.declarations.nodes.find((x) => x === keyDecl); + start = d !== undefined ? (d as ts.VariableDeclaration).initializer : undefined; + } + if (start === undefined || !nonNegativeIntegerLiteral(start)) continue; + if (!this.neverReassigned(objDecl)) continue; + // ... and only grows by integers, + const inc = n.incrementor !== undefined ? unparen(n.incrementor) : undefined; + const grows = + inc !== undefined && + (((ts.isPostfixUnaryExpression(inc) || ts.isPrefixUnaryExpression(inc)) && + inc.operator === ts.SyntaxKind.PlusPlusToken && + names(unparen(inc.operand), keyDecl)) || + (ts.isBinaryExpression(inc) && + inc.operatorToken.kind === ts.SyntaxKind.PlusEqualsToken && + names(unparen(inc.left), keyDecl) && + positiveIntegerLiteral(inc.right))); + if (!grows) continue; + // ... and the body writes neither the index nor the array binding. + let written = false; + const visit = (m: Node): void => { + if (written) return; + if ( + ts.isBinaryExpression(m) && + m.operatorToken.kind >= ts.SyntaxKind.FirstAssignment && + m.operatorToken.kind <= ts.SyntaxKind.LastAssignment && + (names(unparen(m.left), keyDecl) || names(unparen(m.left), objDecl)) + ) + written = true; + else if ( + (ts.isPostfixUnaryExpression(m) || ts.isPrefixUnaryExpression(m)) && + (m.operator === ts.SyntaxKind.PlusPlusToken || + m.operator === ts.SyntaxKind.MinusMinusToken) && + (names(unparen(m.operand), keyDecl) || names(unparen(m.operand), objDecl)) + ) + written = true; + ts.forEachChild(m, (c) => { + visit(c); + return undefined; + }); + }; + visit(n.statement); + if (!written) return true; + } + return false; + } + + private readonly reassignMemo = new Map(); + /** Whether a binding keeps the value it is declared with: a `const`, or a + * parameter or variable that no code in its scope assigns. */ + neverReassigned(decl: Node): boolean { + const memo = this.reassignMemo.get(decl); + if (memo !== undefined) return memo; + let out = false; + if (ts.isVariableDeclaration(decl) || ts.isParameter(decl)) { + const list = decl.parent; + if ( + ts.isVariableDeclaration(decl) && + list !== undefined && + ts.isVariableDeclarationList(list) && + (list.flags & ts.NodeFlags.Const) !== 0 + ) + out = true; + else { + let scope: Node | undefined = decl.parent; + while (scope !== undefined && !ts.isFunctionLike(scope) && !ts.isSourceFile(scope)) + scope = scope.parent; + if (scope !== undefined && ts.isIdentifier((decl as ts.VariableDeclaration).name)) { + const text = ((decl as ts.VariableDeclaration).name as ts.Identifier).text; + let written = false; + const target = (e: ts.Expression): boolean => { + const u = unparen(e); + return ts.isIdentifier(u) && u.text === text && this.resolve(u) === decl; + }; + const visit = (m: Node): void => { + if (written) return; + if ( + (ts.isBinaryExpression(m) && + m.operatorToken.kind >= ts.SyntaxKind.FirstAssignment && + m.operatorToken.kind <= ts.SyntaxKind.LastAssignment && + target(m.left)) || + ((ts.isPostfixUnaryExpression(m) || ts.isPrefixUnaryExpression(m)) && + (m.operator === ts.SyntaxKind.PlusPlusToken || + m.operator === ts.SyntaxKind.MinusMinusToken) && + target(m.operand)) + ) + written = true; + ts.forEachChild(m, (c) => { + visit(c); + return undefined; + }); + }; + visit(scope); + out = !written; + } + } + } + this.reassignMemo.set(decl, out); + return out; + } + + /** The result of arithmetic over the operands: a number, or a bigint + * where an operand may be one. */ + numericResult(operands: number[]): number { + const n = this.newNode(); + this.add(n, this.prim("number")); + for (const o of operands) + this.on(o, (a) => { + if (a.kind === "top" || (a.kind === "prim" && a.label === "bigint")) + this.add(n, this.prim("bigint")); + }); + return n; + } + /** Re-run deferred field copies until nothing new flows. */ finish(): void { for (;;) { const before = this.steps; for (const c of this.lateCopies) c(); this.solve(); + for (const o of this.ops) + if (o.kind === "delete" || o.kind === "lengthWrite") + for (const a of this.nodes[o.recv]!.atoms) + if (a.kind === "array") this.unsafeArrays.add(a); + for (const g of this.guardedIndexReads) + if (!g.done && this.unsafeArrays.has(g.recv)) { + g.done = true; + this.add(g.dst, this.prim("undefined")); + } + this.solve(); // A condition with no values at the fixpoint is one the analysis // could not evaluate: walk both of its branches. let walked = false; @@ -3596,6 +3894,30 @@ class Analyzer { } } +function nonNegativeIntegerLiteral(e: ts.Expression): boolean { + const u = unparen(e); + return ts.isNumericLiteral(u) && Number.isInteger(Number(u.text)) && Number(u.text) >= 0; +} + +function positiveIntegerLiteral(e: ts.Expression): boolean { + const u = unparen(e); + return ts.isNumericLiteral(u) && Number.isInteger(Number(u.text)) && Number(u.text) > 0; +} + +/** `x[x.length] = v`: a store at the end, which appends without a hole. */ +function isAppendIndex(site: Node): boolean { + if (!ts.isElementAccessExpression(site)) return false; + const obj = unparen(site.expression); + const key = unparen(site.argumentExpression); + return ( + ts.isIdentifier(obj) && + ts.isPropertyAccessExpression(key) && + key.name.text === "length" && + ts.isIdentifier(unparen(key.expression)) && + (unparen(key.expression) as ts.Identifier).text === obj.text + ); +} + /** The export name of a CommonJS export declaration the checker resolves * bindings to: a member of the `module.exports` literal, or the target of * `exports.name =` / `module.exports.name =`. */ diff --git a/packages/compiler/src/frontend/shape-inference/constant-graphs.ts b/packages/compiler/src/frontend/shape-inference/constant-graphs.ts index ceea474bd..a13ce850a 100644 --- a/packages/compiler/src/frontend/shape-inference/constant-graphs.ts +++ b/packages/compiler/src/frontend/shape-inference/constant-graphs.ts @@ -385,10 +385,14 @@ export function computeConstantGraphs(analysis: ShapeAnalysis): ConstantGraphs { mixedReads = 0; for (const op of analysis.ops) { if (op.kind !== "read" || op.name === null) continue; - const atoms = analysis.nodes[op.recv]!.atoms; - const constant = [...atoms].filter((a) => constantKeys.get(a)?.has(op.name!) === true); + // A nullish receiver throws (absent keys read undefined in the analysis). + const optional = (op.node.flags & ts.NodeFlags.OptionalChain) !== 0; + const atoms = [...analysis.nodes[op.recv]!.atoms].filter( + (a) => optional || !(a.kind === "prim" && (a.label === "undefined" || a.label === "null")), + ); + const constant = atoms.filter((a) => constantKeys.get(a)?.has(op.name!) === true); if (constant.length === 0) continue; - if (atoms.size === 1) { + if (atoms.length === 1) { foldableReads++; const p = op.node.parent; if (p !== undefined && ts.isCallExpression(p) && p.expression === op.node) foldableCalls++; diff --git a/packages/compiler/src/frontend/shape-inference/fallback.ts b/packages/compiler/src/frontend/shape-inference/fallback.ts index 50857fcfb..81759dad6 100644 --- a/packages/compiler/src/frontend/shape-inference/fallback.ts +++ b/packages/compiler/src/frontend/shape-inference/fallback.ts @@ -235,7 +235,7 @@ function visitReturns(body: ts.Node, f: (e: ts.Expression) => void): void { } /** The innermost node whose token span covers [start, end). */ -function innermostNodeAt(sf: ts.SourceFile, start: number, end: number): ts.Node { +export function innermostNodeAt(sf: ts.SourceFile, start: number, end: number): ts.Node { let node: ts.Node = sf; for (;;) { let next: ts.Node | undefined; diff --git a/packages/compiler/src/frontend/shape-inference/index.ts b/packages/compiler/src/frontend/shape-inference/index.ts index 38dce5143..43e8357cc 100644 --- a/packages/compiler/src/frontend/shape-inference/index.ts +++ b/packages/compiler/src/frontend/shape-inference/index.ts @@ -16,6 +16,7 @@ import { type ShapeOverlayPlan, } from "./overlay.js"; import { attributeOverlayRefusals } from "./fallback.js"; +import { checksInferredFact } from "./inferred-checks.js"; import { planStaticObjectRewrites, type StaticObjectRewrites } from "./static-objects.js"; import { isJsSourceFile } from "../program.js"; import { buildShapeReport } from "./report.js"; @@ -65,7 +66,43 @@ export function shapeOverlayAnnotationCount(): number { return overlayAnnotations; } +/** The overlay's per-load state (the registry is per process, and a later + * load resets it): a load captures it, and the lowering of that load + * restores it. */ +export interface ShapeOverlayLoadState { + readonly renderedFiles: ReadonlySet; + readonly staticRewriteFiles: ReadonlySet; + readonly staticObjectStats: typeof staticObjectStats; + readonly overlayDone: boolean; + readonly overlayFiles: ReadonlySet; + readonly overlayAnnotations: number; + readonly overlayDeadBranches: number; +} + +export function captureShapeOverlayState(): ShapeOverlayLoadState { + return { + renderedFiles, + staticRewriteFiles, + staticObjectStats, + overlayDone, + overlayFiles, + overlayAnnotations, + overlayDeadBranches, + }; +} + +export function restoreShapeOverlayState(state: ShapeOverlayLoadState): void { + renderedFiles = state.renderedFiles; + staticRewriteFiles = state.staticRewriteFiles; + staticObjectStats = state.staticObjectStats; + overlayDone = state.overlayDone; + overlayFiles = state.overlayFiles; + overlayAnnotations = state.overlayAnnotations; + overlayDeadBranches = state.overlayDeadBranches; +} + export function resetShapeOverlay(): void { + renderedFiles = new Set(); staticRewriteFiles = new Set(); staticObjectStats = { classes: 0, @@ -106,10 +143,23 @@ export function beginShapeOverlayBuild(): void { droppedAnnotations = 0; } +/** Turns the overlay off for the rest of the build (true), or back on + * (false): the pipeline compares a load without it when the overlay's + * program fails a check the program without it passes. */ +export function setShapeOverlayDropped(dropped: boolean): void { + overlayDropped = dropped; + droppedAnnotations = dropped ? overlayAnnotations : 0; +} + /** Whether the current load has overlay files or structural rewrites. */ export function shapeOverlayActive(): boolean { const { classes, functions } = staticObjectStats; - return overlayFiles.size > 0 || classes + functions > 0; + return ( + overlayFiles.size > 0 || + renderedFiles.size > 0 || + staticRewriteFiles.size > 0 || + classes + functions > 0 + ); } /** The overlay's answer to refused lowering diagnostics: it excludes what @@ -127,7 +177,7 @@ export function excludeRefusedOverlay( level: "annotations" | "files" | "all" | "fences", ): boolean { if (overlayDropped) return false; - const files = new Set(overlayFiles); + const files = new Set([...overlayFiles, ...renderedFiles]); // Files with structural rewrites only also count as overlay files. for (const f of staticRewriteFiles) files.add(f); if (level === "all") { @@ -166,6 +216,8 @@ export function excludeRefusedOverlay( } let staticRewriteFiles: ReadonlySet = new Set(); +/** Every file the typed round rewrote (typed or only neutralized). */ +let renderedFiles: ReadonlySet = new Set(); function overlayExclusions(): OverlayExclusions { return exclusions; @@ -270,7 +322,8 @@ export function shapeOverlayer( }); } const overlay = renderShapeOverlay(entry.plan, overlayExclusions()); - overlayFiles = new Set(overlay.files.keys()); + overlayFiles = new Set(overlay.typed); + renderedFiles = new Set(overlay.files.keys()); overlayAnnotations = overlay.annotations; overlayDeadBranches = overlay.deadBranches; if (process.env["SCRIPTC_DEBUG_SHAPE_OVERLAY"] !== undefined) { @@ -322,10 +375,14 @@ export function markInferredFactChecks( if (x === null || typeof x !== "object") return; const node = x as { kind?: unknown; - loc?: { file: string; start: number }; + loc?: { file: string; start: number; end: number }; inferredSite?: string; }; - if (node.kind === "dynCheck" && node.loc !== undefined && overlayFiles.has(node.loc.file)) { + if ( + node.kind === "dynCheck" && + node.loc !== undefined && + checksInferredFact(program, node.loc.file, node.loc.start, node.loc.end) + ) { const pkg = npmStaticPackageOfPath(node.loc.file) ?? ""; const rel = node.loc.file.split("/node_modules/").pop() ?? node.loc.file; node.inferredSite = `${pkg} ${rel}:${lineOf(node.loc.file, node.loc.start)}`; diff --git a/packages/compiler/src/frontend/shape-inference/inferred-checks.ts b/packages/compiler/src/frontend/shape-inference/inferred-checks.ts new file mode 100644 index 000000000..d9c841034 --- /dev/null +++ b/packages/compiler/src/frontend/shape-inference/inferred-checks.ts @@ -0,0 +1,88 @@ +/* Which checked conversions check an inferred fact: those whose target type + * is an overlay annotation's (a parameter, return, binding or field the + * overlay typed, or a receiver cast). Only those report a failure as a + * shape analysis bug; every other check (a type the package's JSDoc or a + * declaration file declares, or the checker infers) keeps the default + * build's TypeError. */ +import * as ts from "../ts7/adapter.js"; +import { innermostNodeAt } from "./fallback.js"; +import { OVERLAY_MARK } from "./overlay.js"; + +/** Whether the overlay annotated a declaration (or the statement holding + * it). */ +function annotated(node: ts.Node | undefined): boolean { + for (let n = node, i = 0; n !== undefined && i < 4; n = n.parent, i++) { + const sf = n.getSourceFile(); + if (sf.text.slice(n.pos, ts.tokenStartOfNode(n)).includes(OVERLAY_MARK)) return true; + if ( + ts.isStatement(n) || + ts.isSourceFile(n) || + (n.parent !== undefined && ts.isClassLikeDeclaration(n.parent)) + ) + break; + } + return false; +} + +export function checksInferredFact( + program: ts.Program, + file: string, + start: number, + end: number, +): boolean { + const sf = program.getSourceFile(file); + if (sf === undefined) return false; + let e: ts.Node = innermostNodeAt(sf, start, Math.max(start, end)); + // A receiver cast: `/** @type {…} @scriptcShape */ (this)`. + for (let n: ts.Node | undefined = e; n !== undefined; n = n.parent) { + if (!ts.isParenthesizedExpression(n)) { + if (n !== e) break; + continue; + } + if (sf.text.slice(n.pos, ts.tokenStartOfNode(n)).includes(OVERLAY_MARK)) return true; + } + while (e.parent !== undefined && ts.isParenthesizedExpression(e.parent)) e = e.parent; + const p = e.parent; + if (p === undefined) return false; + if (ts.isReturnStatement(p)) { + let fn: ts.Node | undefined = p.parent; + while (fn !== undefined && !ts.isFunctionLike(fn)) fn = fn.parent; + return annotated(fn); + } + if (ts.isArrowFunction(p) && p.body === e) return annotated(p); + if (ts.isVariableDeclaration(p) && p.initializer === e) return annotated(p); + const checker = program.getTypeChecker(); + const declsOf = (n: ts.Node): readonly ts.Node[] => { + try { + return checker.getSymbolAtLocation(n)?.declarations ?? []; + } catch { + return []; + } + }; + if ((ts.isCallExpression(p) || ts.isNewExpression(p)) && p.expression !== e) { + let decl: ts.Node | undefined; + try { + decl = checker.getResolvedSignature(p)?.declaration ?? undefined; + } catch { + decl = undefined; + } + if (decl !== undefined && annotated(decl)) return true; + const callee = ts.isPropertyAccessExpression(p.expression) ? p.expression.name : p.expression; + return declsOf(callee).some((d) => annotated(d) || annotated(constructorOf(d))); + } + if ( + ts.isBinaryExpression(p) && + p.operatorToken.kind === ts.SyntaxKind.EqualsToken && + p.right === e + ) { + const target = ts.isPropertyAccessExpression(p.left) ? p.left.name : p.left; + return declsOf(target).some((d) => annotated(d)); + } + return false; +} + +/** A class declaration's constructor (`new C(…)` resolves to the class). */ +function constructorOf(decl: ts.Node): ts.Node | undefined { + if (!ts.isClassDeclaration(decl) && !ts.isClassExpression(decl)) return undefined; + return decl.members.nodes.find((m) => ts.isConstructorDeclaration(m)); +} diff --git a/packages/compiler/src/frontend/shape-inference/overlay.ts b/packages/compiler/src/frontend/shape-inference/overlay.ts index 24fdcefd9..f59b2057f 100644 --- a/packages/compiler/src/frontend/shape-inference/overlay.ts +++ b/packages/compiler/src/frontend/shape-inference/overlay.ts @@ -11,8 +11,10 @@ * meets an inferred type therefore fails only when the analysis is wrong; * such checks carry the inferred-fact failure message. * - * The package's own JSDoc is a hint, not a declaration: its tags are - * neutralized in the rewritten file, so only the analysis's types remain. + * The package's own JSDoc is a declaration (maintainer decision + * 2026-10-25): it stays as written, and the overlay adds a type only where + * the package's JSDoc declares none (a documented parameter, return or + * binding keeps its documented type, as in the default build). * * Every edit is inline: no line breaks are added or removed, so positions * keep their lines. */ @@ -28,6 +30,8 @@ export interface ShapeOverlay { readonly annotations: number; /** Branches the analysis proved unreachable, replaced by checks. */ readonly deadBranches: number; + /** The files with annotations or dead-branch checks. */ + readonly typed: ReadonlySet; } interface Edit { @@ -51,11 +55,6 @@ interface OverlayPart { readonly text: string; readonly edits: Edit[]; readonly suffix: string; - /** The leading trivia of each annotated declaration, by annotation id: - * the package's own JSDoc there is neutralized while the annotation is - * kept (package JSDoc is a hint; elsewhere it types the code as it does - * without the overlay). */ - readonly docRanges: { lo: number; hi: number; id: number }[]; } /** Marks the overlay's own comments (an unknown JSDoc tag, ignored). */ @@ -98,13 +97,24 @@ export function planShapeOverlay( packageFiles: readonly ts.SourceFile[], exclusions: OverlayExclusions = NO_EXCLUSIONS, ): ShapeOverlayPlan { + // Debugging: SCRIPTC_DEBUG_SHAPE_FIELD=name prints that field's values per atom. + const debugField = process.env["SCRIPTC_DEBUG_SHAPE_FIELD"]; + if (debugField !== undefined) + for (const [atom, fields] of analysis.fields) { + const id = fields.get(debugField); + if (id !== undefined) + process.stderr.write( + `[shape-field] ${atom.kind}:${atom.label}.${debugField} = ${[...analysis.nodes[id]!.atoms].map((a) => `${a.kind}:${a.label}`).join(" | ")}\n`, + ); + } const pSites = new Map(); const siteVerdicts = new Set(verdicts.map((v) => v.site.atom)); + const dynHeld = dynHeldSites(analysis, verdicts); // Records and arrays print only for sites whose objects never escape: a // native record that reached untyped code would be seen as a copy. const siteKeys = new Map }>(); for (const v of verdicts) { - if (v.verdict !== "P-local") continue; + if (v.verdict !== "P-local" || dynHeld.has(v.site.atom)) continue; if (v.site.kind === "array") { pSites.set(v.site.atom, v); continue; @@ -140,9 +150,7 @@ export function planShapeOverlay( // One annotation: a JSDoc comment at `pos` carrying MARK and the // annotation's id (its position), so a refused lowering can name it and // the next load drop it. `close`: the text that closes a type cast. - const docRanges: { lo: number; hi: number; id: number }[] = []; - const annotate = (pos: number, tags: string, close?: number, from?: number): void => { - if (from !== undefined && from < pos) docRanges.push({ lo: from, hi: pos, id: pos }); + const annotate = (pos: number, tags: string, close?: number): void => { edits.push({ pos, id: pos, @@ -267,6 +275,9 @@ export function planShapeOverlay( // Arms, not atoms: per-site allocations of one record shape are one // typedef. if (arms.size > MAX_ARMS) return inner ? "any" : null; + // Arithmetic on unknown values may answer a number or a bigint: the + // lowering has no useful mixed representation, so those stay untyped. + if (arms.has("number") && arms.has("bigint")) return inner ? "any" : null; if (arms.has("any")) return inner ? "any" : null; return [...arms].join(" | "); }; @@ -458,7 +469,11 @@ export function planShapeOverlay( const paramTags = (fn: ts.FunctionLikeDeclaration): string | null => { const tags: string[] = []; - if (!ts.isConstructorDeclaration(fn)) { + // The package's JSDoc declares types: the overlay adds none where it + // has one, and none at all for a signature it types as a whole. + const doc = packageDocOf(fn); + if (doc.whole) return null; + if (!ts.isConstructorDeclaration(fn) && !doc.returns) { const ret = returnValues(fn); const t = ret !== undefined && ret.size > 0 ? typeOf(ret, false, 0) : null; const why = process.env["SCRIPTC_DEBUG_SHAPE_OVERLAY_WHY"]; @@ -489,6 +504,7 @@ export function planShapeOverlay( // narrow the typed parameter to never inside the branch, which the // lowering refuses: such a parameter stays untyped. if (t !== null && typeofTestsOutside(fn, p.name.text, values)) return; + if (doc.params.has(p.name.text)) return; if (t !== null) tags.push(`@param {${t}} ${p.name.text}`); }); return tags.length > 0 ? tags.join(" ") : null; @@ -509,7 +525,7 @@ export function planShapeOverlay( ) { if (node.body !== undefined) { const tags = paramTags(node); - if (tags !== null) annotate(ts.tokenStartOfNode(node), tags, undefined, node.pos); + if (tags !== null) annotate(ts.tokenStartOfNode(node), tags); } // `this.f = …` fields of a constructor's top-level statements. if (ts.isConstructorDeclaration(node) && node.body !== undefined) { @@ -532,7 +548,8 @@ export function planShapeOverlay( if (seen.has(name)) continue; seen.add(name); const t = cls !== undefined ? fieldType(cls, name) : null; - if (t !== null) annotate(ts.tokenStartOfNode(st), `@type {${t}}`, undefined, st.pos); + if (t !== null && !packageDocOf(st).type) + annotate(ts.tokenStartOfNode(st), `@type {${t}}`); } } } @@ -548,7 +565,12 @@ export function planShapeOverlay( const init = d.initializer; if (ts.isIdentifier(d.name) && init !== undefined && !literalInitializer(init)) { const t = typeOf(analysis.declValues(d), false, 0); - if (t !== null) annotate(ts.tokenStartOfNode(node), `@type {${t}}`, undefined, node.pos); + if (process.env["SCRIPTC_DEBUG_SHAPE_OVERLAY_WHY"] === d.name.text) + process.stderr.write( + `[shape-overlay-why] ${d.name.text}@${sf.getLineAndCharacterOfPosition(ts.tokenStartOfNode(d)).line + 1}: ${t} <- ${[...analysis.declValues(d)].map((a) => `${a.kind}:${a.label}`).join(" | ")}\n`, + ); + if (t !== null && !packageDocOf(node).type) + annotate(ts.tokenStartOfNode(node), `@type {${t}}`); } } ts.forEachChild(node, (c) => { @@ -614,8 +636,8 @@ export function planShapeOverlay( edits.length = 0; edits.push(...kept); } - if (edits.length === 0) continue; - parts.set(sf.fileName, { text: sf.text, edits, suffix: typedefText.join(""), docRanges }); + if (edits.length === 0 && typedefText.length === 0) continue; + parts.set(sf.fileName, { text: sf.text, edits, suffix: typedefText.join("") }); } // Type aliases other files name these files' classes through. for (const [other, names] of exportedClasses) { @@ -631,7 +653,6 @@ export function planShapeOverlay( text: other.text, edits: [], suffix: `\n${aliases}`, - docRanges: [], }); } return { parts }; @@ -643,6 +664,7 @@ export function renderShapeOverlay( exclusions: OverlayExclusions = NO_EXCLUSIONS, ): ShapeOverlay { const files = new Map(); + const typed = new Set(); let annotations = 0; let deadBranches = 0; for (const [file, part] of plan.parts) { @@ -654,28 +676,114 @@ export function renderShapeOverlay( else annotations++; } if (edits.length === 0 && part.suffix === "") continue; - files.set(file, applyEdits(neutralizedDocRanges(part, dropped), edits) + part.suffix); + const text = applyEdits(part.text, edits) + part.suffix; + files.set(file, text); + if (edits.some((e) => e.id !== undefined)) typed.add(file); } - return { files, annotations, deadBranches }; + return { files, typed, annotations, deadBranches }; } -/** The part's text with the package JSDoc neutralized in the leading trivia - * of the annotations a build kept. Each range is one declaration's leading - * trivia, so two ranges are equal or disjoint; the text is built once. */ -function neutralizedDocRanges(part: OverlayPart, dropped: ReadonlySet | undefined): string { - const ranges = part.docRanges - .filter((r) => dropped?.has(r.id) !== true) - .sort((a, b) => a.lo - b.lo || a.hi - b.hi); - if (ranges.length === 0) return part.text; - let out = ""; - let at = 0; - for (const r of ranges) { - const lo = Math.max(r.lo, at); - if (r.hi <= lo) continue; - out += part.text.slice(at, lo) + neutralizeDocTags(part.text.slice(lo, r.hi)); - at = r.hi; +/** Literal sites whose objects some holder keeps in the lowering's dynamic + * representation: a module-level binding of a JavaScript file (the overlay + * does not type those, and the lowering keeps their objects dynamic), and, + * transitively, the fields and elements of those objects and of every + * object or array that is not P-local. A typed position that receives such + * an object converts it to a native record, which is a copy: identity + * comparisons and writes through either reference would diverge. */ +function dynHeldSites(analysis: ShapeAnalysis, verdicts: readonly SiteVerdict[]): Set { + const held = new Set(); + const work: Atom[] = []; + const add = (a: Atom): void => { + if ((a.kind === "object" || a.kind === "array") && !held.has(a)) { + held.add(a); + work.push(a); + } + }; + for (const sf of analysis.files) { + if (!/\.[cm]?jsx?$/.test(sf.fileName)) continue; + for (const st of sf.statements.nodes) { + if (!ts.isVariableStatement(st)) continue; + for (const d of st.declarationList.declarations.nodes) + for (const a of analysis.declValues(d)) add(a); + } } - return out + part.text.slice(at); + const pLocal = new Set(); + for (const v of verdicts) if (v.verdict === "P-local") pLocal.add(v.site.atom); + for (const a of analysis.atoms.atoms) + if ((a.kind === "object" || a.kind === "array") && !pLocal.has(a)) work.push(a); + const seen = new Set(); + while (work.length > 0) { + const a = work.pop()!; + if (seen.has(a)) continue; + seen.add(a); + for (const id of analysis.fields.get(a)?.values() ?? []) + for (const x of analysis.nodes[id]!.atoms) add(x); + const el = analysis.elements.get(a); + if (el !== undefined) for (const x of analysis.nodes[el]!.atoms) add(x); + } + return held; +} + +interface PackageDoc { + /** Documented parameter names. */ + readonly params: ReadonlySet; + readonly returns: boolean; + readonly type: boolean; + /** A tag that types the whole declaration (`@type`, `@template`, + * `@this`, `@overload`, `@callback`). */ + readonly whole: boolean; +} + +/** The type tags of the package's own JSDoc for a declaration: in its + * leading trivia and in that of the statement or property holding it + * (`/** … *\/ const f = function (…) {}`, `exports.f = function …`). */ +function packageDocOf(node: ts.Node): PackageDoc { + let text = ""; + for (let n: ts.Node | undefined = node; n !== undefined; n = n.parent) { + const sf = n.getSourceFile(); + for (const m of sf.text.slice(n.pos, ts.tokenStartOfNode(n)).matchAll(/\/\*\*[\s\S]*?\*\//g)) + if (!m[0].includes(MARK)) text += m[0]; + const p: ts.Node | undefined = n.parent; + if ( + p === undefined || + !( + ts.isVariableDeclaration(p) || + ts.isVariableDeclarationList(p) || + ts.isVariableStatement(p) || + ts.isPropertyAssignment(p) || + ts.isParenthesizedExpression(p) || + ts.isExpressionStatement(p) || + ts.isExportAssignment(p) || + (ts.isBinaryExpression(p) && + p.operatorToken.kind === ts.SyntaxKind.EqualsToken && + p.right === n) + ) + ) + break; + } + const params = new Set(); + for (const m of text.matchAll(/@(?:param|arg|argument)\b/g)) { + let i = m.index + m[0].length; + while (i < text.length && /\s/.test(text[i]!)) i++; + if (text[i] === "{") { + let depth = 0; + for (; i < text.length; i++) { + if (text[i] === "{") depth++; + else if (text[i] === "}" && --depth === 0) { + i++; + break; + } + } + } + const name = /^\s*\[?\s*([A-Za-z_$][\w$]*)/.exec(text.slice(i)); + if (name !== null) params.add(name[1]!); + } + return { + params, + returns: /@returns?\b/.test(text), + type: /@type\b/.test(text), + whole: /@(?:type|template|this|overload|callback)\b/.test(text), + }; } /** The `typeof` results the values can have. */ @@ -760,19 +868,6 @@ function applyEdits(text: string, edits: readonly Edit[]): string { return out + text.slice(at); } -/** The package's JSDoc type tags become plain text: package JSDoc is a - * hint, not a declaration. The overlay's own comments carry MARK and are - * kept. Lengths are unchanged. */ -export function neutralizeDocTags(text: string): string { - return text.replace(/\/\*\*[\s\S]*?\*\//g, (comment) => { - if (comment.includes(MARK)) return comment; - return comment.replace( - /@(param|type|typedef|returns?|template|this|callback|property|prop|arg|argument|enum|implements|extends|augments|satisfies|overload|import)\b/g, - "-$1", - ); - }); -} - /** A record site's keys: its static keys, or for a literal with spreads its * own keys plus every key a spread source contributes (optional unless * every value of that source has it). Null when a computed key or an diff --git a/packages/compiler/src/frontend/shape-inference/static-objects.ts b/packages/compiler/src/frontend/shape-inference/static-objects.ts index 9b67d2a82..96f46c739 100644 --- a/packages/compiler/src/frontend/shape-inference/static-objects.ts +++ b/packages/compiler/src/frontend/shape-inference/static-objects.ts @@ -341,7 +341,14 @@ export function planStaticObjectRewrites( byKey.set(key, (entry = { op, atoms: new Set() })); mergedOps.push(entry); } - for (const a of analysis.nodes[op.recv]!.atoms) entry.atoms.add(a); + // A nullish receiver makes the read throw, so the rewrites decide on + // the other receivers (absent keys read undefined in the analysis: + // `ns.util` before `ns.util = {…}` runs). In an optional chain a + // nullish receiver short-circuits instead, so it counts. + const optional = (op.node.flags & ts.NodeFlags.OptionalChain) !== 0; + for (const a of analysis.nodes[op.recv]!.atoms) + if (optional || !(a.kind === "prim" && (a.label === "undefined" || a.label === "null"))) + entry.atoms.add(a); } } const forIns: ts.Expression[] = []; @@ -759,9 +766,19 @@ export function planStaticObjectRewrites( const node = op.node; if (!ts.isPropertyAccessExpression(node) && !ts.isElementAccessExpression(node)) continue; if (!isPlainRead(node) || topStatement(node) === null) continue; + // An element key needs no parentheses (and a parenthesized literal key + // would read as a computed one). + const key = + node.parent !== undefined && + ts.isElementAccessExpression(node.parent) && + node.parent.argumentExpression === node; callEdits.push({ sf: node.getSourceFile(), - edit: { start: ts.tokenStartOfNode(node), end: node.end, text: `(${literal})` }, + edit: { + start: ts.tokenStartOfNode(node), + end: node.end, + text: key ? literal : `(${literal})`, + }, }); foldedReads++; } diff --git a/packages/compiler/test/shape-overlay-render.test.ts b/packages/compiler/test/shape-overlay-render.test.ts index 8aa539a74..b0758c14f 100644 --- a/packages/compiler/test/shape-overlay-render.test.ts +++ b/packages/compiler/test/shape-overlay-render.test.ts @@ -1,53 +1,34 @@ -/* renderShapeOverlay neutralizes the package JSDoc in the leading trivia of - * the annotations a build keeps, and leaves the trivia of dropped ones. */ +/* renderShapeOverlay keeps the package's JSDoc as written (package JSDoc is + * a declaration) and writes the annotations a build keeps. */ import { expect, test } from "vitest"; -import { - neutralizeDocTags, - OVERLAY_MARK, - renderShapeOverlay, -} from "../src/frontend/shape-inference/overlay.js"; +import { OVERLAY_MARK, renderShapeOverlay } from "../src/frontend/shape-inference/overlay.js"; import type { ShapeOverlayPlan } from "../src/frontend/shape-inference/overlay.js"; -test("each kept annotation's leading JSDoc is neutralized, once per range", () => { +test("package JSDoc stays as written; dropped annotations are left out", () => { const decls = [ - "/** @param {string} a */\nfunction f(a) {}\n", + "/** @param {string} a */\nfunction f(a, b) {}\n", "/** @type {number} */\nvar x = 1;\n", - "/** @returns {void} */\nfunction g() {}\n", + "function g(c) {}\n", ]; const text = decls.join(""); - const docRanges: { lo: number; hi: number; id: number }[] = []; const edits: { pos: number; id: number; text: string }[] = []; + const at: number[] = []; let lo = 0; for (const decl of decls) { - const hi = lo + decl.indexOf("\n") + 1; - docRanges.push({ lo, hi, id: hi }); - // A declaration's first child starts at the same position: its range is - // the same trivia. - docRanges.push({ lo, hi, id: hi }); - edits.push({ pos: hi, id: hi, text: `/** @type {any} ${OVERLAY_MARK} #${hi} */ ` }); + const pos = lo + (decl.startsWith("/**") ? decl.indexOf("\n") + 1 : 0); + at.push(pos); + edits.push({ pos, id: pos, text: `/** @param {number} b ${OVERLAY_MARK} #${pos} */ ` }); lo += decl.length; } - const plan: ShapeOverlayPlan = { - parts: new Map([["a.js", { text, edits, suffix: "", docRanges }]]), - }; - const dropped = docRanges[2]!.id; + const plan: ShapeOverlayPlan = { parts: new Map([["a.js", { text, edits, suffix: "" }]]) }; const overlay = renderShapeOverlay(plan, { - annotations: new Map([["a.js", new Set([dropped])]]), + annotations: new Map([["a.js", new Set([at[1]!])]]), files: new Set(), }); - const annotated = (decl: string, keep: boolean, at: number): string => { - const head = decl.slice(0, decl.indexOf("\n") + 1); - const rest = decl.slice(head.length); - return keep - ? `${neutralizeDocTags(head)}/** @type {any} ${OVERLAY_MARK} #${at} */ ${rest}` - : decl; - }; - expect(overlay.files.get("a.js")).toBe( - decls - .map((decl, i) => annotated(decl, docRanges[2 * i]!.id !== dropped, docRanges[2 * i]!.id)) - .join(""), - ); - expect(overlay.files.get("a.js")).toContain("/** @type {number} */"); - expect(overlay.files.get("a.js")).toContain("/** -param {string} a */"); + const out = overlay.files.get("a.js")!; + expect(out).toContain("/** @param {string} a */\n/** @param {number} b"); + expect(out).toContain("/** @type {number} */\nvar x = 1;"); + expect(out).not.toContain(`#${at[1]}`); + expect(out).toContain(`#${at[2]} */ function g(c) {}`); expect(overlay.annotations).toBe(2); }); diff --git a/tests/fixtures/npm/cases/shape-absent-reads/main.ts b/tests/fixtures/npm/cases/shape-absent-reads/main.ts new file mode 100644 index 000000000..a7351b0c4 --- /dev/null +++ b/tests/fixtures/npm/cases/shape-absent-reads/main.ts @@ -0,0 +1,3 @@ +// Values shape inference must include undefined (or inherited members) for. +import { run } from "absent-reads"; +console.log(run()); diff --git a/tests/fixtures/npm/cases/shape-absent-reads/node_modules/absent-reads/index.d.ts b/tests/fixtures/npm/cases/shape-absent-reads/node_modules/absent-reads/index.d.ts new file mode 100644 index 000000000..da7f0a141 --- /dev/null +++ b/tests/fixtures/npm/cases/shape-absent-reads/node_modules/absent-reads/index.d.ts @@ -0,0 +1 @@ +export declare function run(): string; diff --git a/tests/fixtures/npm/cases/shape-absent-reads/node_modules/absent-reads/index.js b/tests/fixtures/npm/cases/shape-absent-reads/node_modules/absent-reads/index.js new file mode 100644 index 000000000..4f76a10bc --- /dev/null +++ b/tests/fixtures/npm/cases/shape-absent-reads/node_modules/absent-reads/index.js @@ -0,0 +1,48 @@ +'use strict'; +// Reads that can answer undefined: a member inherited through Object.create, +// an absent key under a computed name, an index past the end, a key written +// after the object was created, and a short-circuited optional chain. +function Schema() {} +Schema.prototype.extend = function extend(extra) { + const result = Object.create(Schema.prototype); + result.names = (this.names || []).concat(extra); + return result; +}; +const base = new Schema().extend(['a']); +const grown = base.extend(['b']); +const ESCAPES = {}; +ESCAPES[10] = '\\n'; +ESCAPES[9] = '\\t'; +function escape(text) { + let out = ''; + for (let i = 0; i < text.length; i++) { + const seq = ESCAPES[text.charCodeAt(i)]; + out += seq || text[i]; + } + return out + ' ' + String(ESCAPES[text.charCodeAt(0)]); +} +function lastOf(list) { + const last = list[list.length]; + return last === undefined ? 'past the end' : last; +} +function later() { + const state = { count: 0 }; + const label = state.label; + state.label = 'set'; + return String(label) + ' ' + state.label; +} +function tail(value, key) { + const t = value?.[key].child; + return t === undefined ? 'none' : 'child ' + t; +} +function run() { + return [ + grown.names.join(','), + escape('a\tb\n'), + lastOf([1, 2]), + later(), + tail(null, 'k'), + tail({ k: { child: 3 } }, 'k'), + ].join(' | '); +} +module.exports = { run }; diff --git a/tests/fixtures/npm/cases/shape-absent-reads/node_modules/absent-reads/package.json b/tests/fixtures/npm/cases/shape-absent-reads/node_modules/absent-reads/package.json new file mode 100644 index 000000000..9126f0f10 --- /dev/null +++ b/tests/fixtures/npm/cases/shape-absent-reads/node_modules/absent-reads/package.json @@ -0,0 +1 @@ +{ "name": "absent-reads", "version": "1.0.0", "main": "index.js", "types": "index.d.ts" } diff --git a/tests/fixtures/npm/cases/shape-absent-reads/package.json b/tests/fixtures/npm/cases/shape-absent-reads/package.json new file mode 100644 index 000000000..ff39d6da5 --- /dev/null +++ b/tests/fixtures/npm/cases/shape-absent-reads/package.json @@ -0,0 +1 @@ +{ "name": "shape-absent-reads-case", "private": true, "type": "module" } diff --git a/tests/fixtures/npm/cases/shape-bigint-arith/main.ts b/tests/fixtures/npm/cases/shape-bigint-arith/main.ts new file mode 100644 index 000000000..a3dd19cbb --- /dev/null +++ b/tests/fixtures/npm/cases/shape-bigint-arith/main.ts @@ -0,0 +1,3 @@ +// Values computed by arithmetic on bigints are bigints. +import { run } from "bigint-arith"; +console.log(run()); diff --git a/tests/fixtures/npm/cases/shape-bigint-arith/node_modules/bigint-arith/index.d.ts b/tests/fixtures/npm/cases/shape-bigint-arith/node_modules/bigint-arith/index.d.ts new file mode 100644 index 000000000..da7f0a141 --- /dev/null +++ b/tests/fixtures/npm/cases/shape-bigint-arith/node_modules/bigint-arith/index.d.ts @@ -0,0 +1 @@ +export declare function run(): string; diff --git a/tests/fixtures/npm/cases/shape-bigint-arith/node_modules/bigint-arith/index.js b/tests/fixtures/npm/cases/shape-bigint-arith/node_modules/bigint-arith/index.js new file mode 100644 index 000000000..2949f4c5e --- /dev/null +++ b/tests/fixtures/npm/cases/shape-bigint-arith/node_modules/bigint-arith/index.js @@ -0,0 +1,15 @@ +'use strict'; +// Arithmetic on bigints answers bigints, also through closures. +const nowNanos = (function () { + const scale = BigInt(1000); + return () => BigInt(Date.now()) * scale; +})(); +const elapsed = (function () { + const clock = typeof process === 'object' ? process.hrtime : undefined; + const origin = nowNanos() - clock.bigint(); + return () => origin + clock.bigint(); +})(); +function run() { + return typeof nowNanos() + ' ' + typeof elapsed(); +} +module.exports = { run }; diff --git a/tests/fixtures/npm/cases/shape-bigint-arith/node_modules/bigint-arith/package.json b/tests/fixtures/npm/cases/shape-bigint-arith/node_modules/bigint-arith/package.json new file mode 100644 index 000000000..04084c0a3 --- /dev/null +++ b/tests/fixtures/npm/cases/shape-bigint-arith/node_modules/bigint-arith/package.json @@ -0,0 +1 @@ +{ "name": "bigint-arith", "version": "1.0.0", "main": "index.js", "types": "index.d.ts" } diff --git a/tests/fixtures/npm/cases/shape-bigint-arith/package.json b/tests/fixtures/npm/cases/shape-bigint-arith/package.json new file mode 100644 index 000000000..ce66e72db --- /dev/null +++ b/tests/fixtures/npm/cases/shape-bigint-arith/package.json @@ -0,0 +1 @@ +{ "name": "shape-bigint-arith-case", "private": true, "type": "module" } diff --git a/tests/fixtures/npm/cases/shape-module-identity/main.ts b/tests/fixtures/npm/cases/shape-module-identity/main.ts new file mode 100644 index 000000000..2590da8fc --- /dev/null +++ b/tests/fixtures/npm/cases/shape-module-identity/main.ts @@ -0,0 +1,4 @@ +// Shape inference must not type objects held by module-level bindings as +// native records: a typed return would copy them (identity and writes lost). +import { run } from "module-identity"; +console.log(run()); diff --git a/tests/fixtures/npm/cases/shape-module-identity/node_modules/module-identity/index.d.ts b/tests/fixtures/npm/cases/shape-module-identity/node_modules/module-identity/index.d.ts new file mode 100644 index 000000000..da7f0a141 --- /dev/null +++ b/tests/fixtures/npm/cases/shape-module-identity/node_modules/module-identity/index.d.ts @@ -0,0 +1 @@ +export declare function run(): string; diff --git a/tests/fixtures/npm/cases/shape-module-identity/node_modules/module-identity/index.js b/tests/fixtures/npm/cases/shape-module-identity/node_modules/module-identity/index.js new file mode 100644 index 000000000..a036f74de --- /dev/null +++ b/tests/fixtures/npm/cases/shape-module-identity/node_modules/module-identity/index.js @@ -0,0 +1,21 @@ +'use strict'; +// Objects held by module-level bindings stay the same object when functions +// return them: identity holds and writes through either reference are seen. +const expected = { label: 'same' }; +function wrapper() { return expected; } +function identity(value) { return value; } +const top = { inner: { n: 1 } }; +function topInner() { return top.inner; } +function run() { + const out = []; + out.push(String(wrapper() === expected), String(identity(wrapper()) === expected)); + const t = topInner(); + t.n = 9; + out.push(String(top.inner.n), String(topInner() === top.inner)); + const local = { inner: { n: 1 } }; + const getInner = () => local.inner; + getInner().n = 5; + out.push(String(local.inner.n)); + return out.join(' '); +} +module.exports = { run }; diff --git a/tests/fixtures/npm/cases/shape-module-identity/node_modules/module-identity/package.json b/tests/fixtures/npm/cases/shape-module-identity/node_modules/module-identity/package.json new file mode 100644 index 000000000..2ca8cea54 --- /dev/null +++ b/tests/fixtures/npm/cases/shape-module-identity/node_modules/module-identity/package.json @@ -0,0 +1 @@ +{ "name": "module-identity", "version": "1.0.0", "main": "index.js", "types": "index.d.ts" } diff --git a/tests/fixtures/npm/cases/shape-module-identity/package.json b/tests/fixtures/npm/cases/shape-module-identity/package.json new file mode 100644 index 000000000..1ecd1d81d --- /dev/null +++ b/tests/fixtures/npm/cases/shape-module-identity/package.json @@ -0,0 +1 @@ +{ "name": "shape-module-identity-case", "private": true, "type": "module" } diff --git a/tests/harness/ci-stack-scope.test.ts b/tests/harness/ci-stack-scope.test.ts index 4abd321d0..a2186c512 100644 --- a/tests/harness/ci-stack-scope.test.ts +++ b/tests/harness/ci-stack-scope.test.ts @@ -16,8 +16,11 @@ const workflow = load( ) as { jobs: Record }; const scopeJob = workflow.jobs.validation_scope!; const scopeScript = scopeJob.steps!.find((step) => step.run)!.run!; +// Validation jobs that run only nightly and on manual runs, outside the +// required check (the shape overlay over the real-package suites). +const nightlyJobs = ["shape_overlay_native", "shape_overlay"]; const heavyJobs = Object.keys(workflow.jobs).filter( - (name) => !["validation_scope", "code_quality", "test"].includes(name), + (name) => !["validation_scope", "code_quality", "test", ...nightlyJobs].includes(name), ); const repository = "vercel-labs/scriptc"; @@ -160,6 +163,16 @@ const full = "needs.validation_scope.outputs.full == 'true'"; const laneOf = (name: string) => /needs\.lanes\.outputs\.(\w+) == 'true'/.exec(workflow.jobs[name]!.if ?? "")?.[1]; +test("nightly validation jobs run only on schedule or manual runs, outside the required check", () => { + for (const name of nightlyJobs) { + expect(workflow.jobs[name], name).toBeDefined(); + expect(workflow.jobs[name]!.if, name).toBe( + "${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}", + ); + expect(workflow.jobs.test!.needs, name).not.toContain(name); + } +}); + test("every expensive CI job is gated and included in the required test check", () => { for (const name of heavyJobs) { const job = workflow.jobs[name]!; diff --git a/tests/harness/npm-static.test.ts b/tests/harness/npm-static.test.ts index 388da2138..ea61d4ac4 100644 --- a/tests/harness/npm-static.test.ts +++ b/tests/harness/npm-static.test.ts @@ -24,7 +24,10 @@ const cacheDir = join(repoRoot, "node_modules/.cache/scriptc-tests"); const sanitize = process.env["SCRIPTC_SAN"] === "1"; /* Sanitized builds of the published Effect pilots alone take 290 to 330 s on a 16-vCPU host, most of it in * the sanitized native link: near or past the suite's 300 s default. */ -const effectBuildTimeout = sanitize ? 900_000 : undefined; +// The shape overlay (SCRIPTC_SHAPE_OVERLAY=1, validation runs) analyzes the +// program and may lower it a second time without the overlay. +const effectBuildTimeout = + sanitize || process.env["SCRIPTC_SHAPE_OVERLAY"] === "1" ? 900_000 : undefined; // Measured milliseconds per test on a CI runner (scripts/ci-test-costs.mjs). const testCosts = ( JSON.parse(readFileSync(new URL("./npm-static-costs.json", import.meta.url), "utf8")) as { From 2a0489a3346ed2d29d26ef48f6d388cd96c98b31 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 13:23:54 -0500 Subject: [PATCH 03/21] Keep the TypeError of receivers that may be null in static-object rewrites The rewrites decide on a read's non-nullish receivers, since the analysis also gives an absent key's read undefined (ns.util before ns.util = {...} runs). A rewrite then removed the property access, so a receiver that was in fact null or undefined no longer threw. - A read whose receiver may also be nullish is rewritten to the read as written followed by the replacement in a comma expression: a direct reference becomes (recv.k, name) and a folded member (recv.k, literal), so a nullish receiver throws Node's TypeError and any other receiver yields the same value. - A function table call through such a table reads table[key] first, before its arguments, where the call reads it. - The static-tables case reads, calls and dispatches through receivers that are the constant object or undefined, and records the ts7 baselines of the npm cases added in this change. --- .../shape-inference/static-objects.ts | 71 ++++++++++++++----- .../cases/concat-dynamic-args/main.ts.json | 6 ++ .../npm/cases/evolving-let-index/main.ts.json | 6 ++ .../npm/cases/shape-absent-reads/main.ts.json | 6 ++ .../npm/cases/shape-bigint-arith/main.ts.json | 6 ++ .../cases/shape-module-identity/main.ts.json | 6 ++ .../npm/cases/static-objects/main.ts.json | 6 ++ .../npm/cases/static-tables/main.ts.json | 6 ++ .../npm/cases/static-tables/js-main.cjs | 9 +++ .../node_modules/static-tables/index.js | 16 +++++ tests/harness/static-tables.test.ts | 5 +- 11 files changed, 125 insertions(+), 18 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/concat-dynamic-args/main.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/evolving-let-index/main.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/shape-absent-reads/main.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/shape-bigint-arith/main.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/shape-module-identity/main.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/static-objects/main.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/static-tables/main.ts.json diff --git a/packages/compiler/src/frontend/shape-inference/static-objects.ts b/packages/compiler/src/frontend/shape-inference/static-objects.ts index 96f46c739..609b9d779 100644 --- a/packages/compiler/src/frontend/shape-inference/static-objects.ts +++ b/packages/compiler/src/frontend/shape-inference/static-objects.ts @@ -33,6 +33,25 @@ import { computeConstantGraphs } from "./constant-graphs.js"; type Node = ts.Node; +/** One read, call or write site, with the receivers of every context it + * was analyzed in. `nullish`: a receiver may also be null or undefined, + * which `atoms` leaves out (see planStaticObjectRewrites). */ +interface MergedOp { + op: ShapeAnalysis["ops"][number]; + atoms: Set; + nullish: boolean; +} + +/** The replacement text of the read `node`: `text`, or, when the receiver + * may be nullish, the read as written followed by `text` in a comma + * expression, so a nullish receiver still throws Node's TypeError and any + * other receiver yields `text`'s value. */ +function keepRead(node: ts.Node, text: string, nullish: boolean): string { + if (!nullish) return text; + const sf = node.getSourceFile(); + return `(${sf.text.slice(ts.tokenStartOfNode(node), node.end)}, ${text})`; +} + interface Edit { readonly start: number; readonly end: number; @@ -328,9 +347,9 @@ export function planStaticObjectRewrites( // One entry per (kind, syntax node): a node the analysis walked in // several contexts (receiver classes, per-site clones) is rewritten only // when the receivers of every context together satisfy the conditions. - const mergedOps: { op: (typeof analysis.ops)[number]; atoms: Set }[] = []; + const mergedOps: MergedOp[] = []; { - const byKey = new Map }>(); + const byKey = new Map(); const ids = new Map(); for (const op of analysis.ops) { let id = ids.get(op.node); @@ -338,17 +357,21 @@ export function planStaticObjectRewrites( const key = `${op.kind}:${op.name ?? ""}:${id}`; let entry = byKey.get(key); if (entry === undefined) { - byKey.set(key, (entry = { op, atoms: new Set() })); + byKey.set(key, (entry = { op, atoms: new Set(), nullish: false })); mergedOps.push(entry); } // A nullish receiver makes the read throw, so the rewrites decide on // the other receivers (absent keys read undefined in the analysis: - // `ns.util` before `ns.util = {…}` runs). In an optional chain a - // nullish receiver short-circuits instead, so it counts. + // `ns.util` before `ns.util = {…}` runs), and `nullish` keeps the + // throw: a rewrite of such a read evaluates the read first. In an + // optional chain a nullish receiver short-circuits instead, so it + // counts. const optional = (op.node.flags & ts.NodeFlags.OptionalChain) !== 0; - for (const a of analysis.nodes[op.recv]!.atoms) + for (const a of analysis.nodes[op.recv]!.atoms) { if (optional || !(a.kind === "prim" && (a.label === "undefined" || a.label === "null"))) entry.atoms.add(a); + else entry.nullish = true; + } } } const forIns: ts.Expression[] = []; @@ -712,7 +735,7 @@ export function planStaticObjectRewrites( // Direct references: `e.k` (called, constructed or read) in the same file // whose receiver is exactly the constant object holding a hoisted // function or class. - for (const { op, atoms } of mergedOps) { + for (const { op, atoms, nullish } of mergedOps) { if (op.kind !== "read" || op.name === null) continue; if (atoms.size !== 1) continue; const recv = [...atoms][0]!; @@ -734,14 +757,22 @@ export function planStaticObjectRewrites( if (text === null) continue; callEdits.push({ sf: node.getSourceFile(), - edit: { start: ts.tokenStartOfNode(node), end: node.end, text }, + edit: { + start: ts.tokenStartOfNode(node), + end: node.end, + text: keepRead(node, text, nullish), + }, }); directCalls++; continue; } callEdits.push({ sf: node.getSourceFile(), - edit: { start: ts.tokenStartOfNode(node), end: node.end, text: name }, + edit: { + start: ts.tokenStartOfNode(node), + end: node.end, + text: keepRead(node, name, nullish), + }, }); directCalls++; } @@ -754,7 +785,7 @@ export function planStaticObjectRewrites( // member stored after the object was created could be read before the // store; those stay reads.) let foldedReads = 0; - for (const { op, atoms } of mergedOps) { + for (const { op, atoms, nullish } of mergedOps) { if (op.kind !== "read" || op.name === null) continue; if (atoms.size !== 1) continue; const o = constantAtoms.get([...atoms][0]!); @@ -777,7 +808,7 @@ export function planStaticObjectRewrites( edit: { start: ts.tokenStartOfNode(node), end: node.end, - text: key ? literal : `(${literal})`, + text: nullish ? keepRead(node, literal, true) : key ? literal : `(${literal})`, }, }); foldedReads++; @@ -808,8 +839,13 @@ export function planStaticObjectRewrites( // key reads the table as written. The members do not use `this`, the // table has no accessors, and comparing a string runs no code, so only // the representation of the call changes. - const tableCalls: { sf: ts.SourceFile; call: ts.CallExpression; arms: [string, string][] }[] = []; - for (const { op, atoms } of mergedOps) { + const tableCalls: { + sf: ts.SourceFile; + call: ts.CallExpression; + arms: [string, string][]; + nullish: boolean; + }[] = []; + for (const { op, atoms, nullish } of mergedOps) { if (op.kind !== "computedRead") continue; const node = op.node; if (!ts.isElementAccessExpression(node) || node.questionDotToken !== undefined) continue; @@ -849,9 +885,9 @@ export function planStaticObjectRewrites( kept.push(`${where}: a member is not a hoisted function of this file`); continue; } - tableCalls.push({ sf: call.getSourceFile(), call, arms }); + tableCalls.push({ sf: call.getSourceFile(), call, arms, nullish }); } - for (const { sf, call, arms } of tableCalls) { + for (const { sf, call, arms, nullish } of tableCalls) { const node = call.expression as ts.ElementAccessExpression; const key = node.argumentExpression.getText(); const args = call.arguments.nodes.map((a) => sliceWithCalls(sf, a)).join(", "); @@ -864,7 +900,10 @@ export function planStaticObjectRewrites( edit: { start: ts.tokenStartOfNode(call), end: call.end, - text: `(${text}${table}[${key}](${args}))`, + // A table that may be nullish is read first, where the call reads + // it, so the read throws before the arguments run; on the constant + // object it reads a plain member with a string key. + text: `(${nullish ? `${table}[${key}], ` : ""}${text}${table}[${key}](${args}))`, }, }); tableDispatches++; diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/concat-dynamic-args/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/concat-dynamic-args/main.ts.json new file mode 100644 index 000000000..0fbf33bad --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/concat-dynamic-args/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/concat-dynamic-args/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/evolving-let-index/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/evolving-let-index/main.ts.json new file mode 100644 index 000000000..8c503a074 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/evolving-let-index/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/evolving-let-index/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/shape-absent-reads/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/shape-absent-reads/main.ts.json new file mode 100644 index 000000000..9a8706ad0 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/shape-absent-reads/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/shape-absent-reads/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/shape-bigint-arith/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/shape-bigint-arith/main.ts.json new file mode 100644 index 000000000..5df93d3c8 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/shape-bigint-arith/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/shape-bigint-arith/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/shape-module-identity/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/shape-module-identity/main.ts.json new file mode 100644 index 000000000..1940110e8 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/shape-module-identity/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/shape-module-identity/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/static-objects/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/static-objects/main.ts.json new file mode 100644 index 000000000..69441c729 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/static-objects/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/static-objects/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/static-tables/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/static-tables/main.ts.json new file mode 100644 index 000000000..7c777f751 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/static-tables/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/static-tables/main.ts" + ], + "diags": [] +} diff --git a/tests/fixtures/npm/cases/static-tables/js-main.cjs b/tests/fixtures/npm/cases/static-tables/js-main.cjs index a3e42ff98..b73c08786 100644 --- a/tests/fixtures/npm/cases/static-tables/js-main.cjs +++ b/tests/fixtures/npm/cases/static-tables/js-main.cjs @@ -20,3 +20,12 @@ try { } catch (e) { console.log(e instanceof TypeError); } +for (const holder of lib.holders) { + for (const read of [() => lib.modeOn(holder), () => lib.addVia(holder), () => lib.callVia(holder, "mul")]) { + try { + console.log(read()); + } catch (e) { + console.log(e.constructor.name, e.message); + } + } +} diff --git a/tests/fixtures/npm/cases/static-tables/node_modules/static-tables/index.js b/tests/fixtures/npm/cases/static-tables/node_modules/static-tables/index.js index a8db112e6..ff38d6403 100644 --- a/tests/fixtures/npm/cases/static-tables/node_modules/static-tables/index.js +++ b/tests/fixtures/npm/cases/static-tables/node_modules/static-tables/index.js @@ -80,4 +80,20 @@ lib.Pair.prototype.sum = function () { return this.a + this.b; }; +// Receivers that may be a constant object or undefined: each read keeps the +// TypeError a nullish receiver throws. +lib.holders = [ + { mode: lib.Mode, ops: lib.ops, table: lib.ops }, + { mode: undefined, ops: undefined, table: undefined }, +]; +lib.modeOn = function (holder) { + return holder.mode.ON; +}; +lib.addVia = function (holder) { + return holder.ops.add(2, 3); +}; +lib.callVia = function (holder, op) { + return holder.table[op](4, 5); +}; + module.exports = { lib }; diff --git a/tests/harness/static-tables.test.ts b/tests/harness/static-tables.test.ts index 2d1e88c65..9f7782e2b 100644 --- a/tests/harness/static-tables.test.ts +++ b/tests/harness/static-tables.test.ts @@ -3,7 +3,8 @@ * consumer, js-main.cjs) calls its constant function table directly, reads * constant primitive members as literals, refers to namespace members of * another module through that module's exports, keeps every form whose - * dynamic read is observable, and prints what Node prints. The npm suites + * dynamic read is observable, keeps the TypeError of a receiver that may + * be undefined, and prints what Node prints. The npm suites * run the TypeScript consumer with and without the overlay. */ import { execFile } from "node:child_process"; import { mkdirSync } from "node:fs"; @@ -32,7 +33,7 @@ test( if (!result.ok) throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); const report = (result.warnings ?? []).find((w) => w.code === "SC6103"); - expect(report?.message).toMatch(/1 function table call, 8 constant reads\)/); + expect(report?.message).toMatch(/2 function table calls, 9 constant reads\)/); const pending = execFileAsync(result.binaryPath, [], { cwd: dirname(entry), encoding: "utf8" }); pending.child.stdin?.end(); const { stdout } = await pending; From c3a72b2d3f053817b5a46d251adbf0d587d0dba4 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Sun, 25 Oct 2026 12:12:45 -0700 Subject: [PATCH 04/21] Cut the shape overlay's compile cost: memo across builds, comparison first, one analysis for fold-only rewrites - memo.ts carries a build's overlay verdicts (refused annotations and files, overlay off, the default build's deferred refusals) to the next build of the same JavaScript, in the build cache. - Builds whose only refusals are deferred lower the program without the overlay first, and drop only what the added refusals blame; when the overlay goes off, that lowering is the build's. - Method families join only methods a call reaches together (receiver unions, extends chains): ZodString's _addCheck returns ZodString, so zod 3 needs no reload. - Refusals in a class's fields or at new C blame annotations that name C. - Rewrites that only fold constant reads compose with the typed overlay planned on the same analysis (one rebuild and analysis instead of two). - Shape analysis prefetches symbols only; input tracking notes a file's text once per tracker. - A declaration a project type patch edited gets no overlay annotation (setShapeOverlayProtectedSpans, for the type-patches layer). Co-Authored-By: Claude Opus 5.5 --- .../compiler/src/frontend/input-tracker.ts | 12 +- packages/compiler/src/frontend/pipeline.ts | 136 ++++++++----- packages/compiler/src/frontend/program.ts | 11 + .../src/frontend/shape-inference/analyze.ts | 12 +- .../src/frontend/shape-inference/fallback.ts | 38 ++++ .../src/frontend/shape-inference/index.ts | 189 ++++++++++++++++- .../src/frontend/shape-inference/memo.ts | 129 ++++++++++++ .../src/frontend/shape-inference/overlay.ts | 192 +++++++++++++++++- .../shape-inference/static-objects.ts | 54 ++++- packages/compiler/src/frontend/ts7/checker.ts | 9 + packages/compiler/src/index.ts | 21 +- .../compiler/test/shape-overlay-cost.test.ts | 127 ++++++++++++ .../test/shape-overlay-patches.test.ts | 86 ++++++++ .../test/shape-overlay-render.test.ts | 43 +++- 14 files changed, 987 insertions(+), 72 deletions(-) create mode 100644 packages/compiler/src/frontend/shape-inference/memo.ts create mode 100644 packages/compiler/test/shape-overlay-cost.test.ts create mode 100644 packages/compiler/test/shape-overlay-patches.test.ts diff --git a/packages/compiler/src/frontend/input-tracker.ts b/packages/compiler/src/frontend/input-tracker.ts index c23a87313..3028f5a01 100644 --- a/packages/compiler/src/frontend/input-tracker.ts +++ b/packages/compiler/src/frontend/input-tracker.ts @@ -138,6 +138,16 @@ export class FrontendInputTracker { } } + /** Raw file names whose source text this tracker recorded. */ + private readonly notedTexts = new Set(); + + /** Records a raw read of a source file's text, once per file name. */ + noteText(fileName: string): void { + if (this.notedTexts.has(fileName)) return; + this.notedTexts.add(fileName); + this.record({ op: "text", path: resolve(fileName) }); + } + record(probe: FrontendInputProbe): void { const key = `${probe.op}\0${probe.path}`; if (probe.op === "text" && this.probes.has(key)) return; @@ -198,7 +208,7 @@ function record(probe: FrontendInputProbe): void { // callbacks, line numbers) changes with edits that keep the tokens. observeSourceText((fileName) => { const tracker = synchronousTracker ?? activeTracker.getStore(); - if (tracker !== undefined) tracker.record({ op: "text", path: resolve(fileName) }); + if (tracker !== undefined) tracker.noteText(fileName); }); /** Mark the active frontend as unsafe to persist because an exact host query diff --git a/packages/compiler/src/frontend/pipeline.ts b/packages/compiler/src/frontend/pipeline.ts index e1ea856c5..e4b27af86 100644 --- a/packages/compiler/src/frontend/pipeline.ts +++ b/packages/compiler/src/frontend/pipeline.ts @@ -3,9 +3,11 @@ import { beginShapeOverlayBuild, excludeRefusedOverlay, + recordShapeOverlayMemo, restoreShapeOverlayState, setShapeOverlayDropped, shapeOverlayActive, + shapeOverlayMemoPlainFences, shapeOverlayReport, } from "./shape-inference/index.js"; import type { ScrDiagnostic } from "../diagnostics/diagnostic.js"; @@ -750,10 +752,10 @@ function loadFrontend( // fallback.ts): refusals drop the overlay annotations they depend on // and the program reloads. Rounds that stop converging escalate to // the overlay of every file involved, then to no overlay at all. - // Refusals deferred to run time (runtime fences) drop only the - // annotations they are blamed on: a fence the overlay did not cause - // stays as it is without the overlay. - let level: "annotations" | "files" | "all" | "fences" = "annotations"; + // Refusals deferred to run time (runtime fences) are compared with + // the build without the overlay below: a fence the overlay did not + // cause stays as it is without the overlay. + let level: "annotations" | "files" | "all" = "annotations"; // Deferred refusals the coverage pass found in code the lowering // judged unreached count too: that code can still run. const fencesOf = (l: LowerResult): ScrDiagnostic[] => [ @@ -764,31 +766,23 @@ function loadFrontend( process.stderr.write( `[shape-fallback] lowered: overlay ${shapeOverlayActive()}, ${lowered.diagnostics.length} refusals, ${fencesOf(lowered).length} deferred\n`, ); + // Failing refusals first; deferred refusals are compared with the + // build without the overlay below. for (let round = 0; shapeOverlayActive(); round++) { - const failed = lowered.diagnostics.length > 0; - const refusals = failed ? lowered.diagnostics : fencesOf(lowered); + const refusals = lowered.diagnostics; if (refusals.length === 0) break; - let changed: boolean; - if (!failed) { - level = "fences"; - changed = - round < OVERLAY_ANNOTATION_ROUNDS && - excludeRefusedOverlay(finalLoad.program, refusals, level); - } else { - if (level === "fences") level = "annotations"; - if (round >= OVERLAY_ANNOTATION_ROUNDS && level === "annotations") level = "files"; - else if (round > OVERLAY_ANNOTATION_ROUNDS) level = "all"; + if (round >= OVERLAY_ANNOTATION_ROUNDS && level === "annotations") level = "files"; + else if (round > OVERLAY_ANNOTATION_ROUNDS) level = "all"; + let changed = excludeRefusedOverlay(finalLoad.program, refusals, level); + while (!changed && level !== "all") { + level = level === "annotations" ? "files" : "all"; changed = excludeRefusedOverlay(finalLoad.program, refusals, level); - while (!changed && level !== "all") { - level = level === "annotations" ? "files" : "all"; - changed = excludeRefusedOverlay(finalLoad.program, refusals, level); - } - // The same round also drops what the deferred refusals blame, - // saving a reload. - if (level !== "all" && fencesOf(lowered).length > 0) - changed = - excludeRefusedOverlay(finalLoad.program, fencesOf(lowered), "fences") || changed; } + // The program reloads anyway: the same round also drops what the + // deferred refusals blame, which often saves the comparison below. + if (level !== "all" && fencesOf(lowered).length > 0) + changed = + excludeRefusedOverlay(finalLoad.program, fencesOf(lowered), "fences") || changed; if (process.env["SCRIPTC_DEBUG_SHAPE_FALLBACK"] !== undefined) process.stderr.write( `[shape-fallback] round ${round} level ${level}: ${refusals.length} refusals (${refusals @@ -817,40 +811,79 @@ function loadFrontend( ); } // Deferred refusals the program without the overlay does not have - // mean the overlay changed what runs: the build compiles without it - // (the overlay must not add failures the default build lacks). + // mean the overlay changed what runs (the overlay must not add + // failures the default build lacks). The build lowers the program + // without the overlay once, drops what the added refusals blame and + // reloads; when they persist (or blame nothing), it compiles without + // the overlay, reusing that lowering. A memo of the same JavaScript + // that already knows every remaining refusal skips the comparison. + let plainFences: Set | null = null; if (shapeOverlayActive() && lowered.diagnostics.length === 0) { + const keyOf = (l: typeof finalLoad, d: ScrDiagnostic): string => { + const sf = l.program.getSourceFile(d.loc.file); + const line = sf === undefined ? -1 : sf.getLineAndCharacterOfPosition(d.loc.start).line; + return `${d.code}|${d.loc.file}|${line}`; + }; + const remembered = shapeOverlayMemoPlainFences(); const fences = fencesOf(lowered); - if (fences.length > 0) { - const keyOf = (l: typeof finalLoad, d: ScrDiagnostic): string => { - const sf = l.program.getSourceFile(d.loc.file); - const line = sf === undefined ? -1 : sf.getLineAndCharacterOfPosition(d.loc.start).line; - return `${d.code}|${d.loc.file}|${line}`; - }; + const knownByMemo = + remembered !== null && fences.every((d) => remembered.has(keyOf(finalLoad, d))); + if (fences.length > 0 && !knownByMemo) { const overlayState = finalLoad.shapeOverlayState; setShapeOverlayDropped(true); const plainLoad = loadProgram(entryPath, { npmStatic: finalNpmStatic, externalTypes }); const plainPreflight = checkPreflight(plainLoad); const plain = plainPreflight.length === 0 ? lowerLoad(plainLoad) : null; - const known = new Set( - plain === null ? [] : fencesOf(plain).map((d) => keyOf(plainLoad, d)), - ); - const added = fences.filter((d) => !known.has(keyOf(finalLoad, d))); - if (plain !== null && plain.diagnostics.length === 0 && added.length > 0) { - if (process.env["SCRIPTC_DEBUG_SHAPE_FALLBACK"] !== undefined) - process.stderr.write( - `[shape-fallback] ${added.length} deferred refusals the default build lacks (${added - .slice(0, 3) - .map((d) => `${d.loc.file.split("/").pop()}: ${d.message.slice(0, 60)}`) - .join("; ")}); overlay off\n`, - ); - finalLoad.dispose(); - finalLoad = plainLoad; - lowered = plain; - } else { + setShapeOverlayDropped(false); + if (overlayState !== undefined) restoreShapeOverlayState(overlayState); + if (plain === null || plain.diagnostics.length > 0) { + // Nothing to compare with: the default build itself fails. plainLoad.dispose(); - setShapeOverlayDropped(false); - if (overlayState !== undefined) restoreShapeOverlayState(overlayState); + } else { + const known = new Set(fencesOf(plain).map((d) => keyOf(plainLoad, d))); + plainFences = known; + let usePlain = false; + for (let round = 0; ; round++) { + const added = fencesOf(lowered).filter((d) => !known.has(keyOf(finalLoad, d))); + if (added.length === 0) break; + const changed = + round < OVERLAY_ANNOTATION_ROUNDS && + excludeRefusedOverlay(finalLoad.program, added, "fences"); + if (process.env["SCRIPTC_DEBUG_SHAPE_FALLBACK"] !== undefined) + process.stderr.write( + `[shape-fallback] ${added.length} deferred refusals the default build lacks (${added + .slice(0, 3) + .map((d) => `${d.loc.file.split("/").pop()}: ${d.message.slice(0, 60)}`) + .join("; ")}), changed ${changed}\n`, + ); + if (!changed) { + usePlain = true; + break; + } + finalLoad.dispose(); + finalLoad = loadProgram(entryPath, { npmStatic: finalNpmStatic, externalTypes }); + if (checkPreflight(finalLoad).length > 0) { + usePlain = true; + break; + } + lowered = lowerOnce(); + if (lowered.diagnostics.length > 0) { + usePlain = true; + break; + } + } + if (usePlain) { + if (process.env["SCRIPTC_DEBUG_SHAPE_FALLBACK"] !== undefined) + process.stderr.write(`[shape-fallback] overlay off\n`); + setShapeOverlayDropped(true); + if (plainLoad.shapeOverlayState !== undefined) + restoreShapeOverlayState(plainLoad.shapeOverlayState); + finalLoad.dispose(); + finalLoad = plainLoad; + lowered = plain; + } else { + plainLoad.dispose(); + } } } } @@ -874,6 +907,7 @@ function loadFrontend( return { ...lowered, module: null, diagnostics: reloadPreflight }; lowered = lowerOnce(); } + recordShapeOverlayMemo(plainFences); // The declaration-typed boundary's notes (SC6102) join the // performance warnings; they never fail a build. const notes = [ diff --git a/packages/compiler/src/frontend/program.ts b/packages/compiler/src/frontend/program.ts index b54976013..5a8d40696 100644 --- a/packages/compiler/src/frontend/program.ts +++ b/packages/compiler/src/frontend/program.ts @@ -1021,9 +1021,16 @@ function loadProgram7( if (overlayer !== undefined && jsx.refusal === null) { // Rounds: structural rewrites, then the typed overlay of the rebuilt // program (shape-inference/index.ts). + const timeShapes = process.env["SCRIPTC_DEBUG_SHAPE_TIME"] !== undefined; for (let round = 0; round < 3; round++) { + const t0 = timeShapes ? performance.now() : 0; const overlay = overlayer(program, round); + if (timeShapes) + process.stderr.write( + `[shape-time] round ${round} overlayer ${(performance.now() - t0).toFixed(0)} ms\n`, + ); if (overlay === null || overlay.size === 0) break; + const t1 = timeShapes ? performance.now() : 0; const before = program; // The tsc gate reads the files as written: its diagnostics are // taken before the overlay's files replace them on the host, and @@ -1042,6 +1049,10 @@ function loadProgram7( for (const [file, text] of overlay) host.addVirtualFile(file, text); program = host.createProgram([...programRoots, overridesDtsPath()], options); if (tscWorld.program !== before) before.dispose(); + if (timeShapes) + process.stderr.write( + `[shape-time] round ${round} gate+program ${(performance.now() - t1).toFixed(0)} ms\n`, + ); if (shapeOverlayDone()) break; } } diff --git a/packages/compiler/src/frontend/shape-inference/analyze.ts b/packages/compiler/src/frontend/shape-inference/analyze.ts index 16d982d9e..ac24ca562 100644 --- a/packages/compiler/src/frontend/shape-inference/analyze.ts +++ b/packages/compiler/src/frontend/shape-inference/analyze.ts @@ -324,7 +324,15 @@ function propertyNameText(name: Node): string | null { return null; } +const shortFiles = new Map(); + function shortFile(fileName: string): string { + let short = shortFiles.get(fileName); + if (short === undefined) shortFiles.set(fileName, (short = shortFileOf(fileName))); + return short; +} + +function shortFileOf(fileName: string): string { const norm = fileName.split("\\").join("/"); const nm = norm.lastIndexOf("/node_modules/"); if (nm >= 0) { @@ -506,7 +514,9 @@ class Analyzer { this.files.push(sf); if (inPackage) this.packageFiles++; } - for (const sf of this.files) this.checker.prefetchSourceFile(sf); + // Analysis resolves names and never asks for types: the program it + // runs on is usually replaced by the overlay's before lowering. + for (const sf of this.files) this.checker.prefetchSourceFileSymbols(sf); for (const sf of this.files) this.moduleBody(sf); this.finish(); if (this.profiling) this.profileReport(); diff --git a/packages/compiler/src/frontend/shape-inference/fallback.ts b/packages/compiler/src/frontend/shape-inference/fallback.ts index 81759dad6..0e6f474b8 100644 --- a/packages/compiler/src/frontend/shape-inference/fallback.ts +++ b/packages/compiler/src/frontend/shape-inference/fallback.ts @@ -159,6 +159,42 @@ export function attributeOverlayRefusals( return calls.get(fn) ?? []; }; + // Annotations whose type text names an identifier, by identifier: a + // class the lowering refuses compiles dynamically until a type names it. + let naming: Map | null = null; + const blameNaming = (name: string): void => { + if (naming === null) { + naming = new Map(); + for (const file of overlayFiles) { + const text = program.getSourceFile(file)?.text; + if (text === undefined) continue; + for (const m of text.matchAll(/\/\*\*[\s\S]*?\*\//g)) { + if (!m[0].includes(OVERLAY_MARK)) continue; + const ids = overlayAnnotationIds(m[0]); + for (const w of new Set(m[0].match(/[A-Za-z_$][\w$]*/g) ?? [])) { + let l = naming.get(w); + if (l === undefined) naming.set(w, (l = [])); + for (const id of ids) l.push([file, id]); + } + } + } + } + for (const [file, id] of naming.get(name) ?? []) add(file, id); + }; + // The class a refusal is about: a field or the heritage of a class + // declaration, or a `new C(...)` of a named class. + const refusedClassName = (node: ts.Node): string | null => { + for (let n: ts.Node | undefined = node; n !== undefined; n = n.parent) { + if (ts.isNewExpression(n) && ts.isIdentifier(n.expression)) return n.expression.text; + if (ts.isFunctionLike(n)) return null; + if (ts.isClassLikeDeclaration(n)) { + const name = (n as { name?: ts.Node }).name; + return name !== undefined && ts.isIdentifier(name) ? name.text : null; + } + } + return null; + }; + for (const d of diagnostics) { const sf = program.getSourceFile(d.loc.file); if (sf === undefined) { @@ -175,6 +211,8 @@ export function attributeOverlayRefusals( blameNode(n); if (statement === node && (ts.isStatement(n) || isClassMember(n))) statement = n; } + const refusedClass = refusedClassName(node); + if (refusedClass !== null) blameNaming(refusedClass); const own = blamed; blameReferences(node, 0); if (blamed === own && statement !== node) blameReferences(statement, 0); diff --git a/packages/compiler/src/frontend/shape-inference/index.ts b/packages/compiler/src/frontend/shape-inference/index.ts index 43e8357cc..5f5547e39 100644 --- a/packages/compiler/src/frontend/shape-inference/index.ts +++ b/packages/compiler/src/frontend/shape-inference/index.ts @@ -9,7 +9,9 @@ import { isShapelessPackage } from "../package-shims.js"; import { analyzeShapes, isPackageJsFile, type ShapeAnalysis } from "./analyze.js"; import { createHash } from "node:crypto"; import { + hasShapeOverlayProtectedSpans, OVERLAY_MARK, + planOnRewrittenTexts, planShapeOverlay, renderShapeOverlay, type OverlayExclusions, @@ -17,13 +19,27 @@ import { } from "./overlay.js"; import { attributeOverlayRefusals } from "./fallback.js"; import { checksInferredFact } from "./inferred-checks.js"; -import { planStaticObjectRewrites, type StaticObjectRewrites } from "./static-objects.js"; +import { + planStaticObjectRewrites, + rewrittenPosition, + type StaticObjectRewrites, +} from "./static-objects.js"; import { isJsSourceFile } from "../program.js"; import { buildShapeReport } from "./report.js"; import { computeConstantGraphs, constantGraphReport } from "./constant-graphs.js"; +import { + overlayMemoKey, + readOverlayMemo, + shapeOverlayMemoEnabled, + textHash, + writeOverlayMemo, + type OverlayMemo, +} from "./memo.js"; export { analyzeShapes } from "./analyze.js"; export { buildShapeReport } from "./report.js"; +export { setShapeOverlayMemoDir } from "./memo.js"; +export { setShapeOverlayProtectedSpans } from "./overlay.js"; /** SCRIPTC_DEBUG_SHAPES: `1` reports package code in hot functions, * `verbose` every package site, `all` user files too, any other value only @@ -134,6 +150,18 @@ let exclusions: { annotations: Map>; files: Set } = }; let overlayDropped = false; let droppedAnnotations = 0; +/* The annotations dropped before a comparison load turned the overlay off + * for a moment (setShapeOverlayDropped). */ +let droppedBeforeComparison = 0; + +/* The build's memo (memo.ts): its key, what an earlier build recorded, the + * hashes of the JavaScript files as written, and the last overlay plan. */ +let memoKey: string | null = null; +let memo: OverlayMemo | null = null; +let memoOff = false; +let asWrittenHashes = new Map(); +let lastPlan: ShapeOverlayPlan | null = null; +let memoSeededPlan: ShapeOverlayPlan | null = null; /** Starts a build: no annotation is excluded yet. */ export function beginShapeOverlayBuild(): void { @@ -141,14 +169,100 @@ export function beginShapeOverlayBuild(): void { exclusions = { annotations: new Map(), files: new Set() }; overlayDropped = false; droppedAnnotations = 0; + droppedBeforeComparison = 0; + memoKey = null; + memo = null; + memoOff = false; + asWrittenHashes = new Map(); + lastPlan = null; + memoSeededPlan = null; } /** Turns the overlay off for the rest of the build (true), or back on * (false): the pipeline compares a load without it when the overlay's * program fails a check the program without it passes. */ export function setShapeOverlayDropped(dropped: boolean): void { + if (dropped && !overlayDropped) droppedBeforeComparison = droppedAnnotations; overlayDropped = dropped; - droppedAnnotations = dropped ? overlayAnnotations : 0; + droppedAnnotations = dropped ? overlayAnnotations : droppedBeforeComparison; +} + +/** The memo's deferred refusals of the build without the overlay, if an + * earlier build of the same JavaScript recorded them. */ +export function shapeOverlayMemoPlainFences(): ReadonlySet | null { + return memo === null || memo.plainFences.length === 0 ? null : new Set(memo.plainFences); +} + +/** Records the build's verdicts for the next build of the same + * JavaScript (memo.ts): the refused annotations and files, whether the + * build compiled without the overlay, and the deferred refusals of the + * build without it (`plainFences`, from a comparison). */ +export function recordShapeOverlayMemo(plainFences: Iterable | null): void { + if (memoKey === null || !shapeOverlayMemoEnabled() || memoOff) return; + const annotations: { file: string; hash: string; ids: number[] }[] = []; + for (const [file, ids] of exclusions.annotations) { + const part = lastPlan?.parts.get(file); + if (part === undefined || ids.size === 0) continue; + annotations.push({ file, hash: textHash(part.text), ids: [...ids].sort((a, b) => a - b) }); + } + annotations.sort((a, b) => (a.file < b.file ? -1 : a.file > b.file ? 1 : 0)); + const files: { file: string; hash: string }[] = []; + for (const file of [...exclusions.files].sort()) { + const hash = asWrittenHashes.get(file); + if (hash !== undefined) files.push({ file, hash }); + } + const plain = new Set(memo?.plainFences ?? []); + for (const k of plainFences ?? []) plain.add(k); + const next: OverlayMemo = { + off: overlayDropped, + annotations, + files, + plainFences: [...plain].sort(), + }; + if (memo !== null && JSON.stringify(memo) === JSON.stringify(next)) return; + writeOverlayMemo(memoKey, next); + memo = next; +} + +/* Round 0 of a load: the memo of the program's JavaScript, read once per + * key. A memo that says the build compiled without the overlay turns it + * off before any analysis; whole-file exclusions apply to files whose text + * is unchanged. */ +function readMemoFor(program: ts.Program): void { + if (!shapeOverlayMemoEnabled()) return; + const js = program.getSourceFiles().filter((sf) => !sf.isDeclarationFile && isJsSourceFile(sf)); + const key = overlayMemoKey(js.map((sf) => [sf.fileName, sf.text] as const)); + if (key === memoKey) return; + memoKey = key; + asWrittenHashes = new Map(js.map((sf) => [sf.fileName, textHash(sf.text)] as const)); + memo = readOverlayMemo(key); + memoOff = false; + if (memo === null) return; + if (memo.off) { + memoOff = true; + overlayDropped = true; + return; + } + for (const f of memo.files) + if (asWrittenHashes.get(f.file) === f.hash) exclusions.files.add(f.file); +} + +/* Applies the memo's refused annotations to a plan, once per plan: ids + * count only for a file whose overlay input text is unchanged. */ +function seedMemoAnnotations(plan: ShapeOverlayPlan): void { + if (memo === null || memoSeededPlan === plan) return; + memoSeededPlan = plan; + for (const a of memo.annotations) { + const part = plan.parts.get(a.file); + if (part === undefined || textHash(part.text) !== a.hash) continue; + let set = exclusions.annotations.get(a.file); + if (set === undefined) exclusions.annotations.set(a.file, (set = new Set())); + for (const id of a.ids) { + if (set.has(id)) continue; + set.add(id); + droppedAnnotations++; + } + } } /** Whether the current load has overlay files or structural rewrites. */ @@ -266,7 +380,10 @@ export function shapeOverlayer( program: ts.Program, round: number, ): ReadonlyMap | null { - if (round === 0) resetShapeOverlay(); + if (round === 0) { + resetShapeOverlay(); + readMemoFor(program); + } if (overlayDropped) return null; const key = `${programTextKey(program)}|${[...exclusions.files].sort().join(",")}`; const cached = overlayCache.get(key); @@ -275,9 +392,16 @@ export function shapeOverlayer( if (entry.budgetExceeded === true) return null; let analysis: ShapeAnalysis | null = null; const analyzed = (): ShapeAnalysis | null => { - analysis ??= analyzeShapes(program, program.getTypeChecker(), { - siteClones: overlaySiteClones(), - }); + if (analysis === null) { + const t0 = performance.now(); + analysis = analyzeShapes(program, program.getTypeChecker(), { + siteClones: overlaySiteClones(), + }); + if (process.env["SCRIPTC_DEBUG_SHAPE_TIME"] !== undefined) + process.stderr.write( + `[shape-time] round ${round} analysis ${(performance.now() - t0).toFixed(0)} ms\n`, + ); + } if (analysis.budgetExceeded) { entry.budgetExceeded = true; return null; @@ -292,7 +416,11 @@ export function shapeOverlayer( if (a === null) return null; entry.rewrites = planStaticObjectRewrites( a, - overlayJsFiles(a).filter((sf) => !exclusions.files.has(sf.fileName)), + // Files a project type patch edited stay as served: the patch's + // spans are offsets in that text. + overlayJsFiles(a).filter( + (sf) => !exclusions.files.has(sf.fileName) && !hasShapeOverlayProtectedSpans(sf.fileName), + ), ); } const rewrites = entry.rewrites; @@ -309,7 +437,35 @@ export function shapeOverlayer( for (const [file, text] of rewrites.files) process.stderr.write(`[static-objects] ${file}\n${text}\n`); } - if (rewrites.files.size > 0) return rewrites.files; + if (rewrites.files.size > 0) { + // Rewrites that only replace constant reads keep every declaration + // where it is: the typed overlay is planned on this analysis and + // moved onto the rewritten texts, so the program is rebuilt and + // analyzed once instead of twice. Hoisting classes or functions + // changes the declarations: the rewritten program is analyzed again. + const compose = + rewrites.classes + rewrites.functions === 0 && process.env["SCRIPTC_SHAPE_COMPOSE"] !== "0"; + if (!compose) return rewrites.files; + if (entry.plan === undefined) { + const a = analyzed(); + if (a === null) return null; + const report = buildShapeReport(a, "1"); + const plan = planShapeOverlay(a, report.sites, overlayJsFiles(a), { + annotations: new Map(), + files: exclusions.files, + }); + const moved = new Map number | null }>(); + for (const [file, text] of rewrites.files) { + const edits = rewrites.edits.get(file) ?? []; + moved.set(file, { text, map: (pos) => rewrittenPosition(edits, pos) }); + } + entry.plan = planOnRewrittenTexts(plan, moved); + } + const files = renderTyped(entry.plan); + for (const [file, text] of rewrites.files) if (!files.has(file)) files.set(file, text); + renderedFiles = new Set(files.keys()); + return files; + } } if (round > 1) return null; if (entry.plan === undefined) { @@ -321,7 +477,16 @@ export function shapeOverlayer( files: exclusions.files, }); } - const overlay = renderShapeOverlay(entry.plan, overlayExclusions()); + const files = renderTyped(entry.plan); + return files.size > 0 ? files : null; +} + +/* Renders a typed plan with the build's exclusions and registers the + * overlay files; the typed overlay is a load's last round. */ +function renderTyped(plan: ShapeOverlayPlan): Map { + lastPlan = plan; + seedMemoAnnotations(plan); + const overlay = renderShapeOverlay(plan, overlayExclusions()); overlayFiles = new Set(overlay.typed); renderedFiles = new Set(overlay.files.keys()); overlayAnnotations = overlay.annotations; @@ -332,7 +497,7 @@ export function shapeOverlayer( } // The typed overlay is the last round. overlayDone = true; - return overlay.files.size > 0 ? overlay.files : null; + return overlay.files; } let staticObjectStats = { @@ -409,7 +574,9 @@ export function shapeOverlayReport(entryFile: string): ScrDiagnostic[] { // Annotations whose lowering was refused (fallback.ts); they compile // untyped. const fallback = overlayDropped - ? "the lowering refused code the overlay typed, so the build compiles without it" + ? memoOff + ? "an earlier build of the same JavaScript compiled without the overlay, so this build does too" + : "the lowering refused code the overlay typed, so the build compiles without it" : droppedAnnotations + exclusions.files.size > 0 ? `it dropped ${droppedAnnotations} annotation${droppedAnnotations === 1 ? "" : "s"}` + (exclusions.files.size > 0 diff --git a/packages/compiler/src/frontend/shape-inference/memo.ts b/packages/compiler/src/frontend/shape-inference/memo.ts new file mode 100644 index 000000000..8f5632a42 --- /dev/null +++ b/packages/compiler/src/frontend/shape-inference/memo.ts @@ -0,0 +1,129 @@ +/* The overlay's verdicts carried from one build to the next build of the + * same JavaScript (fallback.ts, pipeline.ts): + * - the annotations and files whose lowering was refused, so the next + * build renders the overlay without them instead of lowering, refusing + * and reloading again; + * - that the build compiled without the overlay, so the next build does + * not analyze at all; + * - the deferred refusals the build without the overlay has, so the next + * build need not lower the program without the overlay to compare. + * + * A memo is keyed by the compiler, the SCRIPTC_SHAPE_* settings and the + * text of every JavaScript file of the program; the program's TypeScript + * files are not part of the key, so editing them keeps the memo. Every + * verdict only makes the overlay smaller or answers a comparison about + * the same JavaScript, and a build still runs the fallback on whatever + * the lowering refuses. The memo directory lives in the build cache, so + * SCRIPTC_NO_CACHE=1 turns it off; it is never read or written without + * one. */ +import { createHash } from "node:crypto"; +import { mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; + +export interface OverlayMemo { + /** The build compiled without the overlay. */ + readonly off: boolean; + /** Refused annotation ids per file, valid for the overlay input text + * with this hash. */ + readonly annotations: readonly MemoFileIds[]; + /** Files whose whole overlay was refused, by their text as written. */ + readonly files: readonly MemoFile[]; + /** Deferred refusals of the build without the overlay + * (`code|file|line`). */ + readonly plainFences: readonly string[]; +} + +export interface MemoFile { + readonly file: string; + readonly hash: string; +} + +export interface MemoFileIds { + readonly file: string; + readonly hash: string; + readonly ids: readonly number[]; +} + +let memoDir: string | null = null; + +/** Where memos live (a directory of the build cache, per compiler), or + * null for none. The compile driver sets it per build. */ +export function setShapeOverlayMemoDir(dir: string | null): void { + memoDir = dir; +} + +export function shapeOverlayMemoEnabled(): boolean { + return memoDir !== null; +} + +export function textHash(text: string): string { + return createHash("sha256").update(text).digest("hex").slice(0, 32); +} + +/** The memo key of a program's JavaScript files (name and text). */ +export function overlayMemoKey(files: readonly (readonly [string, string])[]): string { + const hash = createHash("sha256"); + const settings = Object.keys(process.env) + .filter((k) => k.startsWith("SCRIPTC_SHAPE_") && !k.startsWith("SCRIPTC_SHAPE_CHECK")) + .sort(); + for (const k of settings) + hash + .update(k) + .update("=") + .update(process.env[k] ?? "") + .update("\0"); + const sorted = [...files].sort((a, b) => (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0)); + for (const [name, text] of sorted) hash.update(name).update("\0").update(text).update("\0"); + return hash.digest("hex").slice(0, 40); +} + +export function readOverlayMemo(key: string): OverlayMemo | null { + if (memoDir === null) return null; + try { + const raw: unknown = JSON.parse(readFileSync(join(memoDir, `${key}.json`), "utf8")); + return isOverlayMemo(raw) ? raw : null; + } catch { + return null; + } +} + +export function writeOverlayMemo(key: string, memo: OverlayMemo): void { + if (memoDir === null) return; + try { + mkdirSync(memoDir, { recursive: true, mode: 0o700 }); + const path = join(memoDir, `${key}.json`); + const temp = `${path}.${process.pid}.${Date.now()}.tmp`; + writeFileSync(temp, JSON.stringify(memo), { mode: 0o600 }); + renameSync(temp, path); + } catch { + // The memo only saves work; a build that cannot write it is unaffected. + } +} + +function isOverlayMemo(raw: unknown): raw is OverlayMemo { + if (raw === null || typeof raw !== "object") return false; + const m = raw as Record; + const files = m["files"]; + const annotations = m["annotations"]; + const plain = m["plainFences"]; + return ( + typeof m["off"] === "boolean" && + Array.isArray(files) && + files.every(isMemoFile) && + Array.isArray(annotations) && + annotations.every( + (a: unknown) => + isMemoFile(a) && + Array.isArray((a as Record)["ids"]) && + ((a as Record)["ids"] as unknown[]).every((i) => typeof i === "number"), + ) && + Array.isArray(plain) && + plain.every((s: unknown) => typeof s === "string") + ); +} + +function isMemoFile(raw: unknown): boolean { + if (raw === null || typeof raw !== "object") return false; + const f = raw as Record; + return typeof f["file"] === "string" && typeof f["hash"] === "string"; +} diff --git a/packages/compiler/src/frontend/shape-inference/overlay.ts b/packages/compiler/src/frontend/shape-inference/overlay.ts index f59b2057f..f900f03bf 100644 --- a/packages/compiler/src/frontend/shape-inference/overlay.ts +++ b/packages/compiler/src/frontend/shape-inference/overlay.ts @@ -57,6 +57,50 @@ interface OverlayPart { readonly suffix: string; } +/** Spans of a served package file that a project type patch edited (the + * type-patches layer's `typePatchedSpans`, in served-text offsets). */ +export type ProtectedSpans = (fileName: string) => readonly { start: number; end: number }[]; + +let protectedSpansOf: ProtectedSpans = () => []; + +/** Installs the source of patched spans: the overlay neither annotates nor + * rewrites a declaration a project type patch edited, since the patch is + * the project's own declaration of its type. */ +export function setShapeOverlayProtectedSpans(spans: ProtectedSpans | null): void { + protectedSpansOf = spans ?? (() => []); +} + +/** Whether a file has patched spans (static-objects.ts leaves it as served, + * so the spans keep their offsets). */ +export function hasShapeOverlayProtectedSpans(fileName: string): boolean { + return protectedSpansOf(fileName).length > 0; +} + +/** Whether the declaration whose first token starts at `pos` overlaps a + * patched span: its leading comments and its header (up to its body). */ +export function declarationPatched( + sf: ts.SourceFile, + pos: number, + spans: readonly { start: number; end: number }[], +): boolean { + if (spans.length === 0) return false; + let node: ts.Node = sf; + for (;;) { + let next: ts.Node | undefined; + ts.forEachChild(node, (c) => { + if (next === undefined && c.pos <= pos && c.end > pos) next = c; + return undefined; + }); + if (next === undefined || ts.tokenStartOfNode(next) > pos) break; + node = next; + if (ts.tokenStartOfNode(node) === pos) break; + } + if (node === sf) return false; + const body = (node as { body?: ts.Node }).body; + const end = body !== undefined ? ts.tokenStartOfNode(body) : node.end; + return spans.some((span) => span.start < end && span.end > node.pos); +} + /** Marks the overlay's own comments (an unknown JSDoc tag, ignored). */ export const OVERLAY_MARK = "@scriptcShape"; const MARK = OVERLAY_MARK; @@ -142,6 +186,19 @@ export function planShapeOverlay( list.push(branch); } } + // Classes whose instances meet as the receivers of one member read, by + // member name: their same-named methods need one signature. + const cooccurring = new Map[]>(); + for (const op of analysis.ops) { + if (op.kind !== "read" || op.name === null) continue; + const classes = new Set(); + for (const a of analysis.nodes[op.recv]!.atoms) + if (a.kind === "instance" && a.node !== undefined) classes.add(a.node); + if (classes.size < 2) continue; + let l = cooccurring.get(op.name); + if (l === undefined) cooccurring.set(op.name, (l = [])); + l.push(classes); + } for (const sf of packageFiles) { if (exclusions.files.has(sf.fileName)) continue; const typedefs = new Map(); @@ -150,7 +207,10 @@ export function planShapeOverlay( // One annotation: a JSDoc comment at `pos` carrying MARK and the // annotation's id (its position), so a refused lowering can name it and // the next load drop it. `close`: the text that closes a type cast. + const patchedSpans = protectedSpansOf(sf.fileName); const annotate = (pos: number, tags: string, close?: number): void => { + // A declaration a project type patch edited keeps the patch alone. + if (declarationPatched(sf, pos, patchedSpans)) return; edits.push({ pos, id: pos, @@ -414,9 +474,14 @@ export function planShapeOverlay( if (method.body !== undefined) for (const st of method.body.statements.nodes) scan(st); }; - // Same-named instance methods of the file's classes form a family: a - // call on a union of those classes needs one signature, so each member's - // parameters and result join the whole family's values. + // Same-named instance methods of the file's classes form a family when + // a call can reach several of them: a call on a union of those classes + // needs one signature, so each member's parameters and result join the + // whole family's values. Two members join when their classes' instances + // meet as the receivers of one read of the method, or when one class + // extends the other (an override keeps its base's signature). Members + // no union reaches keep their own values: ZodString's `_addCheck` + // returns a ZodString, not every schema class's `_addCheck` result. const families = new Map(); const familyVisit = (node: ts.Node): void => { if ( @@ -436,6 +501,84 @@ export function planShapeOverlay( }); }; familyVisit(sf); + const familyParent = new Map(); + const familyFind = (m: ts.MethodDeclaration): ts.MethodDeclaration => { + let r = m; + for (let p = familyParent.get(r); p !== undefined && p !== r; p = familyParent.get(r)) r = p; + return r; + }; + const familyUnion = (a: ts.MethodDeclaration, b: ts.MethodDeclaration): void => { + const ra = familyFind(a); + const rb = familyFind(b); + if (ra !== rb) familyParent.set(rb, ra); + }; + const classNameOf = (c: ts.Node): string | null => { + const named = c as { name?: ts.Node }; + if (named.name !== undefined && ts.isIdentifier(named.name)) return named.name.text; + const p = c.parent; + return p !== undefined && ts.isVariableDeclaration(p) && ts.isIdentifier(p.name) + ? p.name.text + : null; + }; + const baseNameOf = (c: ts.Node): string | null => { + const clauses = (c as { heritageClauses?: { nodes: readonly ts.HeritageClause[] } }) + .heritageClauses; + for (const h of list(clauses)) { + if (h.token !== ts.SyntaxKind.ExtendsKeyword) continue; + const t = list(h.types)[0]; + if (t !== undefined && ts.isIdentifier(t.expression)) return t.expression.text; + } + return null; + }; + for (const [name, members] of families) { + if (members.length < 2) continue; + // The member a class's instances run: its own, or its nearest base's + // (by name, within the file). + const byClass = new Map(); + for (const m of members) if (m.parent !== undefined) byClass.set(m.parent, m); + const classesByName = new Map(); + const collect = (node: ts.Node): void => { + if (ts.isClassDeclaration(node) || ts.isClassExpression(node)) { + const n = classNameOf(node); + if (n !== null && !classesByName.has(n)) classesByName.set(n, node); + } + ts.forEachChild(node, (c) => { + collect(c); + return undefined; + }); + }; + const memberOf = (c: ts.Node): ts.MethodDeclaration | undefined => { + let cur: ts.Node | undefined = c; + for (let depth = 0; cur !== undefined && depth < 16; depth++) { + const own = byClass.get(cur); + if (own !== undefined) return own; + const base = baseNameOf(cur); + if (base === null) return undefined; + if (classesByName.size === 0) collect(sf); + cur = classesByName.get(base); + } + return undefined; + }; + // Overrides join their base's member. + for (const m of members) { + const base = m.parent === undefined ? null : baseNameOf(m.parent); + if (base === null) continue; + if (classesByName.size === 0) collect(sf); + const baseClass = classesByName.get(base); + const inherited = baseClass === undefined ? undefined : memberOf(baseClass); + if (inherited !== undefined) familyUnion(inherited, m); + } + for (const classes of cooccurring.get(name) ?? []) { + let first: ts.MethodDeclaration | undefined; + for (const c of classes) { + if (c.getSourceFile() !== sf) continue; + const m = memberOf(c); + if (m === undefined) continue; + if (first === undefined) first = m; + else familyUnion(first, m); + } + } + } // Members the program runs (they have analysis contexts); members it // never reaches would only widen the family's types. const reached = new Set(); @@ -444,7 +587,10 @@ export function planShapeOverlay( const familyOf = (fn: ts.FunctionLikeDeclaration): readonly ts.MethodDeclaration[] => { if (!ts.isMethodDeclaration(fn) || !ts.isIdentifier(fn.name)) return [fn as ts.MethodDeclaration]; - const members = (families.get(fn.name.text) ?? [fn]).filter((m) => reached.has(m)); + const root = familyFind(fn as ts.MethodDeclaration); + const members = (families.get(fn.name.text) ?? [fn]).filter( + (m) => reached.has(m) && familyFind(m) === root, + ); return members.length > 0 ? members : [fn]; }; const paramValues = (fn: ts.FunctionLikeDeclaration, index: number): Set => { @@ -658,6 +804,44 @@ export function planShapeOverlay( return { parts }; } +/** The plan moved onto texts that structural rewrites changed: each + * annotation and dead-branch edit maps through the rewrite's edits, and an + * annotation whose position the rewrite replaced is left out (with its + * closing parenthesis). `rewritten` gives a rewritten file's text and its + * position map. */ +export function planOnRewrittenTexts( + plan: ShapeOverlayPlan, + rewritten: ReadonlyMap number | null }>, +): ShapeOverlayPlan { + const parts = new Map(); + for (const [file, part] of plan.parts) { + const r = rewritten.get(file); + if (r === undefined) { + parts.set(file, part); + continue; + } + const lost = new Set(); + const moved: Edit[] = []; + for (const e of part.edits) { + const pos = r.map(e.pos); + const end = e.end === undefined ? undefined : r.map(e.end); + if (pos === null || end === null || (end !== undefined && end < pos)) { + if (e.id !== undefined) lost.add(e.id); + continue; + } + if (e.end !== undefined && end !== undefined && end - pos !== e.end - e.pos) { + // A rewrite inside a replaced dead branch. + if (e.id !== undefined) lost.add(e.id); + continue; + } + moved.push(end === undefined ? { ...e, pos } : { ...e, pos, end }); + } + const edits = moved.filter((e) => e.id === undefined || !lost.has(e.id)); + parts.set(file, { text: r.text, edits, suffix: part.suffix }); + } + return { parts }; +} + /** The overlay's files with the annotations a build dropped left out. */ export function renderShapeOverlay( plan: ShapeOverlayPlan, diff --git a/packages/compiler/src/frontend/shape-inference/static-objects.ts b/packages/compiler/src/frontend/shape-inference/static-objects.ts index 96f46c739..0a7c5abad 100644 --- a/packages/compiler/src/frontend/shape-inference/static-objects.ts +++ b/packages/compiler/src/frontend/shape-inference/static-objects.ts @@ -39,8 +39,13 @@ interface Edit { readonly text: string; } +export type RewriteEdit = Edit; + export interface StaticObjectRewrites { readonly files: Map; + /** Per rewritten file, the edits its text applied (in order; edits nested + * in a replacement are left out), in the text's original offsets. */ + readonly edits: Map; readonly classes: number; readonly functions: number; readonly directCalls: number; @@ -913,8 +918,53 @@ export function planStaticObjectRewrites( edit(sf, { start: at, end: at, text: props.length > 0 ? `, ${text}` : ` ${text} ` }); } const out = new Map(); - for (const [sf, edits] of editsByFile) out.set(sf.fileName, applyEdits(sf.text, edits)); - return { files: out, classes, functions, directCalls, tableDispatches, foldedReads, kept }; + const applied = new Map(); + for (const [sf, edits] of editsByFile) { + out.set(sf.fileName, applyEdits(sf.text, edits)); + applied.set(sf.fileName, appliedEdits(edits)); + } + return { + files: out, + edits: applied, + classes, + functions, + directCalls, + tableDispatches, + foldedReads, + kept, + }; +} + +/** The edits applyEdits applies, in order. */ +function appliedEdits(edits: readonly Edit[]): Edit[] { + const sorted = [...edits].sort((a, b) => a.start - b.start || b.end - a.end); + const out: Edit[] = []; + let at = 0; + for (const e of sorted) { + if (e.start < at) continue; + out.push(e); + at = e.end > e.start ? e.end : e.start; + } + return out; +} + +/** A position of a rewritten file's original text in its rewritten text, + * or null inside a replaced range or at an insertion (the text there + * changed). */ +export function rewrittenPosition(edits: readonly RewriteEdit[], pos: number): number | null { + let shift = 0; + for (const e of edits) { + if (e.start > pos) break; + if (e.end === e.start) { + if (e.start === pos) return null; + shift += e.text.length; + continue; + } + if (pos === e.start) break; + if (pos < e.end) return null; + shift += e.text.length - (e.end - e.start); + } + return pos + shift; } /** Applies replacements and insertions; an edit inside an earlier diff --git a/packages/compiler/src/frontend/ts7/checker.ts b/packages/compiler/src/frontend/ts7/checker.ts index eb2b9e20b..44839bc2f 100644 --- a/packages/compiler/src/frontend/ts7/checker.ts +++ b/packages/compiler/src/frontend/ts7/checker.ts @@ -345,6 +345,15 @@ export class CheckerFacade { this.prefetchSymbols(sf); } + /** The symbols at a file's identifiers only, for consumers that resolve + * names but never ask for types (shape inference). The file stays open + * to a later whole-file prefetch, which skips the answered nodes. */ + prefetchSourceFileSymbols(sf: SourceFile): void { + this.ensureActive(); + if (this.cache.prefetchedSymbols.has(sf)) return; + this.prefetchSymbolNodes(collectNodes([sf]), false, false); + } + /** Batches the non-body structure of many files as ONE logical wave. * Top-level executable statements, class field initializers/static blocks, * and every declaration header are included; function/method/constructor diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index c46d50374..6711039ca 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -1,4 +1,4 @@ -import { shapeOverlayEnabled } from "./frontend/shape-inference/index.js"; +import { setShapeOverlayMemoDir, shapeOverlayEnabled } from "./frontend/shape-inference/index.js"; import { optimizationField, type NativeOptimization } from "./backend/optimization.js"; import { targetCpuField, targetCpuTargetRefusal } from "./backend/target-cpu.js"; import { compilationTiming, type CompilationTiming, type TimingClock } from "./timing.js"; @@ -1006,6 +1006,16 @@ async function stageDriverBuildSources( * (`runFrontendProcess`). Node's heap keeps the pages a build's frontend used for the life of the * process, so running them in a process of their own leaves the native helper the memory they took. */ +/** The shape overlay's memo directory for the current executable build + * (frontend/shape-inference/memo.ts): per compiler, in the build cache. */ +let shapeOverlayMemo: string | null = null; + +function shapeOverlayMemoDir(cacheRoot: string | null, implementation: string): string | null { + return cacheRoot === null + ? null + : join(cacheRoot, "shape-overlay-v1", implementation.slice(0, 24)); +} + const frontendProcessEntry = fileURLToPath(new URL("./frontend-process.js", import.meta.url)); /** @@ -1085,6 +1095,8 @@ interface FrontendProcessRequest { timing: TimingClock; /** The compile's --provenance-sources mappings, which the frontend process installs. */ provenance: ProvenanceSources | null; + /** The shape overlay's memo directory (shapeOverlayMemoDir). */ + shapeOverlayMemo: string | null; } /** What the frontend process sends back. A thrown error comes back as `failed`. */ @@ -1138,6 +1150,7 @@ async function prepareExecutableInputInChild( buildPlatform, timing: timingClock, provenance: provenanceSources(), + shapeOverlayMemo, }; await writeFile(requestPath, JSON.stringify(request), { mode: 0o600 }); const exitCode = await frontendProcess.run(requestPath, resultPath); @@ -1205,6 +1218,8 @@ export async function runFrontendProcess(requestPath: string, resultPath: string process.env["SCRIPTC_SHAPE_OVERLAY"] = request.opts.shapeOverlay ? "1" : "0"; clearCompileSessionCaches(); setProvenanceSources(request.provenance); + shapeOverlayMemo = request.shapeOverlayMemo; + setShapeOverlayMemoDir(request.shapeOverlayMemo); // The compile that started this process prepared the cache root. configureLibDiagnostics( (request.opts.outputKind ?? "exe") === "exe" && provenanceSources() === null @@ -1274,6 +1289,8 @@ async function compileTracked( const timing = compilationTiming(); const timingStart = Date.now(); entryPath = resolve(entryPath); + shapeOverlayMemo = null; + setShapeOverlayMemoDir(null); const outputKind = opts.outputKind ?? "exe"; if ( (opts.backend !== undefined && opts.backend !== "llvm") || @@ -1574,6 +1591,8 @@ async function compileTracked( implementation: implementation.digest, implementationDependencies: implementation.dependencies, }; + shapeOverlayMemo = shapeOverlayMemoDir(cacheRoot, implementation.digest); + setShapeOverlayMemoDir(shapeOverlayMemo); } const earlyHit = earlyCacheOptions === null diff --git a/packages/compiler/test/shape-overlay-cost.test.ts b/packages/compiler/test/shape-overlay-cost.test.ts new file mode 100644 index 000000000..9a3229aeb --- /dev/null +++ b/packages/compiler/test/shape-overlay-cost.test.ts @@ -0,0 +1,127 @@ +/* The overlay's compile-cost pieces: the memo that carries a build's + * verdicts to the next build of the same JavaScript (memo.ts), and method + * families that join only the methods a call can reach together + * (overlay.ts), so a call on one class keeps that class's result type. */ +import { mkdir, mkdtemp, readdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, expect, test, vi } from "vitest"; +import { compile } from "../src/index.js"; +import { + overlayMemoKey, + readOverlayMemo, + setShapeOverlayMemoDir, + writeOverlayMemo, +} from "../src/frontend/shape-inference/memo.js"; + +const packageSource = `export class Str { + constructor(checks) { this.checks = checks; } + _addCheck(c) { return new Str([...this.checks, c]); } + min(n) { return this._addCheck({ kind: "min", n }); } + max(n) { return this._addCheck({ kind: "max", n }); } +} +export class Num { + constructor(checks) { this.checks = checks; } + _addCheck(c) { return new Num([...this.checks, c]); } + int() { return this._addCheck({ kind: "int" }); } +} +export const str = () => new Str([]); +export const num = () => new Num([]); +`; + +let dir = ""; +const saved: Record = {}; +beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), "scriptc-overlay-cost-")); + const pkg = join(dir, "node_modules/families-fixture"); + await mkdir(pkg, { recursive: true }); + await writeFile( + join(pkg, "package.json"), + JSON.stringify({ + name: "families-fixture", + version: "1.0.0", + type: "module", + main: "index.js", + }), + ); + await writeFile(join(pkg, "index.js"), packageSource); + await writeFile( + join(dir, "package.json"), + JSON.stringify({ type: "module", dependencies: { "families-fixture": "1.0.0" } }), + ); + await writeFile( + join(dir, "main.ts"), + `import { str, num } from "families-fixture";\n` + + `console.log(str().min(1).max(2).checks.length, num().int().checks.length);\n`, + ); + for (const k of ["SCRIPTC_SHAPE_OVERLAY", "SCRIPTC_DEBUG_SHAPE_OVERLAY", "SCRIPTC_NO_CACHE"]) + saved[k] = process.env[k]; +}); +afterEach(async () => { + for (const [k, v] of Object.entries(saved)) { + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + setShapeOverlayMemoDir(null); + vi.restoreAllMocks(); + await rm(dir, { recursive: true, force: true }); +}); + +test("same-named methods no call reaches together keep their own result types", async () => { + process.env["SCRIPTC_SHAPE_OVERLAY"] = "1"; + process.env["SCRIPTC_DEBUG_SHAPE_OVERLAY"] = "1"; + process.env["SCRIPTC_NO_CACHE"] = "1"; + let text = ""; + vi.spyOn(process.stderr, "write").mockImplementation((chunk: string | Uint8Array) => { + text += typeof chunk === "string" ? chunk : Buffer.from(chunk).toString("utf8"); + return true; + }); + const result = await compile(join(dir, "main.ts"), { + outDir: dir, + outPath: join(dir, "main.ir.json"), + outputKind: "ir", + npmStatic: ["families-fixture"], + }); + vi.restoreAllMocks(); + if (!result.ok) throw new Error(result.diagnostics.map((d) => d.message).join("\n")); + expect(text).toMatch(/@returns \{Str\}[^\n]*\*\/ _addCheck\(c\) \{ return new Str/); + expect(text).toMatch(/@returns \{Num\}[^\n]*\*\/ _addCheck\(c\) \{ return new Num/); + expect(text).not.toContain("Str | Num"); +}); + +test("a memo round-trips and is keyed by every JavaScript file's text", async () => { + const memoDir = join(dir, "memo"); + setShapeOverlayMemoDir(memoDir); + const key = overlayMemoKey([ + ["/p/a.js", "a"], + ["/p/b.js", "b"], + ]); + expect( + overlayMemoKey([ + ["/p/b.js", "b"], + ["/p/a.js", "a"], + ]), + ).toBe(key); + expect( + overlayMemoKey([ + ["/p/a.js", "a"], + ["/p/b.js", "b2"], + ]), + ).not.toBe(key); + expect(readOverlayMemo(key)).toBeNull(); + const memo = { + off: false, + annotations: [{ file: "/p/a.js", hash: "h", ids: [3, 7] }], + files: [], + plainFences: ["SC2002|/p/a.js|4"], + }; + writeOverlayMemo(key, memo); + expect(readOverlayMemo(key)).toEqual(memo); + expect((await readdir(memoDir)).filter((f) => f.endsWith(".tmp"))).toEqual([]); + // A malformed memo reads as none. + await writeFile(join(memoDir, `${key}.json`), `{"off": 1}`); + expect(readOverlayMemo(key)).toBeNull(); + // Without a memo directory (SCRIPTC_NO_CACHE) nothing is read or written. + setShapeOverlayMemoDir(null); + expect(readOverlayMemo(key)).toBeNull(); +}); diff --git a/packages/compiler/test/shape-overlay-patches.test.ts b/packages/compiler/test/shape-overlay-patches.test.ts new file mode 100644 index 000000000..bbd6c24eb --- /dev/null +++ b/packages/compiler/test/shape-overlay-patches.test.ts @@ -0,0 +1,86 @@ +/* A declaration a project type patch edited keeps the patch alone: the + * shape overlay writes no annotation on it (overlay.ts + * setShapeOverlayProtectedSpans; the type-patches layer supplies the + * spans). Other declarations of the same file are typed as usual. */ +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, expect, test, vi } from "vitest"; +import { compile } from "../src/index.js"; +import { setShapeOverlayProtectedSpans } from "../src/frontend/shape-inference/index.js"; + +const patchedDoc = "/** @param {number} n patched by the project */"; +const packageSource = `export function twice(n) { return n * 2; } +${patchedDoc} +export function thrice(n) { return n * 3; } +`; + +let dir = ""; +const saved: Record = {}; +beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), "scriptc-overlay-patch-")); + const pkg = join(dir, "node_modules/patched-fixture"); + await mkdir(pkg, { recursive: true }); + await writeFile( + join(pkg, "package.json"), + JSON.stringify({ name: "patched-fixture", version: "1.0.0", type: "module", main: "index.js" }), + ); + await writeFile(join(pkg, "index.js"), packageSource); + await writeFile( + join(dir, "package.json"), + JSON.stringify({ type: "module", dependencies: { "patched-fixture": "1.0.0" } }), + ); + await writeFile( + join(dir, "main.ts"), + `import { twice, thrice } from "patched-fixture";\nconsole.log(twice(2), thrice(3));\n`, + ); + for (const k of ["SCRIPTC_SHAPE_OVERLAY", "SCRIPTC_DEBUG_SHAPE_OVERLAY", "SCRIPTC_NO_CACHE"]) + saved[k] = process.env[k]; +}); +afterEach(async () => { + for (const [k, v] of Object.entries(saved)) { + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + setShapeOverlayProtectedSpans(null); + vi.restoreAllMocks(); + await rm(dir, { recursive: true, force: true }); +}); + +async function overlayText(): Promise { + process.env["SCRIPTC_SHAPE_OVERLAY"] = "1"; + process.env["SCRIPTC_DEBUG_SHAPE_OVERLAY"] = "1"; + process.env["SCRIPTC_NO_CACHE"] = "1"; + let text = ""; + vi.spyOn(process.stderr, "write").mockImplementation((chunk: string | Uint8Array) => { + text += typeof chunk === "string" ? chunk : Buffer.from(chunk).toString("utf8"); + return true; + }); + const result = await compile(join(dir, "main.ts"), { + outDir: dir, + outPath: join(dir, "main.ir.json"), + outputKind: "ir", + npmStatic: ["patched-fixture"], + }); + vi.restoreAllMocks(); + if (!result.ok) throw new Error(result.diagnostics.map((d) => d.message).join("\n")); + const start = text.indexOf("[shape-overlay] "); + return start < 0 ? "" : text.slice(start); +} + +test("the overlay types both functions without a patch", async () => { + const text = await overlayText(); + expect(text).toMatch(/@scriptcShape #\d+ \*\/ export function twice/); + expect(text).toMatch(/@scriptcShape #\d+ \*\/ export function thrice/); +}); + +test("a patched declaration gets no overlay annotation", async () => { + const start = packageSource.indexOf(patchedDoc); + setShapeOverlayProtectedSpans((file) => + file.endsWith("patched-fixture/index.js") ? [{ start, end: start + patchedDoc.length }] : [], + ); + const text = await overlayText(); + expect(text).toMatch(/@scriptcShape #\d+ \*\/ export function twice/); + expect(text).toContain(`${patchedDoc}\nexport function thrice`); + expect(text).not.toMatch(/@scriptcShape #\d+ \*\/ export function thrice/); +}); diff --git a/packages/compiler/test/shape-overlay-render.test.ts b/packages/compiler/test/shape-overlay-render.test.ts index b0758c14f..b1982311b 100644 --- a/packages/compiler/test/shape-overlay-render.test.ts +++ b/packages/compiler/test/shape-overlay-render.test.ts @@ -1,7 +1,12 @@ /* renderShapeOverlay keeps the package's JSDoc as written (package JSDoc is * a declaration) and writes the annotations a build keeps. */ import { expect, test } from "vitest"; -import { OVERLAY_MARK, renderShapeOverlay } from "../src/frontend/shape-inference/overlay.js"; +import { + OVERLAY_MARK, + planOnRewrittenTexts, + renderShapeOverlay, +} from "../src/frontend/shape-inference/overlay.js"; +import { rewrittenPosition } from "../src/frontend/shape-inference/static-objects.js"; import type { ShapeOverlayPlan } from "../src/frontend/shape-inference/overlay.js"; test("package JSDoc stays as written; dropped annotations are left out", () => { @@ -32,3 +37,39 @@ test("package JSDoc stays as written; dropped annotations are left out", () => { expect(out).toContain(`#${at[2]} */ function g(c) {}`); expect(overlay.annotations).toBe(2); }); + +test("a plan moves onto a text whose constant reads were replaced", () => { + // `NS.K` (6..10) folds to `1`; the annotation on `g` follows the shift, + // and an annotation inside the folded read is left out with its closer. + const text = "f(a, NS.K);\nfunction g(c) {}\n"; + const fold = { start: text.indexOf("NS.K"), end: text.indexOf("NS.K") + 4, text: "1" }; + const rewrittenText = text.replace("NS.K", "1"); + const g = text.indexOf("function g"); + const inside = fold.start + 3; + expect(rewrittenPosition([fold], g)).toBe(g - 3); + expect(rewrittenPosition([fold], fold.start)).toBe(fold.start); + expect(rewrittenPosition([fold], inside)).toBeNull(); + expect(rewrittenPosition([{ start: 0, end: 0, text: "x" }], 0)).toBeNull(); + const plan: ShapeOverlayPlan = { + parts: new Map([ + [ + "a.js", + { + text, + edits: [ + { pos: g, id: g, text: `/** @param {number} c ${OVERLAY_MARK} #${g} */ ` }, + { pos: inside, id: inside, text: `/** @type {X} ${OVERLAY_MARK} #${inside} */ (` }, + { pos: fold.end, id: inside, text: ")" }, + ], + suffix: "", + }, + ], + ]), + }; + const moved = planOnRewrittenTexts( + plan, + new Map([["a.js", { text: rewrittenText, map: (p: number) => rewrittenPosition([fold], p) }]]), + ); + const out = renderShapeOverlay(moved).files.get("a.js")!; + expect(out).toBe(`f(a, 1);\n/** @param {number} c ${OVERLAY_MARK} #${g} */ function g(c) {}\n`); +}); From 59d3dfdbfcf0c1a26a1b4b57ef5b6d9aa2544c0d Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Sun, 25 Oct 2026 12:13:11 -0700 Subject: [PATCH 05/21] Keep the compiler self-hosting: a boolean predicate in the concat helper The self-hosted compiler refused the truthiness test on an IrType | null | "dyn" value (SC2003, no layout for the union result). Co-Authored-By: Claude Opus 5.5 --- .../compiler/src/frontend/lowering/containers/array-concat.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/compiler/src/frontend/lowering/containers/array-concat.ts b/packages/compiler/src/frontend/lowering/containers/array-concat.ts index e3164689e..c52e9ccc8 100644 --- a/packages/compiler/src/frontend/lowering/containers/array-concat.ts +++ b/packages/compiler/src/frontend/lowering/containers/array-concat.ts @@ -206,7 +206,7 @@ export function arrayConcatHelper( ), { kind: "return", value: out, loc }, ]; - if (shape.some((type) => type && type !== "dyn" && !typeEquals(type, elem))) { + if (shape.some((type) => type !== null && type !== "dyn" && !typeEquals(type, elem))) { for (const id of ["i.0", "offset.0", "count.0"]) locals.push({ id, name: id, type: F64, mutable: true }); } From 3c95271fc1987347de78d3067fe28a048ee29205 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Sun, 25 Oct 2026 12:30:00 -0700 Subject: [PATCH 06/21] Record the TypeScript 7 order baselines of the stack's npm cases static-objects, static-tables, concat-dynamic-args, evolving-let-index and the three shape-* cases landed in this stack without preflight/order baselines, so the TypeScript 7 parity job failed on every PR above them. Co-Authored-By: Claude Opus 5.5 --- .../fixtures/npm/cases/concat-dynamic-args/main.ts.json | 6 ++++++ .../fixtures/npm/cases/evolving-let-index/main.ts.json | 6 ++++++ .../fixtures/npm/cases/shape-absent-reads/main.ts.json | 6 ++++++ .../fixtures/npm/cases/shape-bigint-arith/main.ts.json | 6 ++++++ .../fixtures/npm/cases/shape-module-identity/main.ts.json | 6 ++++++ .../tests/fixtures/npm/cases/static-objects/main.ts.json | 6 ++++++ .../tests/fixtures/npm/cases/static-tables/main.ts.json | 6 ++++++ 7 files changed, 42 insertions(+) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/concat-dynamic-args/main.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/evolving-let-index/main.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/shape-absent-reads/main.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/shape-bigint-arith/main.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/shape-module-identity/main.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/static-objects/main.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/static-tables/main.ts.json diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/concat-dynamic-args/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/concat-dynamic-args/main.ts.json new file mode 100644 index 000000000..0fbf33bad --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/concat-dynamic-args/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/concat-dynamic-args/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/evolving-let-index/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/evolving-let-index/main.ts.json new file mode 100644 index 000000000..8c503a074 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/evolving-let-index/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/evolving-let-index/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/shape-absent-reads/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/shape-absent-reads/main.ts.json new file mode 100644 index 000000000..9a8706ad0 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/shape-absent-reads/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/shape-absent-reads/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/shape-bigint-arith/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/shape-bigint-arith/main.ts.json new file mode 100644 index 000000000..5df93d3c8 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/shape-bigint-arith/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/shape-bigint-arith/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/shape-module-identity/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/shape-module-identity/main.ts.json new file mode 100644 index 000000000..1940110e8 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/shape-module-identity/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/shape-module-identity/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/static-objects/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/static-objects/main.ts.json new file mode 100644 index 000000000..69441c729 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/static-objects/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/static-objects/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/static-tables/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/static-tables/main.ts.json new file mode 100644 index 000000000..7c777f751 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/static-tables/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/static-tables/main.ts" + ], + "diags": [] +} From b3d4a3f3410f137b488e263abb1c90680ec8c996 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Sun, 25 Oct 2026 13:01:49 -0700 Subject: [PATCH 07/21] Fix the stack's failing unit and snapshot tests - analyze.ts keeps argsFrom in a Map the analyzer owns (compiler sources keep caches on their owners, not in weak collections). - constant-graphs.test.ts: lib.math is stored after lib was created, so the report shows it as undefined|{...} with a graph of its own. - maybe-promise-fences.ts: `record instanceof Promise` lowers to false since the base's neverPromise lowering; the snapshot no longer lists the refusal. Co-Authored-By: Claude Opus 5.5 --- .../compiler/src/frontend/shape-inference/analyze.ts | 2 +- packages/compiler/test/constant-graphs.test.ts | 5 ++++- tests/harness/__snapshots__/maybe-promise-fences.ts.txt | 9 +-------- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/packages/compiler/src/frontend/shape-inference/analyze.ts b/packages/compiler/src/frontend/shape-inference/analyze.ts index ac24ca562..550f9dbc0 100644 --- a/packages/compiler/src/frontend/shape-inference/analyze.ts +++ b/packages/compiler/src/frontend/shape-inference/analyze.ts @@ -2907,7 +2907,7 @@ class Analyzer { * is in a loop, in the context the call was walked in. siteCtx keeps * the first context per syntax node, which a site walked in several * contexts (receiver classes, per-site clones) does not determine. */ - private readonly argsFrom = new WeakMap< + private readonly argsFrom = new Map< number[], { caller: FunctionInfo | null; inLoop: boolean } >(); diff --git a/packages/compiler/test/constant-graphs.test.ts b/packages/compiler/test/constant-graphs.test.ts index e3238ab57..e197359ad 100644 --- a/packages/compiler/test/constant-graphs.test.ts +++ b/packages/compiler/test/constant-graphs.test.ts @@ -89,9 +89,12 @@ test("constant graphs: namespaces, revealing modules and prototypes, with near m const text = lines.join("\n"); // The namespace graph: guarded declarations, the revealing module's // table, and the constructor function. + // `lib.math` is stored after `lib` was created, so a read of it may see + // undefined first; its object is a graph of its own. expect(text).toMatch( - /graph ns-fixture\/index\.js:\d+ \{lib: \{math: \{TAU: number, half: fn\}, text: \{upper: fn\}/, + /graph ns-fixture\/index\.js:\d+ \{lib: \{math: undefined\|\{\}@ns-fixture\/index\.js:\d+, text: \{upper: fn\}/, ); + expect(text).toMatch(/graph ns-fixture\/index\.js:\d+ \{TAU: number, half: fn\}/); // The constructor's prototype object, built by member stores at init. expect(text).toMatch(/graph ns-fixture\/index\.js:\d+ \{constructor: \{\}, norm: fn\}/); // A field written by a function called from a loop. diff --git a/tests/harness/__snapshots__/maybe-promise-fences.ts.txt b/tests/harness/__snapshots__/maybe-promise-fences.ts.txt index 159e88f76..ab325aaaa 100644 --- a/tests/harness/__snapshots__/maybe-promise-fences.ts.txt +++ b/tests/harness/__snapshots__/maybe-promise-fences.ts.txt @@ -48,11 +48,4 @@ maybe-promise-fences.ts:27:50 - error SC2020: 'Promise.resolve over 'Promise Date: Sun, 25 Oct 2026 13:10:08 -0700 Subject: [PATCH 08/21] Format analyze.ts Co-Authored-By: Claude Opus 5.5 --- packages/compiler/src/frontend/shape-inference/analyze.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/compiler/src/frontend/shape-inference/analyze.ts b/packages/compiler/src/frontend/shape-inference/analyze.ts index 550f9dbc0..557972868 100644 --- a/packages/compiler/src/frontend/shape-inference/analyze.ts +++ b/packages/compiler/src/frontend/shape-inference/analyze.ts @@ -2907,10 +2907,7 @@ class Analyzer { * is in a loop, in the context the call was walked in. siteCtx keeps * the first context per syntax node, which a site walked in several * contexts (receiver classes, per-site clones) does not determine. */ - private readonly argsFrom = new Map< - number[], - { caller: FunctionInfo | null; inLoop: boolean } - >(); + private readonly argsFrom = new Map(); call(e: ts.CallExpression): number { this.noteSite(e); From 484fc2cb13ad4bf75d3997e01c1dfe04c1b70828 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Sun, 25 Oct 2026 12:30:10 -0700 Subject: [PATCH 09/21] Fold members stored after their namespace was created; skip the receiver read in JavaScript functions that never use this - static-objects: a primitive member stored once by an unconditional top-level statement of the object's own file (app.math.EPSILON = 1e-9) reads as its literal where the read provably runs after the store: inside functions when the statements before the store run no code, and in later top-level statements of that file. SCRIPTC_SHAPE_FOLD_STORES=0 opts out. - JavaScript functions read the ambient receiver at entry only when this occurs in their parameters or body outside nested non-arrow functions. - npm case static-stores pins both, with the overlay on and off. nsbench (n=20000, overlay on): 64.4 -> 36 ms. Co-Authored-By: Claude Opus 5.5 --- .../src/frontend/lowering/lower-functions.ts | 28 ++++- .../shape-inference/static-objects.ts | 106 +++++++++++++++++- .../npm/cases/static-stores/main.ts.json | 6 + .../npm/cases/static-stores/js-main.cjs | 8 ++ .../fixtures/npm/cases/static-stores/main.ts | 9 ++ .../node_modules/static-stores/index.d.ts | 10 ++ .../node_modules/static-stores/index.js | 47 ++++++++ .../node_modules/static-stores/package.json | 1 + .../npm/cases/static-stores/package.json | 1 + tests/harness/static-stores.test.ts | 46 ++++++++ 10 files changed, 257 insertions(+), 5 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/static-stores/main.ts.json create mode 100644 tests/fixtures/npm/cases/static-stores/js-main.cjs create mode 100644 tests/fixtures/npm/cases/static-stores/main.ts create mode 100644 tests/fixtures/npm/cases/static-stores/node_modules/static-stores/index.d.ts create mode 100644 tests/fixtures/npm/cases/static-stores/node_modules/static-stores/index.js create mode 100644 tests/fixtures/npm/cases/static-stores/node_modules/static-stores/package.json create mode 100644 tests/fixtures/npm/cases/static-stores/package.json create mode 100644 tests/harness/static-stores.test.ts diff --git a/packages/compiler/src/frontend/lowering/lower-functions.ts b/packages/compiler/src/frontend/lowering/lower-functions.ts index c1cca9c64..7b5e9aa46 100644 --- a/packages/compiler/src/frontend/lowering/lower-functions.ts +++ b/packages/compiler/src/frontend/lowering/lower-functions.ts @@ -61,6 +61,29 @@ function declareCallReceiver( }; } +/** Whether a JavaScript function's own `this` can be read: a `this` in its + * parameters or body, outside nested non-arrow functions (which bind their + * own). Conservative inside classes and object literal methods. A function + * that never reads it skips the ambient receiver read at entry. */ +function readsOwnThis(node: ts.FunctionLikeDeclaration): boolean { + let found = false; + const visit = (n: ts.Node): void => { + if (found) return; + if (n.kind === ts.SyntaxKind.ThisKeyword) { + found = true; + return; + } + if (ts.isFunctionDeclaration(n) || ts.isFunctionExpression(n)) return; + ts.forEachChild(n, (c) => { + visit(c); + return undefined; + }); + }; + for (const p of node.parameters.nodes) visit(p); + if (node.body !== undefined) visit(node.body); + return found; +} + /** True when the identifier resolves (through import aliases) to a * top-level function declaration of ANY program file (not merely a * same-named local shadowing one). Functions declared directly in a @@ -335,7 +358,7 @@ export function lowerLambda( try { const receiver = (checkedReceiver || - isJsSourceFile(ts.getSourceFileOfNode(node)) || + (isJsSourceFile(ts.getSourceFileOfNode(node)) && readsOwnThis(node)) || node.parameters.nodes.some(isThisParameter)) && !ts.isArrowFunction(node) ? declareCallReceiver(lowerer, node, isGenerator) @@ -535,7 +558,8 @@ export function lowerFunction(lowerer: Lowerer, decl: ts.FunctionDeclaration): I lowerer.fnStack.push(ctx); try { const receiver = - decl.parameters.nodes.some(isThisParameter) || isJsSourceFile(ts.getSourceFileOfNode(decl)) + decl.parameters.nodes.some(isThisParameter) || + (isJsSourceFile(ts.getSourceFileOfNode(decl)) && readsOwnThis(decl)) ? declareCallReceiver(lowerer, decl, sig.generator !== undefined) : null; const { params, prologue } = lowerer.declareParams(decl.parameters.nodes, sig.params); diff --git a/packages/compiler/src/frontend/shape-inference/static-objects.ts b/packages/compiler/src/frontend/shape-inference/static-objects.ts index 0a7c5abad..a702e4f24 100644 --- a/packages/compiler/src/frontend/shape-inference/static-objects.ts +++ b/packages/compiler/src/frontend/shape-inference/static-objects.ts @@ -167,6 +167,45 @@ function ownLiteralMember(literal: Node, key: string): string | null { return found; } +/** Whether a top-level statement runs no code but its own: no call, `new`, + * tagged template, class (static blocks and field initializers run), getter + * or setter, `await`, `yield` or `delete`. Function bodies do not run. */ +function runsNoCode(stmt: ts.Statement): boolean { + if (ts.isFunctionDeclaration(stmt) || ts.isEmptyStatement(stmt)) return true; + if (!ts.isVariableStatement(stmt) && !ts.isExpressionStatement(stmt)) return false; + let quiet = true; + const visit = (n: Node): void => { + if (!quiet) return; + if (ts.isFunctionExpression(n) || ts.isArrowFunction(n)) return; + if ( + ts.isCallExpression(n) || + ts.isNewExpression(n) || + ts.isTaggedTemplateExpression(n) || + ts.isClassExpression(n) || + ts.isGetAccessorDeclaration(n) || + ts.isSetAccessorDeclaration(n) || + ts.isAwaitExpression(n) || + ts.isYieldExpression(n) || + ts.isDeleteExpression(n) || + ts.isMethodDeclaration(n) + ) { + quiet = false; + return; + } + ts.forEachChild(n, (c) => { + visit(c); + return undefined; + }); + }; + visit(stmt); + return quiet; +} + +function enclosingFunctionLike(n: Node): Node | null { + for (let p = n.parent; p !== undefined; p = p.parent) if (ts.isFunctionLike(p)) return p; + return null; +} + function primitiveLiteralText(e: ts.Expression): string | null { if ( ts.isNumericLiteral(e) || @@ -758,19 +797,80 @@ export function planStaticObjectRewrites( // the object does and nothing writes it, so every read sees it. (A // member stored after the object was created could be read before the // store; those stay reads.) + // + // A member stored once by an unconditional top-level statement of the + // object's own file (`app.math.EPSILON = 1e-9;`) folds too, at reads that + // provably run after the store: reads inside functions, when no code can + // run before the store (the statements before it call nothing, so no + // function runs and no other module starts), and top-level reads after + // it in the same file. + const writesOf = new Map>(); + for (const op of analysis.ops) { + if (op.kind !== "write" || op.name === null) continue; + for (const a of analysis.nodes[op.recv]!.atoms) { + let byKey = writesOf.get(a); + if (byKey === undefined) writesOf.set(a, (byKey = new Map())); + let l = byKey.get(op.name); + if (l === undefined) byKey.set(op.name, (l = [])); + if (!l.includes(op.node)) l.push(op.node); + } + } + const quietPrefix = new Map(); + const storedLiteral = ( + atom: Atom, + site: Node, + key: string, + ): { literal: string; stmt: ts.ExpressionStatement; statements: readonly Node[] } | null => { + const writes = writesOf.get(atom)?.get(key) ?? []; + if (writes.length !== 1) return null; + const target = writes[0]!; + const assign = target.parent; + if ( + assign === undefined || + !ts.isBinaryExpression(assign) || + assign.left !== target || + assign.operatorToken.kind !== ts.SyntaxKind.EqualsToken + ) + return null; + const stmt = assign.parent; + if (stmt === undefined || !ts.isExpressionStatement(stmt) || !ts.isSourceFile(stmt.parent)) + return null; + if (stmt.getSourceFile() !== site.getSourceFile()) return null; + const literal = primitiveLiteralText(assign.right); + if (literal === null) return null; + const statements = stmt.parent.statements.nodes; + let quiet = quietPrefix.get(stmt); + if (quiet === undefined) { + quiet = statements.slice(0, statements.indexOf(stmt)).every(runsNoCode); + quietPrefix.set(stmt, quiet); + } + return quiet ? { literal, stmt, statements } : null; + }; let foldedReads = 0; for (const { op, atoms } of mergedOps) { if (op.kind !== "read" || op.name === null) continue; if (atoms.size !== 1) continue; - const o = constantAtoms.get([...atoms][0]!); + const atom = [...atoms][0]!; + const o = constantAtoms.get(atom); if (o === undefined || o.site.kind !== "literal") continue; const field = o.fields.find((f) => f.key === op.name); if (field === undefined || field.stores !== 1) continue; - const literal = ownLiteralMember(o.site.node, op.name); - if (literal === null) continue; const node = op.node; if (!ts.isPropertyAccessExpression(node) && !ts.isElementAccessExpression(node)) continue; if (!isPlainRead(node) || topStatement(node) === null) continue; + let literal = ownLiteralMember(o.site.node, op.name); + if (literal === null && process.env["SCRIPTC_SHAPE_FOLD_STORES"] !== "0") { + const stored = storedLiteral(atom, o.site.node, op.name); + if (stored === null) continue; + const inFunction = enclosingFunctionLike(node) !== null; + const top = topStatement(node)!; + const after = + top.getSourceFile() === stored.stmt.getSourceFile() && + stored.statements.indexOf(top) > stored.statements.indexOf(stored.stmt); + if (!inFunction && !after) continue; + literal = stored.literal; + } + if (literal === null) continue; // An element key needs no parentheses (and a parenthesized literal key // would read as a computed one). const key = diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/static-stores/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/static-stores/main.ts.json new file mode 100644 index 000000000..10384a91b --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/static-stores/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/static-stores/main.ts" + ], + "diags": [] +} diff --git a/tests/fixtures/npm/cases/static-stores/js-main.cjs b/tests/fixtures/npm/cases/static-stores/js-main.cjs new file mode 100644 index 000000000..c844f5b74 --- /dev/null +++ b/tests/fixtures/npm/cases/static-stores/js-main.cjs @@ -0,0 +1,8 @@ +// The JavaScript consumer of static-stores (main.ts is the TypeScript one). +const { lib, other } = require("static-stores"); + +let near = 0; +for (let i = 0; i < 20; i++) if (lib.near((i - 10) / 10)) near++; +console.log(near, lib.name(), lib.early, lib.after); +console.log(lib.counter.inc(), lib.counter.inc(), lib.twice(21), lib.tagOf.call({ tag: "t" })); +console.log(other.seen, other.check()); diff --git a/tests/fixtures/npm/cases/static-stores/main.ts b/tests/fixtures/npm/cases/static-stores/main.ts new file mode 100644 index 000000000..ad0e3eefc --- /dev/null +++ b/tests/fixtures/npm/cases/static-stores/main.ts @@ -0,0 +1,9 @@ +// Primitive members stored after their namespace was created, and functions +// with and without a receiver (js-main.cjs is the JavaScript consumer). +import { lib, other } from "static-stores"; + +let near = 0; +for (let i = 0; i < 20; i++) if (lib.near((i - 10) / 10)) near++; +console.log(near, lib.name(), lib.early, lib.after); +console.log(lib.counter.inc(), lib.counter.inc(), lib.twice(21), lib.tagOf.call({ tag: "t" })); +console.log(other.seen, other.check()); diff --git a/tests/fixtures/npm/cases/static-stores/node_modules/static-stores/index.d.ts b/tests/fixtures/npm/cases/static-stores/node_modules/static-stores/index.d.ts new file mode 100644 index 000000000..4a98d27dd --- /dev/null +++ b/tests/fixtures/npm/cases/static-stores/node_modules/static-stores/index.d.ts @@ -0,0 +1,10 @@ +export declare const lib: { + early: number | undefined; + after: number; + near(x: number): boolean; + name(): string; + counter: { n: number; inc(): number }; + twice(a: number): number; + tagOf(this: { tag: string }): string; +}; +export declare const other: { seen: number; check(): boolean }; diff --git a/tests/fixtures/npm/cases/static-stores/node_modules/static-stores/index.js b/tests/fixtures/npm/cases/static-stores/node_modules/static-stores/index.js new file mode 100644 index 000000000..429116787 --- /dev/null +++ b/tests/fixtures/npm/cases/static-stores/node_modules/static-stores/index.js @@ -0,0 +1,47 @@ +// Primitive members stored after their namespace object was created. Reads +// that provably run after the store read the stored literal; a read that can +// run before it (top-level code before the store, or any read once code has +// run before the store) stays a read. +var lib = lib || {}; +lib.cfg = lib.cfg || {}; +lib.cfg.EPS = 0.5; +lib.cfg.NAME = "v1"; +lib.early = lib.cfg.LATE; +lib.cfg.LATE = 7; +lib.after = lib.cfg.LATE + 1; +lib.near = function (x) { + return Math.abs(x) <= lib.cfg.EPS; +}; +lib.name = function () { + return lib.cfg.NAME + ":" + lib.cfg.LATE; +}; +// Functions that read their own receiver, through an arrow, and not at all. +lib.counter = { + n: 0, + inc: function () { + this.n++; + return this.n; + }, +}; +lib.twice = function (a) { + return a * 2; +}; +lib.tagOf = function () { + const read = () => this.tag; + return read(); +}; + +// A store after a call: code ran before it, so no read folds. +var other = {}; +other.log = []; +other.note = function (s) { + other.log.push(s); + return other.log.length; +}; +other.seen = other.note("start"); +other.flag = true; +other.check = function () { + return other.flag; +}; + +module.exports = { lib, other }; diff --git a/tests/fixtures/npm/cases/static-stores/node_modules/static-stores/package.json b/tests/fixtures/npm/cases/static-stores/node_modules/static-stores/package.json new file mode 100644 index 000000000..2d202dde0 --- /dev/null +++ b/tests/fixtures/npm/cases/static-stores/node_modules/static-stores/package.json @@ -0,0 +1 @@ +{ "name": "static-stores", "version": "1.0.0", "main": "index.js", "types": "index.d.ts" } diff --git a/tests/fixtures/npm/cases/static-stores/package.json b/tests/fixtures/npm/cases/static-stores/package.json new file mode 100644 index 000000000..38edc4090 --- /dev/null +++ b/tests/fixtures/npm/cases/static-stores/package.json @@ -0,0 +1 @@ +{ "name": "static-stores-case", "private": true, "type": "module" } diff --git a/tests/harness/static-stores.test.ts b/tests/harness/static-stores.test.ts new file mode 100644 index 000000000..050905371 --- /dev/null +++ b/tests/harness/static-stores.test.ts @@ -0,0 +1,46 @@ +/* Static-object specialization (shape-inference/static-objects.ts): with the + * shape overlay on, the npm case static-stores (through its JavaScript + * consumer, js-main.cjs) reads primitive members stored after their + * namespace object was created as literals where the read provably runs + * after the store, keeps the reads that can run before it, and prints what + * Node prints. Its functions with and without a receiver read `this` as + * Node does. The npm suites run the TypeScript consumer with and without + * the overlay. */ +import { execFile } from "node:child_process"; +import { mkdirSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { promisify } from "node:util"; +import { expect, test } from "vitest"; +import { compile } from "@scriptc/compiler"; + +const execFileAsync = promisify(execFile); +const repoRoot = join(import.meta.dirname, "../.."); +const entry = join(repoRoot, "tests/fixtures/npm/cases/static-stores/js-main.cjs"); +const outDir = join(repoRoot, "node_modules/.cache/scriptc-tests/static-stores"); +const sanitize = process.env["SCRIPTC_SAN"] === "1"; + +test( + "members stored after their namespace was created read as literals after the store", + { timeout: 600_000 }, + async () => { + mkdirSync(outDir, { recursive: true }); + const result = await compile(entry, { + outPath: join(outDir, "main"), + outDir, + sanitize, + shapeOverlay: true, + }); + if (!result.ok) + throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); + const report = (result.warnings ?? []).find((w) => w.code === "SC6103"); + expect(report?.message).toMatch(/, 5 constant reads\)/); + const pending = execFileAsync(result.binaryPath, [], { cwd: dirname(entry), encoding: "utf8" }); + pending.child.stdin?.end(); + const { stdout } = await pending; + const node = await execFileAsync(process.execPath, [entry], { + cwd: dirname(entry), + encoding: "utf8", + }); + expect(stdout).toBe(node.stdout); + }, +); From e31e2ebe759a106d62f9cb735920d89d7ec1cd72 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Sun, 25 Oct 2026 13:10:25 -0700 Subject: [PATCH 10/21] Dispatch a constant function table without a fallback when every caller passes one of its keys When the key is a parameter of a function that never escapes, every call passes a string literal key of the table, and the body neither assigns the parameter nor uses arguments, the dispatch needs no arm for other keys. In nsbench the fallback boxed the Vec probe into a dynamic call, so every Vec carried the property bag and lived on the heap; without it the workload inlines into one loop. SCRIPTC_SHAPE_EXHAUSTIVE_TABLES=0 opts out. nsbench (n=20000, overlay on): 36 -> 8.2 ms (Node about 21 ms). Co-Authored-By: Claude Opus 5.5 --- .../shape-inference/static-objects.ts | 82 ++++++++++++++++++- .../npm/cases/static-stores/js-main.cjs | 2 + .../fixtures/npm/cases/static-stores/main.ts | 2 + .../node_modules/static-stores/index.d.ts | 2 + .../node_modules/static-stores/index.js | 17 ++++ tests/harness/static-stores.test.ts | 2 +- 6 files changed, 102 insertions(+), 5 deletions(-) diff --git a/packages/compiler/src/frontend/shape-inference/static-objects.ts b/packages/compiler/src/frontend/shape-inference/static-objects.ts index a702e4f24..ab7c8e5fc 100644 --- a/packages/compiler/src/frontend/shape-inference/static-objects.ts +++ b/packages/compiler/src/frontend/shape-inference/static-objects.ts @@ -913,7 +913,63 @@ export function planStaticObjectRewrites( // key reads the table as written. The members do not use `this`, the // table has no accessors, and comparing a string runs no code, so only // the representation of the call changes. - const tableCalls: { sf: ts.SourceFile; call: ts.CallExpression; arms: [string, string][] }[] = []; + const tableCalls: { + sf: ts.SourceFile; + call: ts.CallExpression; + arms: [string, string][]; + exhaustive: boolean; + }[] = []; + // Whether a key parameter only ever holds one of the table's keys: its + // function never escapes, every call of it passes a string literal key + // of the table, and its body neither assigns the parameter nor uses + // `arguments` (which aliases parameters in sloppy code). The dispatch + // then needs no arm for other keys. + const literalKeysOnly = (keyDecl: Node, keys: ReadonlySet): boolean => { + if (process.env["SCRIPTC_SHAPE_EXHAUSTIVE_TABLES"] === "0") return false; + if (!ts.isParameter(keyDecl) || !ts.isIdentifier(keyDecl.name)) return false; + const fn = keyDecl.parent; + if (fn === undefined || !ts.isFunctionLike(fn) || escapedFns.has(fn)) return false; + for (const a of analysis.atoms.atoms) + if (a.kind === "function" && a.node === fn && analysis.escaped.has(a)) return false; + const index = (fn as ts.FunctionLikeDeclaration).parameters.nodes.indexOf(keyDecl); + const name = keyDecl.name.text; + let touched = false; + const visit = (n: Node): void => { + if (touched) return; + if (ts.isIdentifier(n) && n.text === "arguments") touched = true; + else if ( + ts.isBinaryExpression(n) && + n.operatorToken.kind >= ts.SyntaxKind.FirstAssignment && + n.operatorToken.kind <= ts.SyntaxKind.LastAssignment && + ts.isIdentifier(n.left) && + n.left.text === name + ) + touched = true; + else if ( + (ts.isPrefixUnaryExpression(n) || ts.isPostfixUnaryExpression(n)) && + ts.isIdentifier(n.operand) && + n.operand.text === name + ) + touched = true; + ts.forEachChild(n, (c) => { + visit(c); + return undefined; + }); + }; + const body = (fn as { body?: Node }).body; + if (body === undefined) return false; + visit(body); + if (touched) return false; + const edges = analysis.calls.filter((c) => c.callee.node === fn); + if (edges.length === 0) return false; + return edges.every((e) => { + if (!ts.isCallExpression(e.site)) return false; + const args = e.site.arguments.nodes; + if (args.slice(0, index + 1).some((a) => ts.isSpreadElement(a))) return false; + const arg = args[index]; + return arg !== undefined && ts.isStringLiteral(arg) && keys.has(arg.text); + }); + }; for (const { op, atoms } of mergedOps) { if (op.kind !== "computedRead") continue; const node = op.node; @@ -954,16 +1010,34 @@ export function planStaticObjectRewrites( kept.push(`${where}: a member is not a hoisted function of this file`); continue; } - tableCalls.push({ sf: call.getSourceFile(), call, arms }); + tableCalls.push({ + sf: call.getSourceFile(), + call, + arms, + exhaustive: literalKeysOnly(keyDecl, new Set(arms.map(([k]) => k))), + }); } - for (const { sf, call, arms } of tableCalls) { + for (const { sf, call, arms, exhaustive } of tableCalls) { const node = call.expression as ts.ElementAccessExpression; const key = node.argumentExpression.getText(); const args = call.arguments.nodes.map((a) => sliceWithCalls(sf, a)).join(", "); const table = sliceWithCalls(sf, node.expression); - const text = arms + const tested = exhaustive ? arms.slice(0, -1) : arms; + const text = tested .map(([k, name]) => `${key} === ${JSON.stringify(k)} ? ${name}(${args}) : `) .join(""); + if (exhaustive) { + callEdits.push({ + sf, + edit: { + start: ts.tokenStartOfNode(call), + end: call.end, + text: `(${text}${arms[arms.length - 1]![1]}(${args}))`, + }, + }); + tableDispatches++; + continue; + } callEdits.push({ sf, edit: { diff --git a/tests/fixtures/npm/cases/static-stores/js-main.cjs b/tests/fixtures/npm/cases/static-stores/js-main.cjs index c844f5b74..9925086f2 100644 --- a/tests/fixtures/npm/cases/static-stores/js-main.cjs +++ b/tests/fixtures/npm/cases/static-stores/js-main.cjs @@ -5,4 +5,6 @@ let near = 0; for (let i = 0; i < 20; i++) if (lib.near((i - 10) / 10)) near++; console.log(near, lib.name(), lib.early, lib.after); console.log(lib.counter.inc(), lib.counter.inc(), lib.twice(21), lib.tagOf.call({ tag: "t" })); +const key = process.argv.length > 99 ? "dbl" : "inc"; +console.log(lib.apply("inc", 1), lib.apply("dbl", 4), lib.applyAny("dbl", 5), lib.applyAny(key, 3)); console.log(other.seen, other.check()); diff --git a/tests/fixtures/npm/cases/static-stores/main.ts b/tests/fixtures/npm/cases/static-stores/main.ts index ad0e3eefc..ee05b1adf 100644 --- a/tests/fixtures/npm/cases/static-stores/main.ts +++ b/tests/fixtures/npm/cases/static-stores/main.ts @@ -6,4 +6,6 @@ let near = 0; for (let i = 0; i < 20; i++) if (lib.near((i - 10) / 10)) near++; console.log(near, lib.name(), lib.early, lib.after); console.log(lib.counter.inc(), lib.counter.inc(), lib.twice(21), lib.tagOf.call({ tag: "t" })); +const key = process.argv.length > 99 ? "dbl" : "inc"; +console.log(lib.apply("inc", 1), lib.apply("dbl", 4), lib.applyAny("dbl", 5), lib.applyAny(key, 3)); console.log(other.seen, other.check()); diff --git a/tests/fixtures/npm/cases/static-stores/node_modules/static-stores/index.d.ts b/tests/fixtures/npm/cases/static-stores/node_modules/static-stores/index.d.ts index 4a98d27dd..88a4d6aa8 100644 --- a/tests/fixtures/npm/cases/static-stores/node_modules/static-stores/index.d.ts +++ b/tests/fixtures/npm/cases/static-stores/node_modules/static-stores/index.d.ts @@ -6,5 +6,7 @@ export declare const lib: { counter: { n: number; inc(): number }; twice(a: number): number; tagOf(this: { tag: string }): string; + apply(op: "inc" | "dbl", v: number): number; + applyAny(op: string, v: number): number | string; }; export declare const other: { seen: number; check(): boolean }; diff --git a/tests/fixtures/npm/cases/static-stores/node_modules/static-stores/index.js b/tests/fixtures/npm/cases/static-stores/node_modules/static-stores/index.js index 429116787..8e65f2fc7 100644 --- a/tests/fixtures/npm/cases/static-stores/node_modules/static-stores/index.js +++ b/tests/fixtures/npm/cases/static-stores/node_modules/static-stores/index.js @@ -30,6 +30,23 @@ lib.tagOf = function () { const read = () => this.tag; return read(); }; +// A constant function table: `apply` is only ever called with literal keys +// of the table, so its dispatch needs no arm for other keys; `applyAny` is +// called with a computed key and keeps reading the table for other keys. +lib.ops = { + inc: function (v) { + return v + 1; + }, + dbl: function (v) { + return v * 2; + }, +}; +lib.apply = function (op, v) { + return lib.ops[op](v); +}; +lib.applyAny = function (op, v) { + return lib.ops[op](v); +}; // A store after a call: code ran before it, so no read folds. var other = {}; diff --git a/tests/harness/static-stores.test.ts b/tests/harness/static-stores.test.ts index 050905371..303237392 100644 --- a/tests/harness/static-stores.test.ts +++ b/tests/harness/static-stores.test.ts @@ -33,7 +33,7 @@ test( if (!result.ok) throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); const report = (result.warnings ?? []).find((w) => w.code === "SC6103"); - expect(report?.message).toMatch(/, 5 constant reads\)/); + expect(report?.message).toMatch(/2 function table calls, 5 constant reads\)/); const pending = execFileAsync(result.binaryPath, [], { cwd: dirname(entry), encoding: "utf8" }); pending.child.stdin?.end(); const { stdout } = await pending; From 8b38aa5d8f45a580a4d93da0ba69e970e9b1d40f Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Sun, 25 Oct 2026 13:44:32 -0700 Subject: [PATCH 11/21] Merge perf/shape-overlay-cost; keep the compiler self-hosting; record the new npm cases' order baselines - array-concat.ts: a boolean predicate (the self-hosted compiler refused the truthiness test on an IrType | null | "dyn" value, SC2003). - TypeScript 7 order baselines for concat-dynamic-args and evolving-let-index. Co-Authored-By: Claude Opus 5.5 --- .../src/frontend/lowering/containers/array-concat.ts | 2 +- .../fixtures/npm/cases/concat-dynamic-args/main.ts.json | 6 ++++++ .../fixtures/npm/cases/evolving-let-index/main.ts.json | 6 ++++++ 3 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/concat-dynamic-args/main.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/evolving-let-index/main.ts.json diff --git a/packages/compiler/src/frontend/lowering/containers/array-concat.ts b/packages/compiler/src/frontend/lowering/containers/array-concat.ts index e3164689e..c52e9ccc8 100644 --- a/packages/compiler/src/frontend/lowering/containers/array-concat.ts +++ b/packages/compiler/src/frontend/lowering/containers/array-concat.ts @@ -206,7 +206,7 @@ export function arrayConcatHelper( ), { kind: "return", value: out, loc }, ]; - if (shape.some((type) => type && type !== "dyn" && !typeEquals(type, elem))) { + if (shape.some((type) => type !== null && type !== "dyn" && !typeEquals(type, elem))) { for (const id of ["i.0", "offset.0", "count.0"]) locals.push({ id, name: id, type: F64, mutable: true }); } diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/concat-dynamic-args/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/concat-dynamic-args/main.ts.json new file mode 100644 index 000000000..0fbf33bad --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/concat-dynamic-args/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/concat-dynamic-args/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/evolving-let-index/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/evolving-let-index/main.ts.json new file mode 100644 index 000000000..8c503a074 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/evolving-let-index/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/evolving-let-index/main.ts" + ], + "diags": [] +} From 6f1efbf42b32f005a18815cdda6e52af11b9697e Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 15:56:53 -0500 Subject: [PATCH 12/21] Remove the committed node_modules symlink --- .gitignore | 2 ++ packages/compiler/node_modules | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) delete mode 120000 packages/compiler/node_modules diff --git a/.gitignore b/.gitignore index 81b92ddd5..88dfdc35b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ node_modules/ +# A symlinked node_modules is a file to git, which node_modules/ does not match. +/packages/*/node_modules .pnpm-store/ # test-fixture packages are TEST DATA, committed on purpose !tests/diagnostics/npm-imports/node_modules/ diff --git a/packages/compiler/node_modules b/packages/compiler/node_modules deleted file mode 120000 index 6b01b0eda..000000000 --- a/packages/compiler/node_modules +++ /dev/null @@ -1 +0,0 @@ -../../../shape-reg/packages/compiler/node_modules \ No newline at end of file From 8a87cf015c77015a6e7c60747b1ddcae41d13fd5 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 15:58:45 -0500 Subject: [PATCH 13/21] Remove the committed node_modules symlink --- .gitignore | 2 ++ packages/compiler/node_modules | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) delete mode 120000 packages/compiler/node_modules diff --git a/.gitignore b/.gitignore index 81b92ddd5..88dfdc35b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ node_modules/ +# A symlinked node_modules is a file to git, which node_modules/ does not match. +/packages/*/node_modules .pnpm-store/ # test-fixture packages are TEST DATA, committed on purpose !tests/diagnostics/npm-imports/node_modules/ diff --git a/packages/compiler/node_modules b/packages/compiler/node_modules deleted file mode 120000 index 6b01b0eda..000000000 --- a/packages/compiler/node_modules +++ /dev/null @@ -1 +0,0 @@ -../../../shape-reg/packages/compiler/node_modules \ No newline at end of file From bcbae8a497d2d68f7b18b8930e89392e0510f7ec Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 16:36:06 -0500 Subject: [PATCH 14/21] Address review feedback --- .../shape-inference/static-objects.ts | 206 ++++++++++++++---- .../test/static-object-guards.test.ts | 148 +++++++++++++ 2 files changed, 309 insertions(+), 45 deletions(-) create mode 100644 packages/compiler/test/static-object-guards.test.ts diff --git a/packages/compiler/src/frontend/shape-inference/static-objects.ts b/packages/compiler/src/frontend/shape-inference/static-objects.ts index ecaf664aa..640140e61 100644 --- a/packages/compiler/src/frontend/shape-inference/static-objects.ts +++ b/packages/compiler/src/frontend/shape-inference/static-objects.ts @@ -238,38 +238,142 @@ function ownLiteralMember(literal: Node, key: string): string | null { return found; } -/** Whether a top-level statement runs no code but its own: no call, `new`, - * tagged template, class (static blocks and field initializers run), getter - * or setter, `await`, `yield` or `delete`. Function bodies do not run. */ -function runsNoCode(stmt: ts.Statement): boolean { +/** Whether a top-level statement runs no code but its own. Its expressions + * are literals, identifiers, functions (not run), object and array literals + * without spreads or computed keys, `=` stores, `var` bindings without + * patterns, `&&`, `||`, `??`, `?:`, `,`, `!`, `typeof`, `void`, `===` and + * `!==`, none of which calls a function, and member reads and stores that + * `plainMember` accepts (no getter or setter runs). Every other form can + * run code: a call, `new` or tagged template, a class (static blocks and + * field initializers run), coercion (arithmetic, `+`, templates with + * substitutions, relational and loose comparisons, `in`, `instanceof`, + * computed keys), iteration (spreads, destructuring), `delete`, `await` and + * `yield`. */ +function runsNoCode(stmt: ts.Statement, plainMember: (n: Node) => boolean): boolean { if (ts.isFunctionDeclaration(stmt) || ts.isEmptyStatement(stmt)) return true; - if (!ts.isVariableStatement(stmt) && !ts.isExpressionStatement(stmt)) return false; - let quiet = true; - const visit = (n: Node): void => { - if (!quiet) return; - if (ts.isFunctionExpression(n) || ts.isArrowFunction(n)) return; - if ( - ts.isCallExpression(n) || - ts.isNewExpression(n) || - ts.isTaggedTemplateExpression(n) || - ts.isClassExpression(n) || - ts.isGetAccessorDeclaration(n) || - ts.isSetAccessorDeclaration(n) || - ts.isAwaitExpression(n) || - ts.isYieldExpression(n) || - ts.isDeleteExpression(n) || - ts.isMethodDeclaration(n) - ) { - quiet = false; - return; - } - ts.forEachChild(n, (c) => { - visit(c); - return undefined; + if (ts.isExpressionStatement(stmt)) return quietExpression(stmt.expression, plainMember); + if (!ts.isVariableStatement(stmt)) return false; + return stmt.declarationList.declarations.nodes.every( + (d) => + ts.isIdentifier(d.name) && + (d.initializer === undefined || quietExpression(d.initializer, plainMember)), + ); +} + +function quietExpression(e: ts.Expression, plainMember: (n: Node) => boolean): boolean { + const quiet = (x: ts.Expression): boolean => quietExpression(x, plainMember); + if (ts.isParenthesizedExpression(e)) return quiet(e.expression); + if ( + ts.isIdentifier(e) || + ts.isNumericLiteral(e) || + ts.isBigIntLiteral(e) || + ts.isStringLiteral(e) || + ts.isNoSubstitutionTemplateLiteral(e) || + ts.isRegularExpressionLiteral(e) || + ts.isFunctionExpression(e) || + ts.isArrowFunction(e) || + e.kind === ts.SyntaxKind.TrueKeyword || + e.kind === ts.SyntaxKind.FalseKeyword || + e.kind === ts.SyntaxKind.NullKeyword || + e.kind === ts.SyntaxKind.ThisKeyword + ) + return true; + if (ts.isObjectLiteralExpression(e)) + return e.properties.nodes.every((m) => { + if (ts.isShorthandPropertyAssignment(m)) return m.objectAssignmentInitializer === undefined; + if (!ts.isPropertyAssignment(m)) return false; + const name = m.name; + if (!ts.isIdentifier(name) && !ts.isStringLiteral(name) && !ts.isNumericLiteral(name)) + return false; + return quiet(m.initializer); }); - }; - visit(stmt); - return quiet; + if (ts.isArrayLiteralExpression(e)) + return e.elements.nodes.every( + (x) => ts.isOmittedExpression(x) || (!ts.isSpreadElement(x) && quiet(x)), + ); + if (ts.isTypeOfExpression(e) || ts.isVoidExpression(e)) return quiet(e.expression); + if (ts.isPrefixUnaryExpression(e)) + return e.operator === ts.SyntaxKind.ExclamationToken && quiet(e.operand); + if (ts.isConditionalExpression(e)) + return quiet(e.condition) && quiet(e.whenTrue) && quiet(e.whenFalse); + if (ts.isPropertyAccessExpression(e) || ts.isElementAccessExpression(e)) + return quietMember(e, plainMember); + if (!ts.isBinaryExpression(e)) return false; + switch (e.operatorToken.kind) { + case ts.SyntaxKind.EqualsToken: + if (ts.isIdentifier(e.left)) return quiet(e.right); + return ( + (ts.isPropertyAccessExpression(e.left) || ts.isElementAccessExpression(e.left)) && + quietMember(e.left, plainMember) && + quiet(e.right) + ); + case ts.SyntaxKind.AmpersandAmpersandToken: + case ts.SyntaxKind.BarBarToken: + case ts.SyntaxKind.QuestionQuestionToken: + case ts.SyntaxKind.CommaToken: + case ts.SyntaxKind.EqualsEqualsEqualsToken: + case ts.SyntaxKind.ExclamationEqualsEqualsToken: + return quiet(e.left) && quiet(e.right); + default: + return false; + } +} + +/** A member read or store target with a literal key on a receiver that + * runs no code, which `plainMember` accepts. */ +function quietMember( + e: ts.PropertyAccessExpression | ts.ElementAccessExpression, + plainMember: (n: Node) => boolean, +): boolean { + if (e.questionDotToken !== undefined) return false; + if ( + ts.isElementAccessExpression(e) && + !ts.isStringLiteral(e.argumentExpression) && + !ts.isNumericLiteral(e.argumentExpression) + ) + return false; + return plainMember(e) && quietExpression(e.expression, plainMember); +} + +/** Whether an identifier is written where it appears: an assignment or + * update target, a destructuring assignment or loop target (through array + * and object patterns, defaults and rests), or a `var`, `let`, `const` or + * parameter binding of the name (a nested binding shadows it, which only + * makes the answer conservative). */ +function writesBinding(id: ts.Identifier): boolean { + let n: Node = id; + for (let p = n.parent; p !== undefined; n = p, p = p.parent) { + if (ts.isPrefixUnaryExpression(p) || ts.isPostfixUnaryExpression(p)) + return ( + p.operator === ts.SyntaxKind.PlusPlusToken || p.operator === ts.SyntaxKind.MinusMinusToken + ); + if (ts.isBinaryExpression(p)) { + const op = p.operatorToken.kind; + if (op >= ts.SyntaxKind.FirstAssignment && op <= ts.SyntaxKind.LastAssignment) + return p.left === n; + return false; + } + if (ts.isForOfStatement(p) || ts.isForInStatement(p)) return p.initializer === n; + if (ts.isVariableDeclaration(p) || ts.isParameter(p) || ts.isBindingElement(p)) + return p.name === n; + if ( + ts.isParenthesizedExpression(p) || + ts.isArrayLiteralExpression(p) || + ts.isObjectLiteralExpression(p) || + ts.isSpreadElement(p) || + ts.isSpreadAssignment(p) || + ts.isShorthandPropertyAssignment(p) || + ts.isArrayBindingPattern(p) || + ts.isObjectBindingPattern(p) + ) + continue; + if (ts.isPropertyAssignment(p)) { + if (p.initializer !== n) return false; + continue; + } + return false; + } + return false; } function enclosingFunctionLike(n: Node): Node | null { @@ -1345,6 +1449,24 @@ export function planStaticObjectRewrites( if (!l.includes(op.node)) l.push(op.node); } } + // Member reads and stores whose receiver can only be constant objects or + // primitives: neither has accessors of its own, so no getter or setter + // runs (a primitive receiver of a store throws, or reads its prototype). + const memberReceivers = new Map>(); + for (const op of analysis.ops) { + if ((op.kind !== "read" && op.kind !== "write") || op.name === null) continue; + let set = memberReceivers.get(op.node); + if (set === undefined) memberReceivers.set(op.node, (set = new Set())); + for (const a of analysis.nodes[op.recv]!.atoms) set.add(a); + } + const plainMember = (n: Node): boolean => { + const atoms = memberReceivers.get(n); + return ( + atoms !== undefined && + atoms.size > 0 && + [...atoms].every((a) => a.kind === "prim" || constantAtoms.has(a)) + ); + }; const quietPrefix = new Map(); const storedLiteral = ( atom: Atom, @@ -1371,7 +1493,9 @@ export function planStaticObjectRewrites( const statements = stmt.parent.statements.nodes; let quiet = quietPrefix.get(stmt); if (quiet === undefined) { - quiet = statements.slice(0, statements.indexOf(stmt)).every(runsNoCode); + quiet = statements + .slice(0, statements.indexOf(stmt)) + .every((s) => runsNoCode(s, plainMember)); quietPrefix.set(stmt, quiet); } return quiet ? { literal, stmt, statements } : null; @@ -1472,29 +1596,21 @@ export function planStaticObjectRewrites( if (a.kind === "function" && a.node === fn && analysis.escaped.has(a)) return false; const index = (fn as ts.FunctionLikeDeclaration).parameters.nodes.indexOf(keyDecl); const name = keyDecl.name.text; + // Any write of the name in the function, its other parameters included, + // or a direct eval or `arguments` (which aliases parameters in sloppy + // code) can give the key another value. let touched = false; const visit = (n: Node): void => { if (touched) return; - if (ts.isIdentifier(n) && n.text === "arguments") touched = true; - else if ( - ts.isBinaryExpression(n) && - n.operatorToken.kind >= ts.SyntaxKind.FirstAssignment && - n.operatorToken.kind <= ts.SyntaxKind.LastAssignment && - ts.isIdentifier(n.left) && - n.left.text === name - ) - touched = true; - else if ( - (ts.isPrefixUnaryExpression(n) || ts.isPostfixUnaryExpression(n)) && - ts.isIdentifier(n.operand) && - n.operand.text === name - ) + if (ts.isIdentifier(n) && (n.text === "arguments" || n.text === "eval")) touched = true; + else if (ts.isIdentifier(n) && n.text === name && n !== keyDecl.name && writesBinding(n)) touched = true; ts.forEachChild(n, (c) => { visit(c); return undefined; }); }; + for (const p of (fn as ts.FunctionLikeDeclaration).parameters.nodes) visit(p); const body = (fn as { body?: Node }).body; if (body === undefined) return false; visit(body); diff --git a/packages/compiler/test/static-object-guards.test.ts b/packages/compiler/test/static-object-guards.test.ts new file mode 100644 index 000000000..2aba8dfd6 --- /dev/null +++ b/packages/compiler/test/static-object-guards.test.ts @@ -0,0 +1,148 @@ +/* Static-object specialization (shape-inference/static-objects.ts) keeps two + * reads dynamic when the facts it would fold on can change: + * - a member stored after a statement that can run code (a coercion such as + * `+` can call toString or valueOf) keeps its read in functions, which + * that code could call before the store (the statements checked are all + * the file's top-level statements before the store); + * - a table call whose key parameter the function rebinds (destructuring, + * a loop target, another parameter's default or a `var` redeclaration) + * keeps reading the table for other keys, though every caller passes one + * of the table's keys. + * The checks read the rewritten package text SCRIPTC_DEBUG_SHAPE_OVERLAY + * prints. */ +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, expect, test, vi } from "vitest"; +import { compile } from "../src/index.js"; + +const packageSource = `var quiet = quiet || {}; +quiet.cfg = { U: 1 }; +quiet.read = function () { + return quiet.cfg.V; +}; +quiet.label = quiet.cfg.U; +quiet.cfg.V = 6; + +var late = late || {}; +late.cfg = { U: 1 }; +late.read = function () { + return late.cfg.V; +}; +late.label = "v" + late.cfg.U; +late.cfg.V = 5; + +var t = t || {}; +t.ops = { + a: function (v) { + return v + 1; + }, + b: function (v) { + return v * 2; + }, +}; +t.plain = function (op, v) { + return t.ops[op](v); +}; +t.pattern = function (op, v) { + [op] = ["c"]; + return t.ops[op](v); +}; +t.loop = function (op, v) { + for (op of ["c"]); + return t.ops[op](v); +}; +t.initializer = function (op, v, w = (op = "c")) { + return t.ops[op](v); +}; +t.redeclared = function (op, v) { + var op = "c"; + return t.ops[op](v); +}; +module.exports = { late, quiet, t }; +`; + +const consumer = `const { late, quiet, t } = require("guard-fixture"); +console.log(late.read(), late.label, quiet.read(), quiet.label); +const tryCall = (call) => { + try { + return call(); + } catch (e) { + return e.name; + } +}; +console.log(t.plain("a", 1), t.plain("b", 2)); +console.log(tryCall(() => t.pattern("a", 1)), tryCall(() => t.loop("b", 2))); +console.log(tryCall(() => t.initializer("a", 1)), tryCall(() => t.redeclared("b", 2))); +`; + +let dir = ""; +const saved: Record = {}; +beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), "scriptc-static-guards-")); + const pkg = join(dir, "node_modules/guard-fixture"); + await mkdir(pkg, { recursive: true }); + await writeFile( + join(pkg, "package.json"), + JSON.stringify({ name: "guard-fixture", version: "1.0.0", main: "index.js" }), + ); + await writeFile(join(pkg, "index.js"), packageSource); + await writeFile( + join(dir, "package.json"), + JSON.stringify({ dependencies: { "guard-fixture": "1.0.0" } }), + ); + await writeFile(join(dir, "main.cjs"), consumer); + for (const k of ["SCRIPTC_SHAPE_OVERLAY", "SCRIPTC_DEBUG_SHAPE_OVERLAY", "SCRIPTC_NO_CACHE"]) + saved[k] = process.env[k]; +}); +afterEach(async () => { + for (const [k, v] of Object.entries(saved)) { + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + vi.restoreAllMocks(); + await rm(dir, { recursive: true, force: true }); +}); + +/** The package text after the structural rewrites. */ +async function rewrittenPackage(): Promise { + process.env["SCRIPTC_SHAPE_OVERLAY"] = "1"; + process.env["SCRIPTC_DEBUG_SHAPE_OVERLAY"] = "1"; + process.env["SCRIPTC_NO_CACHE"] = "1"; + let text = ""; + vi.spyOn(process.stderr, "write").mockImplementation((chunk: string | Uint8Array) => { + text += typeof chunk === "string" ? chunk : Buffer.from(chunk).toString("utf8"); + return true; + }); + const result = await compile(join(dir, "main.cjs"), { + outDir: dir, + outPath: join(dir, "main.ir.json"), + outputKind: "ir", + npmStatic: ["guard-fixture"], + }); + vi.restoreAllMocks(); + if (!result.ok) throw new Error(result.diagnostics.map((d) => d.message).join("\n")); + const start = text.indexOf("[static-objects] "); + const end = text.indexOf("[shape-overlay] ", start); + return start < 0 ? "" : text.slice(start, end < 0 ? undefined : end); +} + +/** The body of the function the rewritten text assigns to or hoists as `name`. */ +function body(text: string, name: string): string { + const at = text.search(new RegExp(`function [\\w$]*${name}\\b|${name} = function`)); + return at < 0 ? "" : text.slice(at, text.indexOf("\n}", at)); +} + +test("a store after code that can run keeps the read; a quiet prefix folds it", async () => { + const text = await rewrittenPackage(); + expect(body(text, "quiet_read")).toMatch(/return \(.*\b6\);/); + expect(body(text, "late_read")).toContain("return late.cfg.V"); +}); + +test("a table call whose key parameter is rebound keeps reading the table", async () => { + const text = await rewrittenPackage(); + // Every caller passes "a" or "b": the plain call needs no arm for other keys. + expect(body(text, "t_plain")).not.toContain("t.ops[op](v)"); + for (const name of ["t_pattern", "t_loop", "t_initializer", "t_redeclared"]) + expect(body(text, name), name).toContain("t.ops[op](v)"); +}); From 14e4c7d1ef9ef169d049cca703d8682d236a3a12 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Sun, 25 Oct 2026 14:00:35 -0700 Subject: [PATCH 15/21] Keep static-object rewrites when propertyIsEnumerable asks another class's prototype MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The analysis reads F.prototype as ⊤ beside the class prototype objects it tracks, so every class looked queried by any G.prototype.propertyIsEnumerable call. The query reaches a class's prototype only when G may be that class's function. Co-Authored-By: Claude Opus 5.5 --- .../src/frontend/shape-inference/static-objects.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/compiler/src/frontend/shape-inference/static-objects.ts b/packages/compiler/src/frontend/shape-inference/static-objects.ts index 541366364..c7f36f4cf 100644 --- a/packages/compiler/src/frontend/shape-inference/static-objects.ts +++ b/packages/compiler/src/frontend/shape-inference/static-objects.ts @@ -608,7 +608,16 @@ export function planStaticObjectRewrites( * as the object `propertyIsEnumerable` is asked about. */ const reflectsOn = (atoms: ReadonlySet, ctorPath: string): boolean => enumerabilityEscapes || - enumerabilityQueries.some((recv) => mayBeValue(recv, atoms, ctorPath)) || + enumerabilityQueries.some((recv) => { + // `F.prototype` is one of `atoms` (a class's prototype) only when F + // may be that class's function, which `atoms` holds too: the analysis + // reads a prototype as ⊤ beside the class prototype objects it tracks. + let e: ts.Expression = recv; + while (ts.isParenthesizedExpression(e)) e = e.expression; + if (ts.isPropertyAccessExpression(e) && e.name.text === "prototype") + return mayBeValue(e.expression, atoms, ctorPath); + return mayBeValue(recv, atoms, ctorPath); + }) || reflective.some((call) => call.arguments.nodes.some((arg) => mayBe(arg, atoms, ctorPath))); /** Whether a call that freezes, seals or redefines an object may reach one of `atoms`. */ const reshapes = (atoms: ReadonlySet, ctorPath: string): boolean => From 98e67128c1079b8b59d66feed284b0435f62a63f Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Sun, 25 Oct 2026 14:00:45 -0700 Subject: [PATCH 16/21] static-stores: this.tag on a literal made at the call stays a read The object literal is created in the consumer's call, so the base's created-before check keeps the read. Co-Authored-By: Claude Opus 5.5 --- tests/harness/static-stores.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/harness/static-stores.test.ts b/tests/harness/static-stores.test.ts index 303237392..7296580fd 100644 --- a/tests/harness/static-stores.test.ts +++ b/tests/harness/static-stores.test.ts @@ -33,7 +33,7 @@ test( if (!result.ok) throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); const report = (result.warnings ?? []).find((w) => w.code === "SC6103"); - expect(report?.message).toMatch(/2 function table calls, 5 constant reads\)/); + expect(report?.message).toMatch(/2 function table calls, 4 constant reads\)/); const pending = execFileAsync(result.binaryPath, [], { cwd: dirname(entry), encoding: "utf8" }); pending.child.stdin?.end(); const { stdout } = await pending; From b3d7adb1a0e2160d0de9a619760651f765ff98a3 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Sun, 25 Oct 2026 14:50:39 -0700 Subject: [PATCH 17/21] Remove the committed node_modules symlink Co-Authored-By: Claude Opus 5.5 --- packages/compiler/node_modules | 1 - 1 file changed, 1 deletion(-) delete mode 120000 packages/compiler/node_modules diff --git a/packages/compiler/node_modules b/packages/compiler/node_modules deleted file mode 120000 index 6b01b0eda..000000000 --- a/packages/compiler/node_modules +++ /dev/null @@ -1 +0,0 @@ -../../../shape-reg/packages/compiler/node_modules \ No newline at end of file From bddb5c7205d6302259b3ac4a8439fbcfc5ed89f6 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 17:13:16 -0500 Subject: [PATCH 18/21] Address review feedback --- .../shape-inference/static-objects.ts | 30 ++++++++++++++++--- .../node_modules/static-ns/index.js | 11 +++++++ 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/packages/compiler/src/frontend/shape-inference/static-objects.ts b/packages/compiler/src/frontend/shape-inference/static-objects.ts index c7f36f4cf..b5ea2cfe5 100644 --- a/packages/compiler/src/frontend/shape-inference/static-objects.ts +++ b/packages/compiler/src/frontend/shape-inference/static-objects.ts @@ -604,18 +604,40 @@ export function planStaticObjectRewrites( } return false; }; + /** Whether the read `e` of a `prototype` member may yield one of `atoms` + * through a receiver that is not a function. An object's field holds the + * values stored to it by name (writes through a ⊤ receiver reach every + * object); ⊤, a ⊤ field and an object written by computed key may hold + * anything. */ + const prototypeFieldMayBe = ( + e: ts.PropertyAccessExpression, + atoms: ReadonlySet, + ): boolean => { + const op = readOps.get(e); + if (op === undefined) return true; + for (const a of analysis.nodes[op.recv]!.atoms) { + if (analysis.protoOf.get(a) !== undefined) continue; + if (a === analysis.top || computedRecv.has(a)) return true; + const field = analysis.fields.get(a)?.get("prototype"); + if (field === undefined) continue; + for (const v of analysis.nodes[field]!.atoms) + if (v === analysis.top || atoms.has(v)) return true; + } + return false; + }; /** Whether a reflective call may reach one of `atoms`: as an argument, or * as the object `propertyIsEnumerable` is asked about. */ const reflectsOn = (atoms: ReadonlySet, ctorPath: string): boolean => enumerabilityEscapes || enumerabilityQueries.some((recv) => { - // `F.prototype` is one of `atoms` (a class's prototype) only when F - // may be that class's function, which `atoms` holds too: the analysis - // reads a prototype as ⊤ beside the class prototype objects it tracks. + // `F.prototype` is one of `atoms` (a class's prototype) when F may be + // that class's function, which `atoms` holds too: the analysis reads + // a prototype as ⊤ beside the class prototype objects it tracks. It + // also is when F is an object whose `prototype` field holds it. let e: ts.Expression = recv; while (ts.isParenthesizedExpression(e)) e = e.expression; if (ts.isPropertyAccessExpression(e) && e.name.text === "prototype") - return mayBeValue(e.expression, atoms, ctorPath); + return mayBeValue(e.expression, atoms, ctorPath) || prototypeFieldMayBe(e, atoms); return mayBeValue(recv, atoms, ctorPath); }) || reflective.some((call) => call.arguments.nodes.some((arg) => mayBe(arg, atoms, ctorPath))); diff --git a/tests/fixtures/npm/cases/static-objects/node_modules/static-ns/index.js b/tests/fixtures/npm/cases/static-objects/node_modules/static-ns/index.js index b1884948c..92ab5dda2 100644 --- a/tests/fixtures/npm/cases/static-objects/node_modules/static-ns/index.js +++ b/tests/fixtures/npm/cases/static-objects/node_modules/static-ns/index.js @@ -171,6 +171,16 @@ ns.Listed.prototype.item = function () { return this.l; }; +// Its prototype is reached through a plain object's field, and that field +// is asked whether a method is enumerable. +ns.Held = function () { + this.h = 2; +}; +ns.Held.prototype.hold = function () { + return this.h; +}; +var protoHolder = { prototype: ns.Held.prototype }; + ns.run = function () { var out = []; var p = new ns.Point(3, 4); @@ -197,6 +207,7 @@ ns.run = function () { out.push(uninitialized, aliasLater.twice(2), tdzAlias.twice(3)); out.push(probeLog.join("|"), new ns.Gadget().spin()); out.push(ns.Listed.prototype.propertyIsEnumerable("item"), new ns.Listed().item()); + out.push(protoHolder.prototype.propertyIsEnumerable("hold"), new ns.Held().hold()); return out.join(" "); }; From 09de11920c77d28f1faaf166fec5a9133a47b2e8 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Sun, 25 Oct 2026 15:15:13 -0700 Subject: [PATCH 19/21] Keep the receiver read in every JavaScript function for now Skipping it in functions without `this` missed methods that read the receiver through super (8302-class-super-method-values, prototype-method-overrides) and changes the default build's dynamic-tier counts. It changes the default JavaScript lowering, so it gets its own PR. Co-Authored-By: Claude Opus 5.5 --- .../src/frontend/lowering/lower-functions.ts | 28 ++----------------- 1 file changed, 2 insertions(+), 26 deletions(-) diff --git a/packages/compiler/src/frontend/lowering/lower-functions.ts b/packages/compiler/src/frontend/lowering/lower-functions.ts index 444134203..bd9b5f55d 100644 --- a/packages/compiler/src/frontend/lowering/lower-functions.ts +++ b/packages/compiler/src/frontend/lowering/lower-functions.ts @@ -62,29 +62,6 @@ function declareCallReceiver( }; } -/** Whether a JavaScript function's own `this` can be read: a `this` in its - * parameters or body, outside nested non-arrow functions (which bind their - * own). Conservative inside classes and object literal methods. A function - * that never reads it skips the ambient receiver read at entry. */ -function readsOwnThis(node: ts.FunctionLikeDeclaration): boolean { - let found = false; - const visit = (n: ts.Node): void => { - if (found) return; - if (n.kind === ts.SyntaxKind.ThisKeyword) { - found = true; - return; - } - if (ts.isFunctionDeclaration(n) || ts.isFunctionExpression(n)) return; - ts.forEachChild(n, (c) => { - visit(c); - return undefined; - }); - }; - for (const p of node.parameters.nodes) visit(p); - if (node.body !== undefined) visit(node.body); - return found; -} - /** True when the identifier resolves (through import aliases) to a * top-level function declaration of ANY program file (not merely a * same-named local shadowing one). Functions declared directly in a @@ -359,7 +336,7 @@ export function lowerLambda( try { const receiver = (checkedReceiver || - (isJsSourceFile(ts.getSourceFileOfNode(node)) && readsOwnThis(node)) || + isJsSourceFile(ts.getSourceFileOfNode(node)) || node.parameters.nodes.some(isThisParameter)) && !ts.isArrowFunction(node) ? declareCallReceiver(lowerer, node, isGenerator) @@ -564,8 +541,7 @@ export function lowerFunction(lowerer: Lowerer, decl: ts.FunctionDeclaration): I const argumentCount: IrStmt[] = []; declareArgumentCount(lowerer, decl, argumentCount); const receiver = - decl.parameters.nodes.some(isThisParameter) || - (isJsSourceFile(ts.getSourceFileOfNode(decl)) && readsOwnThis(decl)) + decl.parameters.nodes.some(isThisParameter) || isJsSourceFile(ts.getSourceFileOfNode(decl)) ? declareCallReceiver(lowerer, decl, sig.generator !== undefined) : null; const { params, prologue } = lowerer.declareParams(decl.parameters.nodes, sig.params); From d294af682a56664e7ed61276f69eb8eef014b1b5 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 17:53:23 -0500 Subject: [PATCH 20/21] Fix the static-objects order test --- .../src/frontend/shape-inference/static-objects.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/compiler/src/frontend/shape-inference/static-objects.ts b/packages/compiler/src/frontend/shape-inference/static-objects.ts index d20f53cfc..e36b67d41 100644 --- a/packages/compiler/src/frontend/shape-inference/static-objects.ts +++ b/packages/compiler/src/frontend/shape-inference/static-objects.ts @@ -1394,10 +1394,6 @@ export function planStaticObjectRewrites( if (parent === undefined || ts.isDeleteExpression(parent)) continue; if (!ts.isPropertyAccessExpression(node)) continue; if (topStatement(node) === null) continue; - // The receiver's evaluation is dropped, and the binding exists from the - // store on (a function) or from its declaration on (a class). - if (!pureReceiver(node.expression, node)) continue; - if (!runsAfter(node, storeOf.get(field.values[0]!)!)) continue; // The reference replaces the whole member expression: its receiver // must be one that evaluating runs no code. if (!effectFree(node.expression)) { @@ -1408,6 +1404,10 @@ export function planStaticObjectRewrites( kept.push(`reference ${node.getText()}: it may run before the member is stored`); continue; } + // The receiver's evaluation is dropped, and the binding exists from the + // store on (a function) or from its declaration on (a class). + if (!pureReceiver(node.expression, node)) continue; + if (!runsAfter(node, storeOf.get(field.values[0]!)!)) continue; if (node.getSourceFile() !== field.values[0]!.node!.getSourceFile()) { const text = crossFileReference(node, field.values[0]!.node!.getSourceFile(), name); if (text === null) continue; From 97787329c2c91f246004a8bb83ee08808d09b499 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 22:43:19 -0500 Subject: [PATCH 21/21] Dispatch a table exhaustively only from calls that name the function A key parameter took no fallback arm when every call edge of its function passed a string literal key of the table. A call through `call` or `apply` passes the this value first, so `f.call("a", "c")` looked like a call with the key "a", and the dispatch for "c" called the table's last member where Node throws a TypeError. A builtin that calls the function back chooses its own arguments too. Each call now has to name the function itself: its callee is a binding, or a constant object's member, whose values are only that function. --- .../shape-inference/static-objects.ts | 27 ++++++++++++++++++- .../test/static-object-guards.test.ts | 19 ++++++++++++- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/packages/compiler/src/frontend/shape-inference/static-objects.ts b/packages/compiler/src/frontend/shape-inference/static-objects.ts index daf862a53..bf8835b19 100644 --- a/packages/compiler/src/frontend/shape-inference/static-objects.ts +++ b/packages/compiler/src/frontend/shape-inference/static-objects.ts @@ -1648,8 +1648,33 @@ export function planStaticObjectRewrites( if (touched) return false; const edges = analysis.calls.filter((c) => c.callee.node === fn); if (edges.length === 0) return false; + // A call lines its arguments up with the parameters only when its callee + // is the function itself: a binding, or a constant object's member, that + // holds nothing else. `call` and `apply` pass the this value first, and + // a builtin that calls the function back chooses its own arguments. + const isFn = (a: Atom): boolean => a.kind === "function" && a.node === fn; + const onlyFn = (values: ReadonlySet): boolean => + values.size > 0 && [...values].every(isFn); + const callsFn = (callee: ts.Expression): boolean => { + while (ts.isParenthesizedExpression(callee)) callee = callee.expression; + if (ts.isIdentifier(callee)) { + const d = declOf(callee); + return d !== undefined && onlyFn(analysis.declValues(d)); + } + if (!ts.isPropertyAccessExpression(callee)) return false; + const op = readOps.get(callee); + if (op === undefined || op.name === null) return false; + const recv = analysis.nodes[op.recv]!.atoms; + if (recv.size === 0) return false; + for (const a of recv) { + if (!constantAtoms.has(a)) return false; + const field = analysis.fields.get(a)?.get(op.name); + if (field === undefined || !onlyFn(analysis.nodes[field]!.atoms)) return false; + } + return true; + }; return edges.every((e) => { - if (!ts.isCallExpression(e.site)) return false; + if (!ts.isCallExpression(e.site) || !callsFn(e.site.expression)) return false; const args = e.site.arguments.nodes; if (args.slice(0, index + 1).some((a) => ts.isSpreadElement(a))) return false; const arg = args[index]; diff --git a/packages/compiler/test/static-object-guards.test.ts b/packages/compiler/test/static-object-guards.test.ts index 2aba8dfd6..e14793c9a 100644 --- a/packages/compiler/test/static-object-guards.test.ts +++ b/packages/compiler/test/static-object-guards.test.ts @@ -7,7 +7,9 @@ * - a table call whose key parameter the function rebinds (destructuring, * a loop target, another parameter's default or a `var` redeclaration) * keeps reading the table for other keys, though every caller passes one - * of the table's keys. + * of the table's keys; + * - so does one whose function is called through `call` or `apply`, where + * the first argument is the this value and not the key. * The checks read the rewritten package text SCRIPTC_DEBUG_SHAPE_OVERLAY * prints. */ import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; @@ -59,6 +61,12 @@ t.redeclared = function (op, v) { var op = "c"; return t.ops[op](v); }; +t.viaCall = function (op, v) { + return t.ops[op](v); +}; +t.viaApply = function (op, v) { + return t.ops[op](v); +}; module.exports = { late, quiet, t }; `; @@ -74,6 +82,7 @@ const tryCall = (call) => { console.log(t.plain("a", 1), t.plain("b", 2)); console.log(tryCall(() => t.pattern("a", 1)), tryCall(() => t.loop("b", 2))); console.log(tryCall(() => t.initializer("a", 1)), tryCall(() => t.redeclared("b", 2))); +console.log(tryCall(() => t.viaCall.call("a", "c", 1)), tryCall(() => t.viaApply.apply("b", ["c", 2]))); `; let dir = ""; @@ -146,3 +155,11 @@ test("a table call whose key parameter is rebound keeps reading the table", asyn for (const name of ["t_pattern", "t_loop", "t_initializer", "t_redeclared"]) expect(body(text, name), name).toContain("t.ops[op](v)"); }); + +test("a table call whose function is called through call or apply keeps reading the table", async () => { + // The first argument of those calls, a key of the table, is the this + // value; the key each call passes, "c", is not one. + const text = await rewrittenPackage(); + for (const name of ["t_viaCall", "t_viaApply"]) + expect(body(text, name), name).toContain("t.ops[op](v)"); +});