diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f9b725472..9770dc009 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,6 +17,11 @@ on: type: choice options: [full, darwin] default: full + scope: + description: Every lane, or only the shape overlay validation (shape_overlay) + type: choice + options: [all, shape_overlay] + default: all concurrency: # A new push to a pull request cancels its previous run. Every push to main runs to completion in a group of @@ -55,7 +60,10 @@ jobs: const upperLayer = process.env.GITHUB_EVENT_NAME === 'pull_request' && sameRepository && trustedAuthor && typeof stackBase === 'string' && stackBase.length > 0 && typeof prBase === 'string' && prBase.length > 0 && stackBase !== prBase; - const full = !upperLayer; + // A manual run of the shape overlay validation alone skips the other lanes. + const shapeOnly = process.env.GITHUB_EVENT_NAME === 'workflow_dispatch' && + event.inputs?.scope === 'shape_overlay'; + const full = !upperLayer && !shapeOnly; appendFileSync(process.env.GITHUB_OUTPUT, `full=${full}\n`); console.log(full ? 'Run full CI' : 'Defer full CI until this PR becomes the lowest unmerged layer'); NODE @@ -379,6 +387,81 @@ jobs: - run: tar -xzf "$RUNNER_TEMP/native-linux-x64-gnu.tar.gz" - run: pnpm test tests/harness/three.test.ts -t "'nodes' runs statically through LLVM on 'native'" + # Shape inference's overlay (opt-in, SCRIPTC_SHAPE_OVERLAY=1) over the + # suites of real packages: nightly and on manual runs only (scope + # shape_overlay runs it alone), outside the `test` roll-up. Every case + # must build and run as in the default build. It builds its own native + # packages, so it does not depend on the other lanes. + shape_overlay_native: + if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} + name: build (Linux x64 native packages, shape overlay) + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + with: + version: 11.1.3 + - uses: actions/setup-node@v4 + with: + node-version-file: .node-version + cache: pnpm + - uses: ./.github/actions/llvm-linux + - run: pnpm install --frozen-lockfile + - uses: ./.github/actions/zig + - uses: ./.github/actions/native-packages + with: + target: linux-x64-gnu + - name: Preserve executable permissions in shared packages + run: >- + tar -czf "$RUNNER_TEMP/native-linux-x64-gnu.tar.gz" + packages/llvm-linux-x64-gnu/bin + packages/runtime-linux-x64-gnu/artifacts + packages/runtime-linux-x64-gnu/runtime-pack.json + - uses: actions/upload-artifact@v4 + with: + name: native-linux-x64-gnu-shape-overlay + path: ${{ runner.temp }}/native-linux-x64-gnu.tar.gz + retention-days: 1 + + shape_overlay: + needs: [shape_overlay_native] + if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} + name: test (Linux, shape overlay, ${{ matrix.suite }} ${{ matrix.shard }}/8) + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + suite: [npm, npm-static, three, effect] + shard: [1, 2, 3, 4, 5, 6, 7, 8] + env: + SCRIPTC_SHAPE_OVERLAY: "1" + SCRIPTC_TEST_WORKERS: "1" + SCRIPTC_EFFECT_TEST_CONCURRENCY: "1" + SCRIPTC_TEST_SHARD: ${{ matrix.shard }}/8 + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + with: + version: 11.1.3 + - uses: actions/setup-node@v4 + with: + node-version-file: .node-version + cache: pnpm + - run: pnpm install --frozen-lockfile + - uses: actions/download-artifact@v4 + with: + name: native-linux-x64-gnu-shape-overlay + path: ${{ runner.temp }} + - run: tar -xzf "$RUNNER_TEMP/native-linux-x64-gnu.tar.gz" + - name: Run the suite with the overlay on + run: | + case "${{ matrix.suite }}" in + npm) pnpm test tests/harness/npm-shape-overlay.test.ts ;; + npm-static) pnpm test tests/harness/npm-static.test.ts ;; + three) pnpm test tests/harness/three.test.ts ;; + effect) pnpm test tests/harness/effect.test.ts tests/harness/effect4.test.ts ;; + esac + # Build each instrumented seed once. Downstream contracts retain the same # compiler and source artifacts without repeating seed construction. bootstrap_seed: diff --git a/packages/compiler/src/executable/prepare.ts b/packages/compiler/src/executable/prepare.ts index 027bd6ca6..b6538fe01 100644 --- a/packages/compiler/src/executable/prepare.ts +++ b/packages/compiler/src/executable/prepare.ts @@ -68,11 +68,14 @@ export function prepareExecutableModule( if (fe.preflight.length > 0) return fail(fe.preflight); try { - lowered = fe.lower({ - targetPlatform: buildPlatform, - ...(ffi !== null ? { ffiImports: ffi.functions } : {}), - ...(opts.tier !== undefined ? { tierMode: opts.tier } : {}), - }); + lowered = fe.lower( + { + targetPlatform: buildPlatform, + ...(ffi !== null ? { ffiImports: ffi.functions } : {}), + ...(opts.tier !== undefined ? { tierMode: opts.tier } : {}), + }, + (module) => validateModule(module).length === 0, + ); timing("lower"); } catch (e) { // The last-resort panic fence: an upstream tsgo panic that crossed a diff --git a/packages/compiler/src/frontend/input-tracker.ts b/packages/compiler/src/frontend/input-tracker.ts index c23a87313..3028f5a01 100644 --- a/packages/compiler/src/frontend/input-tracker.ts +++ b/packages/compiler/src/frontend/input-tracker.ts @@ -138,6 +138,16 @@ export class FrontendInputTracker { } } + /** Raw file names whose source text this tracker recorded. */ + private readonly notedTexts = new Set(); + + /** Records a raw read of a source file's text, once per file name. */ + noteText(fileName: string): void { + if (this.notedTexts.has(fileName)) return; + this.notedTexts.add(fileName); + this.record({ op: "text", path: resolve(fileName) }); + } + record(probe: FrontendInputProbe): void { const key = `${probe.op}\0${probe.path}`; if (probe.op === "text" && this.probes.has(key)) return; @@ -198,7 +208,7 @@ function record(probe: FrontendInputProbe): void { // callbacks, line numbers) changes with edits that keep the tokens. observeSourceText((fileName) => { const tracker = synchronousTracker ?? activeTracker.getStore(); - if (tracker !== undefined) tracker.record({ op: "text", path: resolve(fileName) }); + if (tracker !== undefined) tracker.noteText(fileName); }); /** Mark the active frontend as unsafe to persist because an exact host query diff --git a/packages/compiler/src/frontend/lowering/containers/array-concat.ts b/packages/compiler/src/frontend/lowering/containers/array-concat.ts index 440cbe9e2..83d149677 100644 --- a/packages/compiler/src/frontend/lowering/containers/array-concat.ts +++ b/packages/compiler/src/frontend/lowering/containers/array-concat.ts @@ -4,6 +4,7 @@ import { elementMayBeForeign, foreignElementThrow } from "../array-values.js"; import { countedFor, numLit, varRef } from "../../../ir/build.js"; import { BOOL, + DYN, F64, arrayOf, typeEquals, @@ -19,15 +20,18 @@ import type { Lowerer } from "../lowerer.js"; /** Copy into fresh storage after every argument has evaluated. Array * arguments retain their own element ABI; only copied values are widened. - * Missing slots stay holes and present undefined stays present. */ + * Missing slots stay holes and present undefined stays present. A shape + * entry is an array argument's element type, null for an element value, or + * "dyn" for a checked-dynamic argument, which JS spreads when it is an + * array at run time (IsConcatSpreadable) and appends otherwise. */ export function arrayConcatHelper( lowerer: Lowerer, elem: IrType, - shape: (IrType | null)[], + shape: (IrType | null | "dyn")[], node: ts.CallExpression, loc: SrcLoc, ): string { - const key = `concat:${typeKey(elem)}:${shape.map((type) => (type ? `a:${typeKey(type)}` : "e")).join(";")}`; + const key = `concat:${typeKey(elem)}:${shape.map((type) => (type === "dyn" ? "d" : type ? `a:${typeKey(type)}` : "e")).join(";")}`; const existing = lowerer.arrHofHelpers.get(key); if (existing) return existing; const name = keyedHelperName(lowerer, "%arr.concat", key); @@ -37,7 +41,7 @@ export function arrayConcatHelper( ...shape.map((type, i) => ({ localId: `x.${i}`, name: `x${i}`, - type: type ? arrayOf(type) : elem, + type: type === "dyn" ? DYN : type ? arrayOf(type) : elem, })), ]; const locals: IrLocal[] = params.map((p) => ({ @@ -173,6 +177,43 @@ export function arrayConcatHelper( }, ]; }; + // A checked-dynamic argument: an array spreads its elements, converted + // to the element type; any other value is appended as one element. + const appendDyn = (source: IrExpr, site: ts.Node): IrStmt[] => [ + { + kind: "if", + cond: { kind: "dynTest", test: "array", value: source, type: BOOL, loc }, + then: [ + { + kind: "exprStmt", + expr: { + kind: "arrIntrinsic", + method: "concatSpread", + receiver: out, + args: [lowerer.coerceInto(site, source, arrT)], + type: F64, + loc, + }, + loc, + }, + ], + else_: [ + { + kind: "exprStmt", + expr: { + kind: "arrIntrinsic", + method: "push", + receiver: out, + args: [lowerer.coerceInto(site, source, elem)], + type: F64, + loc, + }, + loc, + }, + ], + loc, + }, + ]; const body: IrStmt[] = [ { kind: "varDecl", @@ -182,11 +223,17 @@ export function arrayConcatHelper( }, ...append(varRef("a.0", arrT, loc), elem, node.expression), ...shape.flatMap((type, i) => - append(varRef(`x.${i}`, type ? arrayOf(type) : elem, loc), type, node.arguments.nodes[i]!), + type === "dyn" + ? appendDyn(varRef(`x.${i}`, DYN, loc), node.arguments.nodes[i]!) + : append( + varRef(`x.${i}`, type ? arrayOf(type) : elem, loc), + type, + node.arguments.nodes[i]!, + ), ), { kind: "return", value: out, loc }, ]; - if (shape.some((type) => type && !typeEquals(type, elem))) { + if (shape.some((type) => type !== null && type !== "dyn" && !typeEquals(type, elem))) { for (const id of ["i.0", "offset.0", "count.0"]) locals.push({ id, name: id, type: F64, mutable: true }); } diff --git a/packages/compiler/src/frontend/lowering/containers/array-methods.ts b/packages/compiler/src/frontend/lowering/containers/array-methods.ts index 3b16e714f..3a7bb86ba 100644 --- a/packages/compiler/src/frontend/lowering/containers/array-methods.ts +++ b/packages/compiler/src/frontend/lowering/containers/array-methods.ts @@ -998,7 +998,7 @@ export function lowerArrayMethodCall( // an array-of-arrays receiver given a bare inner array, where TS // types it as an element but JS would SPREAD it — is fenced. const receiver = lowerer.lowerExpr(access.expression); - const shape: (IrType | null)[] = []; + const shape: (IrType | null | "dyn")[] = []; const args: IrExpr[] = []; for (const argNode of call.arguments.nodes) { if (ts.isSpreadElement(argNode)) { @@ -1032,6 +1032,16 @@ export function lowerArrayMethodCall( args.push(arg); continue; } + // A checked-dynamic value: whether JS spreads it is decided at run + // time (an array spreads one level, anything else is one element). + if ( + (argIr === null || argIr.kind === "dyn") && + (probedArg ?? tryLowerExpression(lowerer, argNode))?.type.kind === "dyn" + ) { + shape.push("dyn"); + args.push(lowerer.lowerExpr(argNode)); + continue; + } // An element value — union elements wrap exactly like a push. shape.push(null); args.push(lowerer.lowerExprExpecting(argNode, elem)); diff --git a/packages/compiler/src/frontend/lowering/expressions/binary.ts b/packages/compiler/src/frontend/lowering/expressions/binary.ts index 0d24561de..d7d2cf3ef 100644 --- a/packages/compiler/src/frontend/lowering/expressions/binary.ts +++ b/packages/compiler/src/frontend/lowering/expressions/binary.ts @@ -2326,6 +2326,33 @@ export function lowerStrictEquality( negated: boolean, loc: SrcLoc, ): IrExpr { + // JavaScript: a checked-dynamic name the checker's flow analysis narrowed + // (`let c; ... (c = s[i]) === " "`) is compared unchecked. Strict + // equality never throws, and the flow type is the checker's inference + // (an indexed read past the end is undefined whatever it says). + const flowNarrowed = (n: ts.Node, e: IrExpr): IrExpr | null => + ts.isIdentifier(n) && + e.kind === "dynCheck" && + e.value.type.kind === "dyn" && + isJsSourceFile(ts.getSourceFileOfNode(n)) + ? e.value + : null; + const scalar = (e: IrExpr): boolean => + e.type.kind === "string" || e.type.kind === "f64" || e.type.kind === "bool"; + { + const l = flowNarrowed(leftNode, left); + const r = flowNarrowed(rightNode, right); + if ((l !== null && (scalar(right) || r !== null)) || (r !== null && scalar(left))) { + const cmp: IrExpr = { + kind: "dynScalarEq", + left: l ?? lowerer.coerceInto(leftNode, left, DYN), + right: r ?? lowerer.coerceInto(rightNode, right, DYN), + type: BOOL, + loc, + }; + return negated ? { kind: "unary", op: "!", operand: cmp, type: BOOL, loc } : cmp; + } + } const plainBothNum = left.type.kind === "f64" && right.type.kind === "f64"; const bothStr = left.type.kind === "string" && right.type.kind === "string"; const indexed = tryLowerIndexedComparison(lowerer, left, right, negated, loc); diff --git a/packages/compiler/src/frontend/lowering/expressions/narrowing.ts b/packages/compiler/src/frontend/lowering/expressions/narrowing.ts index 91b63cabc..b23916e66 100644 --- a/packages/compiler/src/frontend/lowering/expressions/narrowing.ts +++ b/packages/compiler/src/frontend/lowering/expressions/narrowing.ts @@ -515,6 +515,27 @@ export function maybeNarrow(lowerer: Lowerer, expr: IrExpr, node: ts.Node): IrEx ) ) return expr; + // A `let c;` binding without a type takes its flow types from what + // the code assigns, and the checker's types for those values can omit + // undefined (`c = s[i]` past the end): reads keep the actual value. + if ( + symbol && + symbol.declarations.some( + (decl) => + ts.isVariableDeclaration(decl) && + decl.initializer === undefined && + decl.type === undefined && + ts.isIdentifier(decl.name) && + decl.parent?.parent !== undefined && + ts.isVariableStatement(decl.parent.parent) && + !/@type\b/.test( + ts + .getSourceFileOfNode(decl) + .text.slice(decl.parent?.parent?.pos ?? decl.pos, ts.tokenStartOfNode(decl)), + ), + ) + ) + return expr; if ( symbol && symbol.declarations.some( diff --git a/packages/compiler/src/frontend/lowering/lower-stmts.ts b/packages/compiler/src/frontend/lowering/lower-stmts.ts index 2eae0af9b..9275d138b 100644 --- a/packages/compiler/src/frontend/lowering/lower-stmts.ts +++ b/packages/compiler/src/frontend/lowering/lower-stmts.ts @@ -5563,6 +5563,14 @@ function inferredEvolvingLetType(lowerer: Lowerer, decl: ts.VariableDeclaration) invalid = true; return "stop"; } + // An index read's checker type omits the undefined a read past the + // end (or of an absent key) answers. + let right = node.right; + while (ts.isParenthesizedExpression(right)) right = right.expression; + if (ts.isElementAccessExpression(right)) { + invalid = true; + return "stop"; + } const mapped = lowerer.mapTypeOf(lowerer.typeOf(node.right)); if (!mapped || mapped.kind === "dyn" || mapped.kind === "void" || isUnitType(mapped)) { invalid = true; diff --git a/packages/compiler/src/frontend/pipeline.ts b/packages/compiler/src/frontend/pipeline.ts index 208178861..04e82cb9a 100644 --- a/packages/compiler/src/frontend/pipeline.ts +++ b/packages/compiler/src/frontend/pipeline.ts @@ -3,10 +3,15 @@ import { beginShapeOverlayBuild, excludeRefusedOverlay, + recordShapeOverlayMemo, + restoreShapeOverlayState, + setShapeOverlayDropped, shapeOverlayActive, + shapeOverlayMemoPlainFences, shapeOverlayReport, } from "./shape-inference/index.js"; import type { ScrDiagnostic } from "../diagnostics/diagnostic.js"; +import type { IrModule } from "../ir/ir.js"; import type { SrcLoc } from "../ir/ir.js"; import type { NpmStaticStatus } from "../coverage/report.js"; import { entryFunctionExports, type EntryExportInfo } from "./lib-exports.js"; @@ -113,7 +118,9 @@ export interface Frontend { * dispose — it reads the ts7 AST). */ entryContract: () => ContractFacts; sourceTexts: () => Map; - lower: (opts: LowerOptions) => LowerResult; + /** `valid`: the caller's IR validation; a module it rejects with the + * shape overlay on is lowered again without the overlay. */ + lower: (opts: LowerOptions, valid?: (module: IrModule) => boolean) => LowerResult; /** Each requested or detected package's native compilation outcome, * including the first refusal reason for ineligible packages. */ npmStatic: NpmStaticStatus[]; @@ -598,6 +605,32 @@ function loadFrontend( // and library builds with a package-specific diagnostic. let load = reusableScout ?? loadProgram(entryPath, { npmStatic: requested, externalTypes }); let preflight = reusablePreflight ?? checkPreflight(load); + // Shape inference never makes the preflight fail: when the overlay's + // program reports diagnostics the program without it does not (its type + // references can load package files the program would not), the build + // continues without the overlay. + if (preflight.length > 0 && load.shapeOverlayState !== undefined) { + restoreShapeOverlayState(load.shapeOverlayState); + if (shapeOverlayActive()) { + const key = (d: ScrDiagnostic): string => `${d.code}|${d.loc.file}|${d.loc.start}`; + setShapeOverlayDropped(true); + const plain = loadProgram(entryPath, { npmStatic: requested, externalTypes }); + const plainPreflight = checkPreflight(plain); + const known = new Set(plainPreflight.map(key)); + if (preflight.some((d) => !known.has(key(d)))) { + load.dispose(); + load = plain; + preflight = plainPreflight; + if (process.env["SCRIPTC_DEBUG_SHAPE_FALLBACK"] !== undefined) + process.stderr.write( + `[shape-fallback] preflight: the overlay's program failed; overlay off\n`, + ); + } else { + plain.dispose(); + setShapeOverlayDropped(false); + } + } + } // Library mode's fixpoint: the opted-in packages' files joined the // program just now, and THEIR bare edges (import statements and // top-level requires) name packages the scout could not see. Judge each @@ -801,57 +834,65 @@ function loadFrontend( texts.set(projectConfig.manifest, projectConfig.manifestText); return texts; }, - lower: (opts) => { - const lowerOnce = (): LowerResult => - lowerToIr(finalLoad.program, finalLoad.entry, finalLoad.moduleOrder, { + lower: (opts, valid) => { + const lowerLoad = (l: typeof finalLoad): LowerResult => { + if (l.shapeOverlayState !== undefined) restoreShapeOverlayState(l.shapeOverlayState); + return lowerToIr(l.program, l.entry, l.moduleOrder, { ...opts, - frontendServices: finalLoad.services, - startupCrash: finalLoad.startupCrash ?? null, - inlineRequireCycleCalls: finalLoad.inlineRequireCycleCalls, - moduleEdges: finalLoad.moduleEdges, - projectCheckJs: finalLoad.typeWorld?.projectCheckJs ?? false, - externalTypes: finalLoad.externalTypes, - externalTypeSpecifiersByFile: finalLoad.externalTypeSpecifiersByFile, + frontendServices: l.services, + startupCrash: l.startupCrash ?? null, + inlineRequireCycleCalls: l.inlineRequireCycleCalls, + moduleEdges: l.moduleEdges, + projectCheckJs: l.typeWorld?.projectCheckJs ?? false, + externalTypes: l.externalTypes, + externalTypeSpecifiersByFile: l.externalTypeSpecifiersByFile, }); + }; + const lowerOnce = (): LowerResult => lowerLoad(finalLoad); let lowered = lowerOnce(); // Shape inference never breaks compilation (shape-inference/ // fallback.ts): refusals drop the overlay annotations they depend on // and the program reloads. Rounds that stop converging escalate to // the overlay of every file involved, then to no overlay at all. - // Refusals deferred to run time (runtime fences) drop only the - // annotations they are blamed on: a fence the overlay did not cause - // stays as it is without the overlay. - let level: "annotations" | "files" | "all" | "fences" = "annotations"; + // Refusals deferred to run time (runtime fences) are compared with + // the build without the overlay below: a fence the overlay did not + // cause stays as it is without the overlay. + let level: "annotations" | "files" | "all" = "annotations"; + // Deferred refusals the coverage pass found in code the lowering + // judged unreached count too: that code can still run. + const fencesOf = (l: LowerResult): ScrDiagnostic[] => [ + ...l.runtimeFences, + ...(l.unreached?.runtimeFences ?? []), + ]; + if (process.env["SCRIPTC_DEBUG_SHAPE_FALLBACK"] !== undefined) + process.stderr.write( + `[shape-fallback] lowered: overlay ${shapeOverlayActive()}, ${lowered.diagnostics.length} refusals, ${fencesOf(lowered).length} deferred\n`, + ); + // Failing refusals first; deferred refusals are compared with the + // build without the overlay below. for (let round = 0; shapeOverlayActive(); round++) { - const failed = lowered.diagnostics.length > 0; - const refusals = failed ? lowered.diagnostics : lowered.runtimeFences; + const refusals = lowered.diagnostics; if (refusals.length === 0) break; - let changed: boolean; - if (!failed) { - level = "fences"; - changed = - round < OVERLAY_ANNOTATION_ROUNDS && - excludeRefusedOverlay(finalLoad.program, refusals, level); - } else { - if (level === "fences") level = "annotations"; - if (round >= OVERLAY_ANNOTATION_ROUNDS && level === "annotations") level = "files"; - else if (round > OVERLAY_ANNOTATION_ROUNDS) level = "all"; + if (round >= OVERLAY_ANNOTATION_ROUNDS && level === "annotations") level = "files"; + else if (round > OVERLAY_ANNOTATION_ROUNDS) level = "all"; + let changed = excludeRefusedOverlay(finalLoad.program, refusals, level); + while (!changed && level !== "all") { + level = level === "annotations" ? "files" : "all"; changed = excludeRefusedOverlay(finalLoad.program, refusals, level); - while (!changed && level !== "all") { - level = level === "annotations" ? "files" : "all"; - changed = excludeRefusedOverlay(finalLoad.program, refusals, level); - } - // The same round also drops what the deferred refusals blame, - // saving a reload. - if (level !== "all" && lowered.runtimeFences.length > 0) - changed = - excludeRefusedOverlay(finalLoad.program, lowered.runtimeFences, "fences") || changed; } + // The program reloads anyway: the same round also drops what the + // deferred refusals blame, which often saves the comparison below. + if (level !== "all" && fencesOf(lowered).length > 0) + changed = + excludeRefusedOverlay(finalLoad.program, fencesOf(lowered), "fences") || changed; if (process.env["SCRIPTC_DEBUG_SHAPE_FALLBACK"] !== undefined) process.stderr.write( `[shape-fallback] round ${round} level ${level}: ${refusals.length} refusals (${refusals .slice(0, process.env["SCRIPTC_DEBUG_SHAPE_FALLBACK"] === "all" ? undefined : 3) - .map((d) => `${d.loc.file.split("/").pop()}:${d.loc.start} ${d.message.slice(0, 60)}`) + .map( + (d) => + `${d.loc.file.split("/").pop()}:${d.loc.start} ${process.env["SCRIPTC_DEBUG_SHAPE_FALLBACK"] === "all" ? d.message : d.message.slice(0, 60)}`, + ) .join("; ")}), changed ${changed}\n`, ); if (!changed) break; @@ -871,6 +912,104 @@ function loadFrontend( `[shape-fallback] reload ${(t1 - t0).toFixed(0)} ms, preflight ${(t2 - t1).toFixed(0)} ms, lower ${(performance.now() - t2).toFixed(0)} ms\n`, ); } + // Deferred refusals the program without the overlay does not have + // mean the overlay changed what runs (the overlay must not add + // failures the default build lacks). The build lowers the program + // without the overlay once, drops what the added refusals blame and + // reloads; when they persist (or blame nothing), it compiles without + // the overlay, reusing that lowering. A memo of the same JavaScript + // that already knows every remaining refusal skips the comparison. + let plainFences: Set | null = null; + if (shapeOverlayActive() && lowered.diagnostics.length === 0) { + const keyOf = (l: typeof finalLoad, d: ScrDiagnostic): string => { + const sf = l.program.getSourceFile(d.loc.file); + const line = sf === undefined ? -1 : sf.getLineAndCharacterOfPosition(d.loc.start).line; + return `${d.code}|${d.loc.file}|${line}`; + }; + const remembered = shapeOverlayMemoPlainFences(); + const fences = fencesOf(lowered); + const knownByMemo = + remembered !== null && fences.every((d) => remembered.has(keyOf(finalLoad, d))); + if (fences.length > 0 && !knownByMemo) { + const overlayState = finalLoad.shapeOverlayState; + setShapeOverlayDropped(true); + const plainLoad = loadProgram(entryPath, { npmStatic: finalNpmStatic, externalTypes }); + const plainPreflight = checkPreflight(plainLoad); + const plain = plainPreflight.length === 0 ? lowerLoad(plainLoad) : null; + setShapeOverlayDropped(false); + if (overlayState !== undefined) restoreShapeOverlayState(overlayState); + if (plain === null || plain.diagnostics.length > 0) { + // Nothing to compare with: the default build itself fails. + plainLoad.dispose(); + } else { + const known = new Set(fencesOf(plain).map((d) => keyOf(plainLoad, d))); + plainFences = known; + let usePlain = false; + for (let round = 0; ; round++) { + const added = fencesOf(lowered).filter((d) => !known.has(keyOf(finalLoad, d))); + if (added.length === 0) break; + const changed = + round < OVERLAY_ANNOTATION_ROUNDS && + excludeRefusedOverlay(finalLoad.program, added, "fences"); + if (process.env["SCRIPTC_DEBUG_SHAPE_FALLBACK"] !== undefined) + process.stderr.write( + `[shape-fallback] ${added.length} deferred refusals the default build lacks (${added + .slice(0, 3) + .map((d) => `${d.loc.file.split("/").pop()}: ${d.message.slice(0, 60)}`) + .join("; ")}), changed ${changed}\n`, + ); + if (!changed) { + usePlain = true; + break; + } + finalLoad.dispose(); + finalLoad = loadProgram(entryPath, { npmStatic: finalNpmStatic, externalTypes }); + if (checkPreflight(finalLoad).length > 0) { + usePlain = true; + break; + } + lowered = lowerOnce(); + if (lowered.diagnostics.length > 0) { + usePlain = true; + break; + } + } + if (usePlain) { + if (process.env["SCRIPTC_DEBUG_SHAPE_FALLBACK"] !== undefined) + process.stderr.write(`[shape-fallback] overlay off\n`); + setShapeOverlayDropped(true); + if (plainLoad.shapeOverlayState !== undefined) + restoreShapeOverlayState(plainLoad.shapeOverlayState); + finalLoad.dispose(); + finalLoad = plainLoad; + lowered = plain; + } else { + plainLoad.dispose(); + } + } + } + } + // A module the caller's IR validator rejects (a lowering bug the + // overlay's types reach) compiles without the overlay. + if ( + shapeOverlayActive() && + lowered.module !== null && + valid !== undefined && + !valid(lowered.module) + ) { + if (process.env["SCRIPTC_DEBUG_SHAPE_FALLBACK"] !== undefined) + process.stderr.write( + `[shape-fallback] the IR validator rejected the module; overlay off\n`, + ); + setShapeOverlayDropped(true); + finalLoad.dispose(); + finalLoad = loadProgram(entryPath, { npmStatic: finalNpmStatic, externalTypes }); + const reloadPreflight = checkPreflight(finalLoad); + if (reloadPreflight.length > 0) + return { ...lowered, module: null, diagnostics: reloadPreflight }; + lowered = lowerOnce(); + } + recordShapeOverlayMemo(plainFences); // The declaration-typed boundary's notes (SC6102) join the // performance warnings; they never fail a build. const notes = [ diff --git a/packages/compiler/src/frontend/program.ts b/packages/compiler/src/frontend/program.ts index 8884b0c83..74ccb01c8 100644 --- a/packages/compiler/src/frontend/program.ts +++ b/packages/compiler/src/frontend/program.ts @@ -154,6 +154,8 @@ import { shapeOverlayDone, shapeOverlayEnabled, shapeOverlayer, + captureShapeOverlayState, + type ShapeOverlayLoadState, } from "./shape-inference/index.js"; import { inferredJsDiagnosticSuppressed } from "./inferred-js-diagnostics.js"; import { @@ -372,6 +374,9 @@ export interface TypeWorldDifferences { export interface LoadResult { program: ts.Program; entry: ts.SourceFile; + /** Shape inference's overlay state for this load (restored before its + * lowering: a later load resets the per-process registry). */ + shapeOverlayState?: ShapeOverlayLoadState; /** User source files in Node's evaluation order: depth-first postorder * of the import graph from the entry, each module once. The entry is * last. Empty until checkPreflight runs (and empty when preflight fails @@ -1052,9 +1057,16 @@ function loadProgram7( if (overlayer !== undefined && jsx.refusal === null) { // Rounds: structural rewrites, then the typed overlay of the rebuilt // program (shape-inference/index.ts). + const timeShapes = process.env["SCRIPTC_DEBUG_SHAPE_TIME"] !== undefined; for (let round = 0; round < 3; round++) { + const t0 = timeShapes ? performance.now() : 0; const overlay = overlayer(program, round); + if (timeShapes) + process.stderr.write( + `[shape-time] round ${round} overlayer ${(performance.now() - t0).toFixed(0)} ms\n`, + ); if (overlay === null || overlay.size === 0) break; + const t1 = timeShapes ? performance.now() : 0; const before = program; // The tsc gate reads the files as written: its diagnostics are // taken before the overlay's files replace them on the host, and @@ -1073,6 +1085,10 @@ function loadProgram7( for (const [file, text] of overlay) host.addVirtualFile(file, text); program = host.createProgram([...programRoots, overridesDtsPath()], options); if (tscWorld.program !== before) before.dispose(); + if (timeShapes) + process.stderr.write( + `[shape-time] round ${round} gate+program ${(performance.now() - t1).toFixed(0)} ms\n`, + ); if (shapeOverlayDone()) break; } } @@ -1083,6 +1099,7 @@ function loadProgram7( return { program, entry, + shapeOverlayState: captureShapeOverlayState(), moduleOrder: [], configDiags: config.diags, ...(declBoundaryNotes.length > 0 ? { declBoundaryNotes } : {}), diff --git a/packages/compiler/src/frontend/shape-inference/analyze.ts b/packages/compiler/src/frontend/shape-inference/analyze.ts index 06f453fb4..670d4a2f8 100644 --- a/packages/compiler/src/frontend/shape-inference/analyze.ts +++ b/packages/compiler/src/frontend/shape-inference/analyze.ts @@ -198,6 +198,25 @@ const ARRAY_METHODS: ReadonlySet = new Set([ ]); const MAX_STEPS = 20_000_000; +/** The well-known symbols, static members of `Symbol` (Node 24). */ +const WELL_KNOWN_SYMBOLS: ReadonlySet = new Set([ + "asyncDispose", + "asyncIterator", + "dispose", + "hasInstance", + "isConcatSpreadable", + "iterator", + "match", + "matchAll", + "replace", + "search", + "species", + "split", + "toPrimitive", + "toStringTag", + "unscopables", +]); + /** The members String.prototype, Number.prototype and Boolean.prototype * define (Node 24); Object.prototype's come from OBJECT_PROTOTYPE_MEMBERS. */ const PRIMITIVE_MEMBERS: Readonly>> = { @@ -319,7 +338,15 @@ function propertyNameText(name: Node): string | null { return null; } +const shortFiles = new Map(); + function shortFile(fileName: string): string { + let short = shortFiles.get(fileName); + if (short === undefined) shortFiles.set(fileName, (short = shortFileOf(fileName))); + return short; +} + +function shortFileOf(fileName: string): string { const norm = fileName.split("\\").join("/"); const nm = norm.lastIndexOf("/node_modules/"); if (nm >= 0) { @@ -557,7 +584,9 @@ class Analyzer { this.files.push(sf); if (inPackage) this.packageFiles++; } - for (const sf of this.files) this.checker.prefetchSourceFile(sf); + // Analysis resolves names and never asks for types: the program it + // runs on is usually replaced by the overlay's before lowering. + for (const sf of this.files) this.checker.prefetchSourceFileSymbols(sf); for (const sf of this.files) this.moduleBody(sf); this.finish(); if (this.profiling) this.profileReport(); @@ -680,6 +709,26 @@ class Analyzer { * and which function each `new F()` instance atom belongs to. */ readonly protoOf = new Map(); readonly fnOfInstance = new Map(); + /** Per dictionary atom, the prototypes it was created with (Object.create). */ + private readonly dictProtos = new Map(); + /** Per object literal atom, the keys it has from its creation: another + * key may be absent when read (flow-insensitively, before its write). */ + private readonly literalOwnKeys = new Map>(); + private readonly protoReads = new Set(); + /** Arrays whose elements an index read inside its bounds may still miss: + * shrunk (pop, shift, splice, a length write), holey (`new Array(n)`, a + * delete, a store at an index that is not proven in bounds or the end). */ + private readonly unsafeArrays = new Set(); + /** Index reads proven in bounds by their loop (`for (let i = 0; i < + * a.length; i++) a[i]`): undefined joins them only if the array turns + * out unsafe (finish). */ + private readonly guardedIndexReads: { recv: Atom; dst: number; done: boolean }[] = []; + private readonly boundsMemo = new Map(); + dictProto(d: Atom): number { + let id = this.dictProtos.get(d); + if (id === undefined) this.dictProtos.set(d, (id = this.newNode())); + return id; + } protoAtom(fn: Atom): Atom { let p = this.protoOf.get(fn); if (p === undefined) { @@ -1482,6 +1531,15 @@ class Analyzer { this.edgeFor(site, info, args); } + /** The prototype object of a class declaration (`C.prototype`). */ + classProto(decl: Node): Atom { + return this.atoms.intern(`cproto:${this.nodeKey(decl)}`, () => ({ + kind: "object", + label: `${this.className(decl)}.prototype`, + node: decl, + })); + } + /** Per-call-site clones of init factories: a function called from * module initialization (top-level code outside loops, or another such * clone) is analyzed once per call site, so the objects it creates are @@ -1729,6 +1787,13 @@ class Analyzer { varDecl(d: ts.VariableDeclaration): void { const b = this.binding(d); if (d.initializer !== undefined) this.edge(this.expr(d.initializer), b); + else { + // `let x;` holds undefined until it is assigned (a for-in/for-of + // binding gets its values from the loop). + const loop = d.parent?.parent; + if (loop === undefined || !(ts.isForOfStatement(loop) || ts.isForInStatement(loop))) + this.add(b, this.prim("undefined")); + } this.bindPattern(d.name, b); } @@ -1787,7 +1852,15 @@ class Analyzer { const key = this.ctxKey(e); const memo = this.exprMemo.get(key); if (memo !== undefined) return memo; - const id = this.exprUncached(e); + let id = this.exprUncached(e); + // A member of an optional chain (`a?.b.c`, `f?.()`) is undefined when + // the chain short-circuits. + if ((e.flags & ts.NodeFlags.OptionalChain) !== 0) { + const out = this.newNode(); + this.edge(id, out); + this.add(out, this.prim("undefined")); + id = out; + } this.exprMemo.set(key, id); if (this.traceTop && !this.nodeLabel.has(id)) this.nodeLabel.set( @@ -1912,15 +1985,19 @@ class Analyzer { } if (ts.isPrefixUnaryExpression(e) || ts.isPostfixUnaryExpression(e)) { const operand = e.operand; - this.expr(operand); + const value = this.expr(operand); + if (ts.isPrefixUnaryExpression(e) && e.operator === ts.SyntaxKind.ExclamationToken) + return this.constNode(this.prim("boolean")); + // `+x` is a number (a bigint throws); `-x`, `~x`, `++`/`--` keep a bigint. + if (ts.isPrefixUnaryExpression(e) && e.operator === ts.SyntaxKind.PlusToken) + return this.constNode(this.prim("number")); + const result = this.numericResult([value]); if ( e.operator === ts.SyntaxKind.PlusPlusToken || e.operator === ts.SyntaxKind.MinusMinusToken ) - this.assignTo(operand, this.constNode(this.prim("number"))); - if (ts.isPrefixUnaryExpression(e) && e.operator === ts.SyntaxKind.ExclamationToken) - return this.constNode(this.prim("boolean")); - return this.constNode(this.prim("number")); + this.assignTo(operand, result); + return result; } if (ts.isSpreadElement(e)) return this.expr(e.expression); if (ts.isTaggedTemplateExpression(e)) return this.taggedTemplate(e); @@ -1948,6 +2025,8 @@ class Analyzer { if (!forwarding) fn.usesArguments = true; const arr = this.syntheticArray(id, "arguments"); for (const p of fn.params) this.edge(p, this.element(arr)); + // Arguments past the declared parameters are not tracked. + this.add(this.element(arr), this.top); return this.constNode(arr); } } @@ -2241,6 +2320,7 @@ class Analyzer { if (key !== null) this.edge(value, this.field(atom, key)); else this.edge(value, this.anyField(atom)); } + if (!this.literalOwnKeys.has(atom)) this.literalOwnKeys.set(atom, new Set(keys)); if (!this.siteAtoms.has(atom) && this.siteAtoms.add(atom)) this.sites.push({ atom, @@ -2451,16 +2531,11 @@ class Analyzer { return n; } if (op >= ts.SyntaxKind.FirstCompoundAssignment && op <= ts.SyntaxKind.LastCompoundAssignment) { - this.expr(e.left); - this.expr(e.right); - const result = this.constNode( - op === ts.SyntaxKind.PlusEqualsToken ? this.top : this.prim("number"), - ); - if (op === ts.SyntaxKind.PlusEqualsToken) { - this.nodes[result]!.atoms.clear(); - this.add(result, this.prim("string")); - this.add(result, this.prim("number")); - } + const lv = this.expr(e.left); + const rv = this.expr(e.right); + // Arithmetic on bigints answers bigints; `+=` may also concatenate. + const result = this.numericResult([lv, rv]); + if (op === ts.SyntaxKind.PlusEqualsToken) this.add(result, this.prim("string")); this.assignTo(e.left, result); return result; } @@ -2507,13 +2582,15 @@ class Analyzer { ) this.add(n, this.prim("number")); if (a.kind === "top") this.add(n, this.prim("number")); + if (a.kind === "top" || (a.kind === "prim" && a.label === "bigint")) + this.add(n, this.prim("bigint")); }; this.on(l, strOrNum); this.on(r, strOrNum); return n; } default: - return this.constNode(this.prim("number")); + return this.numericResult([l, r]); } } @@ -2617,6 +2694,8 @@ class Analyzer { this.unknownKeysInto(a, dst); return; } + // Members stored on a class's prototype object after the fact. + for (const c of this.classChain(decl)) this.edge(this.field(this.classProto(c), name), dst); const member = this.lookupMember(decl, name, false); if (member?.kind === "method") this.add( @@ -2640,7 +2719,9 @@ class Analyzer { case "class": { const decl = a.node!; if (name === "prototype" || name === "name") { - this.add(dst, name === "prototype" ? this.top : this.prim("string")); + // A class's prototype object: members stored on it later + // (`C.prototype.isVector3 = true`) are members of its instances. + this.add(dst, name === "prototype" ? this.classProto(decl) : this.prim("string")); this.fieldInto(a, name, dst); this.unknownKeysInto(a, dst); return; @@ -2686,8 +2767,10 @@ class Analyzer { return; case "array": if (name === "length") this.add(dst, this.prim("number")); - else if (/^\d+$/.test(name)) this.edge(this.element(a), dst); - else { + else if (/^\d+$/.test(name)) { + this.edge(this.element(a), dst); + this.add(dst, this.prim("undefined")); + } else { // A named property, written by name or under an unknown key; an own field shadows a method. if (ARRAY_METHODS.has(name)) { this.add(dst, this.builtin(`array.${name}`, a)); @@ -2697,8 +2780,19 @@ class Analyzer { } return; case "object": { + // A read through a class's prototype object (`C.prototype.m`) may + // be any member: unknown, as before prototype objects were tracked. + if ( + a.label.endsWith(".prototype") && + a.node !== undefined && + ts.isClassLikeDeclaration(a.node) + ) + this.add(dst, this.top); this.edge(this.field(a, name), dst); this.edge(this.anyField(a), dst); + const own = this.literalOwnKeys.get(a); + if (own !== undefined && !own.has(name) && !OBJECT_PROTOTYPE_MEMBERS.has(name)) + this.add(dst, this.prim("undefined")); if (this.namespaceAtoms.has(a) && name !== "exports") { this.loadConstraint(this.field(a, "exports"), name, dst, site); } @@ -2722,10 +2816,23 @@ class Analyzer { this.add(dst, this.builtin(`object.${name}`, a)); return; } - case "dictionary": + case "dictionary": { this.edge(this.anyField(a), dst); this.edge(this.field(a, name), dst); + // An absent key reads undefined, or what the prototype has. + this.add(dst, this.prim("undefined")); + const proto = this.dictProtos.get(a); + // Once per dictionary, name and destination: prototype chains can + // be cyclic in the analysis (`Object.create(this)`). + const key = `${a.id}:${name}:${dst}`; + if (proto !== undefined && !this.protoReads.has(key)) { + this.protoReads.add(key); + this.on(proto, (p) => { + if (p.kind !== "prim") this.loadAtom(p, name, dst, site); + }); + } return; + } case "promise": if (name === "then" || name === "catch" || name === "finally") this.add(dst, this.builtin(`promise.${name}`, a)); @@ -2761,8 +2868,13 @@ class Analyzer { ) this.add(dst, this.prim("number")); else if (path === "Math" && /^[A-Z0-9_]+$/.test(name)) this.add(dst, this.prim("number")); - else if (path === "Symbol") this.add(dst, this.prim("symbol")); - else if (path === "Error.instance" || path === "error") { + else if (path === "Symbol") { + // The well-known symbols are symbols; `Symbol.for` and `Symbol.keyFor` + // are functions (a call of a symbol would answer nothing). + if (WELL_KNOWN_SYMBOLS.has(name)) this.add(dst, this.prim("symbol")); + else if (name === "for" || name === "keyFor") this.add(dst, this.builtin(`Symbol.${name}`)); + else this.add(dst, this.top); + } else if (path === "Error.instance" || path === "error") { if (name === "message" || name === "stack" || name === "name") this.add(dst, this.prim("string")); else this.add(dst, this.top); @@ -2803,6 +2915,7 @@ class Analyzer { this.edge(value, this.field(a, name)); return; case "array": + if (/^\d+$/.test(name) || name === "length") this.unsafeArrays.add(a); if (/^\d+$/.test(name)) this.edge(value, this.element(a)); else if (name !== "length") this.edge(value, this.field(a, name)); return; @@ -2822,11 +2935,20 @@ class Analyzer { this.add(dst, this.top); return; case "array": + // An index past the end (or a hole) reads undefined, unless the + // loop proves the index in bounds of an array without holes. + this.edge(this.element(a), dst); + if (this.indexInBounds(site)) this.guardedIndexReads.push({ recv: a, dst, done: false }); + else this.add(dst, this.prim("undefined")); + return; case "collection": this.edge(this.element(a), dst); return; case "prim": - if (a.label === "string") this.add(dst, this.prim("string")); + if (a.label === "string") { + this.add(dst, this.prim("string")); + if (!this.indexInBounds(site)) this.add(dst, this.prim("undefined")); + } return; case "object": case "dictionary": @@ -2845,6 +2967,8 @@ class Analyzer { copy(); this.lateCopies.push(copy); if (a.kind === "instance" || a.kind === "class") this.add(dst, this.top); + // A key the object does not have reads undefined. + else this.add(dst, this.prim("undefined")); return; } default: @@ -2865,6 +2989,7 @@ class Analyzer { // An index, or a named property under an unknown key. this.edge(value, this.element(a)); this.edge(value, this.anyField(a)); + if (!this.indexInBounds(site) && !isAppendIndex(site)) this.unsafeArrays.add(a); return; case "prim": return; @@ -3106,6 +3231,8 @@ class Analyzer { case "builtin": this.constructBuiltin(a.path ?? "", args, dst, e); return; + case "prim": + return; // `new` of a primitive throws default: for (const x of args) this.sink(x, `argument of ⊤ constructor at ${this.loc(e)}`); this.add(dst, this.top); @@ -3162,7 +3289,10 @@ class Analyzer { this.add(dst, this.builtin("RegExp.instance")); return; case "Array": { - this.add(dst, this.syntheticArray(site, "Array")); + const arr = this.syntheticArray(site, "Array"); + // `new Array(n)` has holes. + this.unsafeArrays.add(arr); + this.add(dst, arr); return; } case "Proxy": @@ -3228,6 +3358,7 @@ class Analyzer { case "pop": case "shift": case "at": + if (method !== "at") this.unsafeArrays.add(recv); this.edge(el, dst); this.add(dst, this.prim("undefined")); return; @@ -3280,6 +3411,7 @@ class Analyzer { case "toSorted": case "toReversed": case "flat": { + if (method === "splice") this.unsafeArrays.add(recv); if (method === "filter") callback(0, [el, num, recvN], null); if (method === "sort" || method === "toSorted") callback(0, [el, el], null); if (method === "reverse" || method === "sort") { @@ -3554,6 +3686,9 @@ class Analyzer { label: `Object.create@${this.loc(site)}`, node: site, })); + // Its members are inherited from the prototype argument. + this.edge(arg(0), this.dictProto(d)); + if (args[1] !== undefined) this.sink(args[1], `property descriptors at ${locLabel(site)}`); this.add(dst, d); return; } @@ -3744,12 +3879,185 @@ class Analyzer { escapeAll(); } + /** Whether an index access `x[i]` reads inside the bounds of `x` on + * every evaluation: an enclosing `for (let i = ; i < x.length; + * i++ | ++i | i += 0>)` whose body assigns neither `i` nor `x`. + * (The array's own length may still shrink or have holes: unsafeArrays.) */ + indexInBounds(site: Node): boolean { + if (!ts.isElementAccessExpression(site) || site.questionDotToken !== undefined) return false; + const memo = this.boundsMemo.get(site); + if (memo !== undefined) return memo; + const out = this.indexInBoundsUncached(site); + this.boundsMemo.set(site, out); + return out; + } + + private indexInBoundsUncached(site: ts.ElementAccessExpression): boolean { + const obj = unparen(site.expression); + const key = unparen(site.argumentExpression); + if (!ts.isIdentifier(obj) || !ts.isIdentifier(key)) return false; + const objDecl = this.resolve(obj); + const keyDecl = this.resolve(key); + if (objDecl === undefined || keyDecl === undefined) return false; + const names = (id: Node, decl: Node): boolean => + ts.isIdentifier(id) && this.resolve(id) === decl; + for (let n: Node | undefined = site.parent; n !== undefined; n = n.parent) { + if (ts.isFunctionLike(n) || ts.isSourceFile(n)) return false; + if (!ts.isForStatement(n) || n.statement === undefined) continue; + if (!(site.pos >= n.statement.pos && site.end <= n.statement.end)) continue; + const cond = n.condition !== undefined ? unparen(n.condition) : undefined; + if ( + cond === undefined || + !ts.isBinaryExpression(cond) || + cond.operatorToken.kind !== ts.SyntaxKind.LessThanToken || + !names(unparen(cond.left), keyDecl) + ) + continue; + const bound = unparen(cond.right); + if ( + !ts.isPropertyAccessExpression(bound) || + bound.name.text !== "length" || + !names(unparen(bound.expression), objDecl) + ) + continue; + // The index starts at a non-negative integer literal ... + const init = n.initializer; + let start: ts.Expression | undefined; + // (declared by the loop, so only the loop's own code can write it) + if ( + init !== undefined && + ts.isVariableDeclarationList(init) && + (init.flags & (ts.NodeFlags.Let | ts.NodeFlags.Const)) !== 0 + ) { + const d = init.declarations.nodes.find((x) => x === keyDecl); + start = d !== undefined ? (d as ts.VariableDeclaration).initializer : undefined; + } + if (start === undefined || !nonNegativeIntegerLiteral(start)) continue; + if (!this.neverReassigned(objDecl)) continue; + // ... and only grows by integers, + const inc = n.incrementor !== undefined ? unparen(n.incrementor) : undefined; + const grows = + inc !== undefined && + (((ts.isPostfixUnaryExpression(inc) || ts.isPrefixUnaryExpression(inc)) && + inc.operator === ts.SyntaxKind.PlusPlusToken && + names(unparen(inc.operand), keyDecl)) || + (ts.isBinaryExpression(inc) && + inc.operatorToken.kind === ts.SyntaxKind.PlusEqualsToken && + names(unparen(inc.left), keyDecl) && + positiveIntegerLiteral(inc.right))); + if (!grows) continue; + // ... and the body writes neither the index nor the array binding. + let written = false; + const visit = (m: Node): void => { + if (written) return; + if ( + ts.isBinaryExpression(m) && + m.operatorToken.kind >= ts.SyntaxKind.FirstAssignment && + m.operatorToken.kind <= ts.SyntaxKind.LastAssignment && + (names(unparen(m.left), keyDecl) || names(unparen(m.left), objDecl)) + ) + written = true; + else if ( + (ts.isPostfixUnaryExpression(m) || ts.isPrefixUnaryExpression(m)) && + (m.operator === ts.SyntaxKind.PlusPlusToken || + m.operator === ts.SyntaxKind.MinusMinusToken) && + (names(unparen(m.operand), keyDecl) || names(unparen(m.operand), objDecl)) + ) + written = true; + ts.forEachChild(m, (c) => { + visit(c); + return undefined; + }); + }; + visit(n.statement); + if (!written) return true; + } + return false; + } + + private readonly reassignMemo = new Map(); + /** Whether a binding keeps the value it is declared with: a `const`, or a + * parameter or variable that no code in its scope assigns. */ + neverReassigned(decl: Node): boolean { + const memo = this.reassignMemo.get(decl); + if (memo !== undefined) return memo; + let out = false; + if (ts.isVariableDeclaration(decl) || ts.isParameter(decl)) { + const list = decl.parent; + if ( + ts.isVariableDeclaration(decl) && + list !== undefined && + ts.isVariableDeclarationList(list) && + (list.flags & ts.NodeFlags.Const) !== 0 + ) + out = true; + else { + let scope: Node | undefined = decl.parent; + while (scope !== undefined && !ts.isFunctionLike(scope) && !ts.isSourceFile(scope)) + scope = scope.parent; + if (scope !== undefined && ts.isIdentifier((decl as ts.VariableDeclaration).name)) { + const text = ((decl as ts.VariableDeclaration).name as ts.Identifier).text; + let written = false; + const target = (e: ts.Expression): boolean => { + const u = unparen(e); + return ts.isIdentifier(u) && u.text === text && this.resolve(u) === decl; + }; + const visit = (m: Node): void => { + if (written) return; + if ( + (ts.isBinaryExpression(m) && + m.operatorToken.kind >= ts.SyntaxKind.FirstAssignment && + m.operatorToken.kind <= ts.SyntaxKind.LastAssignment && + target(m.left)) || + ((ts.isPostfixUnaryExpression(m) || ts.isPrefixUnaryExpression(m)) && + (m.operator === ts.SyntaxKind.PlusPlusToken || + m.operator === ts.SyntaxKind.MinusMinusToken) && + target(m.operand)) + ) + written = true; + ts.forEachChild(m, (c) => { + visit(c); + return undefined; + }); + }; + visit(scope); + out = !written; + } + } + } + this.reassignMemo.set(decl, out); + return out; + } + + /** The result of arithmetic over the operands: a number, or a bigint + * where an operand may be one. */ + numericResult(operands: number[]): number { + const n = this.newNode(); + this.add(n, this.prim("number")); + for (const o of operands) + this.on(o, (a) => { + if (a.kind === "top" || (a.kind === "prim" && a.label === "bigint")) + this.add(n, this.prim("bigint")); + }); + return n; + } + /** Re-run deferred field copies until nothing new flows. */ finish(): void { for (;;) { const before = this.steps; for (const c of this.lateCopies) c(); this.solve(); + for (const o of this.ops) + if (o.kind === "delete" || o.kind === "lengthWrite") + for (const a of this.nodes[o.recv]!.atoms) + if (a.kind === "array") this.unsafeArrays.add(a); + for (const g of this.guardedIndexReads) + if (!g.done && this.unsafeArrays.has(g.recv)) { + g.done = true; + this.add(g.dst, this.prim("undefined")); + } + this.solve(); // A condition with no values at the fixpoint is one the analysis // could not evaluate: walk both of its branches. let walked = false; @@ -3770,6 +4078,30 @@ class Analyzer { } } +function nonNegativeIntegerLiteral(e: ts.Expression): boolean { + const u = unparen(e); + return ts.isNumericLiteral(u) && Number.isInteger(Number(u.text)) && Number(u.text) >= 0; +} + +function positiveIntegerLiteral(e: ts.Expression): boolean { + const u = unparen(e); + return ts.isNumericLiteral(u) && Number.isInteger(Number(u.text)) && Number(u.text) > 0; +} + +/** `x[x.length] = v`: a store at the end, which appends without a hole. */ +function isAppendIndex(site: Node): boolean { + if (!ts.isElementAccessExpression(site)) return false; + const obj = unparen(site.expression); + const key = unparen(site.argumentExpression); + return ( + ts.isIdentifier(obj) && + ts.isPropertyAccessExpression(key) && + key.name.text === "length" && + ts.isIdentifier(unparen(key.expression)) && + (unparen(key.expression) as ts.Identifier).text === obj.text + ); +} + /** The export name of a CommonJS export declaration the checker resolves * bindings to: a member of the `module.exports` literal, or the target of * `exports.name =` / `module.exports.name =`. */ diff --git a/packages/compiler/src/frontend/shape-inference/constant-graphs.ts b/packages/compiler/src/frontend/shape-inference/constant-graphs.ts index ceea474bd..a13ce850a 100644 --- a/packages/compiler/src/frontend/shape-inference/constant-graphs.ts +++ b/packages/compiler/src/frontend/shape-inference/constant-graphs.ts @@ -385,10 +385,14 @@ export function computeConstantGraphs(analysis: ShapeAnalysis): ConstantGraphs { mixedReads = 0; for (const op of analysis.ops) { if (op.kind !== "read" || op.name === null) continue; - const atoms = analysis.nodes[op.recv]!.atoms; - const constant = [...atoms].filter((a) => constantKeys.get(a)?.has(op.name!) === true); + // A nullish receiver throws (absent keys read undefined in the analysis). + const optional = (op.node.flags & ts.NodeFlags.OptionalChain) !== 0; + const atoms = [...analysis.nodes[op.recv]!.atoms].filter( + (a) => optional || !(a.kind === "prim" && (a.label === "undefined" || a.label === "null")), + ); + const constant = atoms.filter((a) => constantKeys.get(a)?.has(op.name!) === true); if (constant.length === 0) continue; - if (atoms.size === 1) { + if (atoms.length === 1) { foldableReads++; const p = op.node.parent; if (p !== undefined && ts.isCallExpression(p) && p.expression === op.node) foldableCalls++; diff --git a/packages/compiler/src/frontend/shape-inference/fallback.ts b/packages/compiler/src/frontend/shape-inference/fallback.ts index 50857fcfb..0e6f474b8 100644 --- a/packages/compiler/src/frontend/shape-inference/fallback.ts +++ b/packages/compiler/src/frontend/shape-inference/fallback.ts @@ -159,6 +159,42 @@ export function attributeOverlayRefusals( return calls.get(fn) ?? []; }; + // Annotations whose type text names an identifier, by identifier: a + // class the lowering refuses compiles dynamically until a type names it. + let naming: Map | null = null; + const blameNaming = (name: string): void => { + if (naming === null) { + naming = new Map(); + for (const file of overlayFiles) { + const text = program.getSourceFile(file)?.text; + if (text === undefined) continue; + for (const m of text.matchAll(/\/\*\*[\s\S]*?\*\//g)) { + if (!m[0].includes(OVERLAY_MARK)) continue; + const ids = overlayAnnotationIds(m[0]); + for (const w of new Set(m[0].match(/[A-Za-z_$][\w$]*/g) ?? [])) { + let l = naming.get(w); + if (l === undefined) naming.set(w, (l = [])); + for (const id of ids) l.push([file, id]); + } + } + } + } + for (const [file, id] of naming.get(name) ?? []) add(file, id); + }; + // The class a refusal is about: a field or the heritage of a class + // declaration, or a `new C(...)` of a named class. + const refusedClassName = (node: ts.Node): string | null => { + for (let n: ts.Node | undefined = node; n !== undefined; n = n.parent) { + if (ts.isNewExpression(n) && ts.isIdentifier(n.expression)) return n.expression.text; + if (ts.isFunctionLike(n)) return null; + if (ts.isClassLikeDeclaration(n)) { + const name = (n as { name?: ts.Node }).name; + return name !== undefined && ts.isIdentifier(name) ? name.text : null; + } + } + return null; + }; + for (const d of diagnostics) { const sf = program.getSourceFile(d.loc.file); if (sf === undefined) { @@ -175,6 +211,8 @@ export function attributeOverlayRefusals( blameNode(n); if (statement === node && (ts.isStatement(n) || isClassMember(n))) statement = n; } + const refusedClass = refusedClassName(node); + if (refusedClass !== null) blameNaming(refusedClass); const own = blamed; blameReferences(node, 0); if (blamed === own && statement !== node) blameReferences(statement, 0); @@ -235,7 +273,7 @@ function visitReturns(body: ts.Node, f: (e: ts.Expression) => void): void { } /** The innermost node whose token span covers [start, end). */ -function innermostNodeAt(sf: ts.SourceFile, start: number, end: number): ts.Node { +export function innermostNodeAt(sf: ts.SourceFile, start: number, end: number): ts.Node { let node: ts.Node = sf; for (;;) { let next: ts.Node | undefined; diff --git a/packages/compiler/src/frontend/shape-inference/index.ts b/packages/compiler/src/frontend/shape-inference/index.ts index 38dce5143..5f5547e39 100644 --- a/packages/compiler/src/frontend/shape-inference/index.ts +++ b/packages/compiler/src/frontend/shape-inference/index.ts @@ -9,20 +9,37 @@ import { isShapelessPackage } from "../package-shims.js"; import { analyzeShapes, isPackageJsFile, type ShapeAnalysis } from "./analyze.js"; import { createHash } from "node:crypto"; import { + hasShapeOverlayProtectedSpans, OVERLAY_MARK, + planOnRewrittenTexts, planShapeOverlay, renderShapeOverlay, type OverlayExclusions, type ShapeOverlayPlan, } from "./overlay.js"; import { attributeOverlayRefusals } from "./fallback.js"; -import { planStaticObjectRewrites, type StaticObjectRewrites } from "./static-objects.js"; +import { checksInferredFact } from "./inferred-checks.js"; +import { + planStaticObjectRewrites, + rewrittenPosition, + type StaticObjectRewrites, +} from "./static-objects.js"; import { isJsSourceFile } from "../program.js"; import { buildShapeReport } from "./report.js"; import { computeConstantGraphs, constantGraphReport } from "./constant-graphs.js"; +import { + overlayMemoKey, + readOverlayMemo, + shapeOverlayMemoEnabled, + textHash, + writeOverlayMemo, + type OverlayMemo, +} from "./memo.js"; export { analyzeShapes } from "./analyze.js"; export { buildShapeReport } from "./report.js"; +export { setShapeOverlayMemoDir } from "./memo.js"; +export { setShapeOverlayProtectedSpans } from "./overlay.js"; /** SCRIPTC_DEBUG_SHAPES: `1` reports package code in hot functions, * `verbose` every package site, `all` user files too, any other value only @@ -65,7 +82,43 @@ export function shapeOverlayAnnotationCount(): number { return overlayAnnotations; } +/** The overlay's per-load state (the registry is per process, and a later + * load resets it): a load captures it, and the lowering of that load + * restores it. */ +export interface ShapeOverlayLoadState { + readonly renderedFiles: ReadonlySet; + readonly staticRewriteFiles: ReadonlySet; + readonly staticObjectStats: typeof staticObjectStats; + readonly overlayDone: boolean; + readonly overlayFiles: ReadonlySet; + readonly overlayAnnotations: number; + readonly overlayDeadBranches: number; +} + +export function captureShapeOverlayState(): ShapeOverlayLoadState { + return { + renderedFiles, + staticRewriteFiles, + staticObjectStats, + overlayDone, + overlayFiles, + overlayAnnotations, + overlayDeadBranches, + }; +} + +export function restoreShapeOverlayState(state: ShapeOverlayLoadState): void { + renderedFiles = state.renderedFiles; + staticRewriteFiles = state.staticRewriteFiles; + staticObjectStats = state.staticObjectStats; + overlayDone = state.overlayDone; + overlayFiles = state.overlayFiles; + overlayAnnotations = state.overlayAnnotations; + overlayDeadBranches = state.overlayDeadBranches; +} + export function resetShapeOverlay(): void { + renderedFiles = new Set(); staticRewriteFiles = new Set(); staticObjectStats = { classes: 0, @@ -97,6 +150,18 @@ let exclusions: { annotations: Map>; files: Set } = }; let overlayDropped = false; let droppedAnnotations = 0; +/* The annotations dropped before a comparison load turned the overlay off + * for a moment (setShapeOverlayDropped). */ +let droppedBeforeComparison = 0; + +/* The build's memo (memo.ts): its key, what an earlier build recorded, the + * hashes of the JavaScript files as written, and the last overlay plan. */ +let memoKey: string | null = null; +let memo: OverlayMemo | null = null; +let memoOff = false; +let asWrittenHashes = new Map(); +let lastPlan: ShapeOverlayPlan | null = null; +let memoSeededPlan: ShapeOverlayPlan | null = null; /** Starts a build: no annotation is excluded yet. */ export function beginShapeOverlayBuild(): void { @@ -104,12 +169,111 @@ export function beginShapeOverlayBuild(): void { exclusions = { annotations: new Map(), files: new Set() }; overlayDropped = false; droppedAnnotations = 0; + droppedBeforeComparison = 0; + memoKey = null; + memo = null; + memoOff = false; + asWrittenHashes = new Map(); + lastPlan = null; + memoSeededPlan = null; +} + +/** Turns the overlay off for the rest of the build (true), or back on + * (false): the pipeline compares a load without it when the overlay's + * program fails a check the program without it passes. */ +export function setShapeOverlayDropped(dropped: boolean): void { + if (dropped && !overlayDropped) droppedBeforeComparison = droppedAnnotations; + overlayDropped = dropped; + droppedAnnotations = dropped ? overlayAnnotations : droppedBeforeComparison; +} + +/** The memo's deferred refusals of the build without the overlay, if an + * earlier build of the same JavaScript recorded them. */ +export function shapeOverlayMemoPlainFences(): ReadonlySet | null { + return memo === null || memo.plainFences.length === 0 ? null : new Set(memo.plainFences); +} + +/** Records the build's verdicts for the next build of the same + * JavaScript (memo.ts): the refused annotations and files, whether the + * build compiled without the overlay, and the deferred refusals of the + * build without it (`plainFences`, from a comparison). */ +export function recordShapeOverlayMemo(plainFences: Iterable | null): void { + if (memoKey === null || !shapeOverlayMemoEnabled() || memoOff) return; + const annotations: { file: string; hash: string; ids: number[] }[] = []; + for (const [file, ids] of exclusions.annotations) { + const part = lastPlan?.parts.get(file); + if (part === undefined || ids.size === 0) continue; + annotations.push({ file, hash: textHash(part.text), ids: [...ids].sort((a, b) => a - b) }); + } + annotations.sort((a, b) => (a.file < b.file ? -1 : a.file > b.file ? 1 : 0)); + const files: { file: string; hash: string }[] = []; + for (const file of [...exclusions.files].sort()) { + const hash = asWrittenHashes.get(file); + if (hash !== undefined) files.push({ file, hash }); + } + const plain = new Set(memo?.plainFences ?? []); + for (const k of plainFences ?? []) plain.add(k); + const next: OverlayMemo = { + off: overlayDropped, + annotations, + files, + plainFences: [...plain].sort(), + }; + if (memo !== null && JSON.stringify(memo) === JSON.stringify(next)) return; + writeOverlayMemo(memoKey, next); + memo = next; +} + +/* Round 0 of a load: the memo of the program's JavaScript, read once per + * key. A memo that says the build compiled without the overlay turns it + * off before any analysis; whole-file exclusions apply to files whose text + * is unchanged. */ +function readMemoFor(program: ts.Program): void { + if (!shapeOverlayMemoEnabled()) return; + const js = program.getSourceFiles().filter((sf) => !sf.isDeclarationFile && isJsSourceFile(sf)); + const key = overlayMemoKey(js.map((sf) => [sf.fileName, sf.text] as const)); + if (key === memoKey) return; + memoKey = key; + asWrittenHashes = new Map(js.map((sf) => [sf.fileName, textHash(sf.text)] as const)); + memo = readOverlayMemo(key); + memoOff = false; + if (memo === null) return; + if (memo.off) { + memoOff = true; + overlayDropped = true; + return; + } + for (const f of memo.files) + if (asWrittenHashes.get(f.file) === f.hash) exclusions.files.add(f.file); +} + +/* Applies the memo's refused annotations to a plan, once per plan: ids + * count only for a file whose overlay input text is unchanged. */ +function seedMemoAnnotations(plan: ShapeOverlayPlan): void { + if (memo === null || memoSeededPlan === plan) return; + memoSeededPlan = plan; + for (const a of memo.annotations) { + const part = plan.parts.get(a.file); + if (part === undefined || textHash(part.text) !== a.hash) continue; + let set = exclusions.annotations.get(a.file); + if (set === undefined) exclusions.annotations.set(a.file, (set = new Set())); + for (const id of a.ids) { + if (set.has(id)) continue; + set.add(id); + droppedAnnotations++; + } + } } /** Whether the current load has overlay files or structural rewrites. */ export function shapeOverlayActive(): boolean { const { classes, functions } = staticObjectStats; - return overlayFiles.size > 0 || classes + functions > 0; + return ( + overlayFiles.size > 0 || + renderedFiles.size > 0 || + staticRewriteFiles.size > 0 || + classes + functions > 0 + ); } /** The overlay's answer to refused lowering diagnostics: it excludes what @@ -127,7 +291,7 @@ export function excludeRefusedOverlay( level: "annotations" | "files" | "all" | "fences", ): boolean { if (overlayDropped) return false; - const files = new Set(overlayFiles); + const files = new Set([...overlayFiles, ...renderedFiles]); // Files with structural rewrites only also count as overlay files. for (const f of staticRewriteFiles) files.add(f); if (level === "all") { @@ -166,6 +330,8 @@ export function excludeRefusedOverlay( } let staticRewriteFiles: ReadonlySet = new Set(); +/** Every file the typed round rewrote (typed or only neutralized). */ +let renderedFiles: ReadonlySet = new Set(); function overlayExclusions(): OverlayExclusions { return exclusions; @@ -214,7 +380,10 @@ export function shapeOverlayer( program: ts.Program, round: number, ): ReadonlyMap | null { - if (round === 0) resetShapeOverlay(); + if (round === 0) { + resetShapeOverlay(); + readMemoFor(program); + } if (overlayDropped) return null; const key = `${programTextKey(program)}|${[...exclusions.files].sort().join(",")}`; const cached = overlayCache.get(key); @@ -223,9 +392,16 @@ export function shapeOverlayer( if (entry.budgetExceeded === true) return null; let analysis: ShapeAnalysis | null = null; const analyzed = (): ShapeAnalysis | null => { - analysis ??= analyzeShapes(program, program.getTypeChecker(), { - siteClones: overlaySiteClones(), - }); + if (analysis === null) { + const t0 = performance.now(); + analysis = analyzeShapes(program, program.getTypeChecker(), { + siteClones: overlaySiteClones(), + }); + if (process.env["SCRIPTC_DEBUG_SHAPE_TIME"] !== undefined) + process.stderr.write( + `[shape-time] round ${round} analysis ${(performance.now() - t0).toFixed(0)} ms\n`, + ); + } if (analysis.budgetExceeded) { entry.budgetExceeded = true; return null; @@ -240,7 +416,11 @@ export function shapeOverlayer( if (a === null) return null; entry.rewrites = planStaticObjectRewrites( a, - overlayJsFiles(a).filter((sf) => !exclusions.files.has(sf.fileName)), + // Files a project type patch edited stay as served: the patch's + // spans are offsets in that text. + overlayJsFiles(a).filter( + (sf) => !exclusions.files.has(sf.fileName) && !hasShapeOverlayProtectedSpans(sf.fileName), + ), ); } const rewrites = entry.rewrites; @@ -257,7 +437,35 @@ export function shapeOverlayer( for (const [file, text] of rewrites.files) process.stderr.write(`[static-objects] ${file}\n${text}\n`); } - if (rewrites.files.size > 0) return rewrites.files; + if (rewrites.files.size > 0) { + // Rewrites that only replace constant reads keep every declaration + // where it is: the typed overlay is planned on this analysis and + // moved onto the rewritten texts, so the program is rebuilt and + // analyzed once instead of twice. Hoisting classes or functions + // changes the declarations: the rewritten program is analyzed again. + const compose = + rewrites.classes + rewrites.functions === 0 && process.env["SCRIPTC_SHAPE_COMPOSE"] !== "0"; + if (!compose) return rewrites.files; + if (entry.plan === undefined) { + const a = analyzed(); + if (a === null) return null; + const report = buildShapeReport(a, "1"); + const plan = planShapeOverlay(a, report.sites, overlayJsFiles(a), { + annotations: new Map(), + files: exclusions.files, + }); + const moved = new Map number | null }>(); + for (const [file, text] of rewrites.files) { + const edits = rewrites.edits.get(file) ?? []; + moved.set(file, { text, map: (pos) => rewrittenPosition(edits, pos) }); + } + entry.plan = planOnRewrittenTexts(plan, moved); + } + const files = renderTyped(entry.plan); + for (const [file, text] of rewrites.files) if (!files.has(file)) files.set(file, text); + renderedFiles = new Set(files.keys()); + return files; + } } if (round > 1) return null; if (entry.plan === undefined) { @@ -269,8 +477,18 @@ export function shapeOverlayer( files: exclusions.files, }); } - const overlay = renderShapeOverlay(entry.plan, overlayExclusions()); - overlayFiles = new Set(overlay.files.keys()); + const files = renderTyped(entry.plan); + return files.size > 0 ? files : null; +} + +/* Renders a typed plan with the build's exclusions and registers the + * overlay files; the typed overlay is a load's last round. */ +function renderTyped(plan: ShapeOverlayPlan): Map { + lastPlan = plan; + seedMemoAnnotations(plan); + const overlay = renderShapeOverlay(plan, overlayExclusions()); + overlayFiles = new Set(overlay.typed); + renderedFiles = new Set(overlay.files.keys()); overlayAnnotations = overlay.annotations; overlayDeadBranches = overlay.deadBranches; if (process.env["SCRIPTC_DEBUG_SHAPE_OVERLAY"] !== undefined) { @@ -279,7 +497,7 @@ export function shapeOverlayer( } // The typed overlay is the last round. overlayDone = true; - return overlay.files.size > 0 ? overlay.files : null; + return overlay.files; } let staticObjectStats = { @@ -322,10 +540,14 @@ export function markInferredFactChecks( if (x === null || typeof x !== "object") return; const node = x as { kind?: unknown; - loc?: { file: string; start: number }; + loc?: { file: string; start: number; end: number }; inferredSite?: string; }; - if (node.kind === "dynCheck" && node.loc !== undefined && overlayFiles.has(node.loc.file)) { + if ( + node.kind === "dynCheck" && + node.loc !== undefined && + checksInferredFact(program, node.loc.file, node.loc.start, node.loc.end) + ) { const pkg = npmStaticPackageOfPath(node.loc.file) ?? ""; const rel = node.loc.file.split("/node_modules/").pop() ?? node.loc.file; node.inferredSite = `${pkg} ${rel}:${lineOf(node.loc.file, node.loc.start)}`; @@ -352,7 +574,9 @@ export function shapeOverlayReport(entryFile: string): ScrDiagnostic[] { // Annotations whose lowering was refused (fallback.ts); they compile // untyped. const fallback = overlayDropped - ? "the lowering refused code the overlay typed, so the build compiles without it" + ? memoOff + ? "an earlier build of the same JavaScript compiled without the overlay, so this build does too" + : "the lowering refused code the overlay typed, so the build compiles without it" : droppedAnnotations + exclusions.files.size > 0 ? `it dropped ${droppedAnnotations} annotation${droppedAnnotations === 1 ? "" : "s"}` + (exclusions.files.size > 0 diff --git a/packages/compiler/src/frontend/shape-inference/inferred-checks.ts b/packages/compiler/src/frontend/shape-inference/inferred-checks.ts new file mode 100644 index 000000000..d9c841034 --- /dev/null +++ b/packages/compiler/src/frontend/shape-inference/inferred-checks.ts @@ -0,0 +1,88 @@ +/* Which checked conversions check an inferred fact: those whose target type + * is an overlay annotation's (a parameter, return, binding or field the + * overlay typed, or a receiver cast). Only those report a failure as a + * shape analysis bug; every other check (a type the package's JSDoc or a + * declaration file declares, or the checker infers) keeps the default + * build's TypeError. */ +import * as ts from "../ts7/adapter.js"; +import { innermostNodeAt } from "./fallback.js"; +import { OVERLAY_MARK } from "./overlay.js"; + +/** Whether the overlay annotated a declaration (or the statement holding + * it). */ +function annotated(node: ts.Node | undefined): boolean { + for (let n = node, i = 0; n !== undefined && i < 4; n = n.parent, i++) { + const sf = n.getSourceFile(); + if (sf.text.slice(n.pos, ts.tokenStartOfNode(n)).includes(OVERLAY_MARK)) return true; + if ( + ts.isStatement(n) || + ts.isSourceFile(n) || + (n.parent !== undefined && ts.isClassLikeDeclaration(n.parent)) + ) + break; + } + return false; +} + +export function checksInferredFact( + program: ts.Program, + file: string, + start: number, + end: number, +): boolean { + const sf = program.getSourceFile(file); + if (sf === undefined) return false; + let e: ts.Node = innermostNodeAt(sf, start, Math.max(start, end)); + // A receiver cast: `/** @type {…} @scriptcShape */ (this)`. + for (let n: ts.Node | undefined = e; n !== undefined; n = n.parent) { + if (!ts.isParenthesizedExpression(n)) { + if (n !== e) break; + continue; + } + if (sf.text.slice(n.pos, ts.tokenStartOfNode(n)).includes(OVERLAY_MARK)) return true; + } + while (e.parent !== undefined && ts.isParenthesizedExpression(e.parent)) e = e.parent; + const p = e.parent; + if (p === undefined) return false; + if (ts.isReturnStatement(p)) { + let fn: ts.Node | undefined = p.parent; + while (fn !== undefined && !ts.isFunctionLike(fn)) fn = fn.parent; + return annotated(fn); + } + if (ts.isArrowFunction(p) && p.body === e) return annotated(p); + if (ts.isVariableDeclaration(p) && p.initializer === e) return annotated(p); + const checker = program.getTypeChecker(); + const declsOf = (n: ts.Node): readonly ts.Node[] => { + try { + return checker.getSymbolAtLocation(n)?.declarations ?? []; + } catch { + return []; + } + }; + if ((ts.isCallExpression(p) || ts.isNewExpression(p)) && p.expression !== e) { + let decl: ts.Node | undefined; + try { + decl = checker.getResolvedSignature(p)?.declaration ?? undefined; + } catch { + decl = undefined; + } + if (decl !== undefined && annotated(decl)) return true; + const callee = ts.isPropertyAccessExpression(p.expression) ? p.expression.name : p.expression; + return declsOf(callee).some((d) => annotated(d) || annotated(constructorOf(d))); + } + if ( + ts.isBinaryExpression(p) && + p.operatorToken.kind === ts.SyntaxKind.EqualsToken && + p.right === e + ) { + const target = ts.isPropertyAccessExpression(p.left) ? p.left.name : p.left; + return declsOf(target).some((d) => annotated(d)); + } + return false; +} + +/** A class declaration's constructor (`new C(…)` resolves to the class). */ +function constructorOf(decl: ts.Node): ts.Node | undefined { + if (!ts.isClassDeclaration(decl) && !ts.isClassExpression(decl)) return undefined; + return decl.members.nodes.find((m) => ts.isConstructorDeclaration(m)); +} diff --git a/packages/compiler/src/frontend/shape-inference/memo.ts b/packages/compiler/src/frontend/shape-inference/memo.ts new file mode 100644 index 000000000..8f5632a42 --- /dev/null +++ b/packages/compiler/src/frontend/shape-inference/memo.ts @@ -0,0 +1,129 @@ +/* The overlay's verdicts carried from one build to the next build of the + * same JavaScript (fallback.ts, pipeline.ts): + * - the annotations and files whose lowering was refused, so the next + * build renders the overlay without them instead of lowering, refusing + * and reloading again; + * - that the build compiled without the overlay, so the next build does + * not analyze at all; + * - the deferred refusals the build without the overlay has, so the next + * build need not lower the program without the overlay to compare. + * + * A memo is keyed by the compiler, the SCRIPTC_SHAPE_* settings and the + * text of every JavaScript file of the program; the program's TypeScript + * files are not part of the key, so editing them keeps the memo. Every + * verdict only makes the overlay smaller or answers a comparison about + * the same JavaScript, and a build still runs the fallback on whatever + * the lowering refuses. The memo directory lives in the build cache, so + * SCRIPTC_NO_CACHE=1 turns it off; it is never read or written without + * one. */ +import { createHash } from "node:crypto"; +import { mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; + +export interface OverlayMemo { + /** The build compiled without the overlay. */ + readonly off: boolean; + /** Refused annotation ids per file, valid for the overlay input text + * with this hash. */ + readonly annotations: readonly MemoFileIds[]; + /** Files whose whole overlay was refused, by their text as written. */ + readonly files: readonly MemoFile[]; + /** Deferred refusals of the build without the overlay + * (`code|file|line`). */ + readonly plainFences: readonly string[]; +} + +export interface MemoFile { + readonly file: string; + readonly hash: string; +} + +export interface MemoFileIds { + readonly file: string; + readonly hash: string; + readonly ids: readonly number[]; +} + +let memoDir: string | null = null; + +/** Where memos live (a directory of the build cache, per compiler), or + * null for none. The compile driver sets it per build. */ +export function setShapeOverlayMemoDir(dir: string | null): void { + memoDir = dir; +} + +export function shapeOverlayMemoEnabled(): boolean { + return memoDir !== null; +} + +export function textHash(text: string): string { + return createHash("sha256").update(text).digest("hex").slice(0, 32); +} + +/** The memo key of a program's JavaScript files (name and text). */ +export function overlayMemoKey(files: readonly (readonly [string, string])[]): string { + const hash = createHash("sha256"); + const settings = Object.keys(process.env) + .filter((k) => k.startsWith("SCRIPTC_SHAPE_") && !k.startsWith("SCRIPTC_SHAPE_CHECK")) + .sort(); + for (const k of settings) + hash + .update(k) + .update("=") + .update(process.env[k] ?? "") + .update("\0"); + const sorted = [...files].sort((a, b) => (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0)); + for (const [name, text] of sorted) hash.update(name).update("\0").update(text).update("\0"); + return hash.digest("hex").slice(0, 40); +} + +export function readOverlayMemo(key: string): OverlayMemo | null { + if (memoDir === null) return null; + try { + const raw: unknown = JSON.parse(readFileSync(join(memoDir, `${key}.json`), "utf8")); + return isOverlayMemo(raw) ? raw : null; + } catch { + return null; + } +} + +export function writeOverlayMemo(key: string, memo: OverlayMemo): void { + if (memoDir === null) return; + try { + mkdirSync(memoDir, { recursive: true, mode: 0o700 }); + const path = join(memoDir, `${key}.json`); + const temp = `${path}.${process.pid}.${Date.now()}.tmp`; + writeFileSync(temp, JSON.stringify(memo), { mode: 0o600 }); + renameSync(temp, path); + } catch { + // The memo only saves work; a build that cannot write it is unaffected. + } +} + +function isOverlayMemo(raw: unknown): raw is OverlayMemo { + if (raw === null || typeof raw !== "object") return false; + const m = raw as Record; + const files = m["files"]; + const annotations = m["annotations"]; + const plain = m["plainFences"]; + return ( + typeof m["off"] === "boolean" && + Array.isArray(files) && + files.every(isMemoFile) && + Array.isArray(annotations) && + annotations.every( + (a: unknown) => + isMemoFile(a) && + Array.isArray((a as Record)["ids"]) && + ((a as Record)["ids"] as unknown[]).every((i) => typeof i === "number"), + ) && + Array.isArray(plain) && + plain.every((s: unknown) => typeof s === "string") + ); +} + +function isMemoFile(raw: unknown): boolean { + if (raw === null || typeof raw !== "object") return false; + const f = raw as Record; + return typeof f["file"] === "string" && typeof f["hash"] === "string"; +} diff --git a/packages/compiler/src/frontend/shape-inference/overlay.test.ts b/packages/compiler/src/frontend/shape-inference/overlay.test.ts deleted file mode 100644 index f0effae77..000000000 --- a/packages/compiler/src/frontend/shape-inference/overlay.test.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { afterAll, expect, test } from "vitest"; -import { closeSourceParser, parseSourceFile } from "../ts7/source-parser-node.js"; -import { neutralizeDocTags, OVERLAY_MARK } from "./overlay.js"; - -afterAll(closeSourceParser); - -test("neutralizeDocTags rewrites JSDoc comments and keeps literals", () => { - const source = [ - "/** @param {number} a */", - "function f(a) {", - ' const s = "/** @type {number} */";', - " const t = `/** @returns {string} ${a} */`;", - " const r = /\\/\\*\\* @param \\*\\//;", - " // /** @type {string} */", - ` /** @type {string} ${OVERLAY_MARK} */ const u = s + t;`, - " return a / 2 /** @returns {number} */;", - "}", - ].join("\n"); - const out = neutralizeDocTags(parseSourceFile("overlay.js", source, "js")); - expect(out.length).toBe(source.length); - expect(out.split("\n")).toEqual([ - "/** -param {number} a */", - "function f(a) {", - ' const s = "/** @type {number} */";', - " const t = `/** @returns {string} ${a} */`;", - " const r = /\\/\\*\\* @param \\*\\//;", - " // /** @type {string} */", - ` /** @type {string} ${OVERLAY_MARK} */ const u = s + t;`, - " return a / 2 /** -returns {number} */;", - "}", - ]); -}); diff --git a/packages/compiler/src/frontend/shape-inference/overlay.ts b/packages/compiler/src/frontend/shape-inference/overlay.ts index a0d28b87c..a39e0f318 100644 --- a/packages/compiler/src/frontend/shape-inference/overlay.ts +++ b/packages/compiler/src/frontend/shape-inference/overlay.ts @@ -11,8 +11,10 @@ * meets an inferred type therefore fails only when the analysis is wrong; * such checks carry the inferred-fact failure message. * - * The package's own JSDoc is a hint, not a declaration: its tags are - * neutralized in the rewritten file, so only the analysis's types remain. + * The package's own JSDoc is a declaration (maintainer decision + * 2026-10-25): it stays as written, and the overlay adds a type only where + * the package's JSDoc declares none (a documented parameter, return or + * binding keeps its documented type, as in the default build). * * Every edit is inline: no line breaks are added or removed, so positions * keep their lines. */ @@ -28,6 +30,8 @@ export interface ShapeOverlay { readonly annotations: number; /** Branches the analysis proved unreachable, replaced by checks. */ readonly deadBranches: number; + /** The files with annotations or dead-branch checks. */ + readonly typed: ReadonlySet; } interface Edit { @@ -51,27 +55,50 @@ interface OverlayPart { readonly text: string; readonly edits: Edit[]; readonly suffix: string; - /** The leading trivia of each annotated declaration, by annotation id: - * the package's own JSDoc there is neutralized while the annotation is - * kept (package JSDoc is a hint; elsewhere it types the code as it does - * without the overlay). */ - readonly docRanges: DocRange[]; } -interface DocRange { - readonly lo: number; - readonly hi: number; - readonly id: number; - /** For a function's annotation, the tags it writes: only the package's - * tags for the same result and parameters (and a `@type` or `@overload` - * signature) are neutralized, so the parameters it leaves untyped keep - * their documented types. Unset: every type tag in the range is. */ - readonly replaces?: FunctionTags; +/** Spans of a served package file that a project type patch edited (the + * type-patches layer's `typePatchedSpans`, in served-text offsets). */ +export type ProtectedSpans = (fileName: string) => readonly { start: number; end: number }[]; + +let protectedSpansOf: ProtectedSpans = () => []; + +/** Installs the source of patched spans: the overlay neither annotates nor + * rewrites a declaration a project type patch edited, since the patch is + * the project's own declaration of its type. */ +export function setShapeOverlayProtectedSpans(spans: ProtectedSpans | null): void { + protectedSpansOf = spans ?? (() => []); } -interface FunctionTags { - readonly returns: boolean; - readonly params: readonly string[]; +/** Whether a file has patched spans (static-objects.ts leaves it as served, + * so the spans keep their offsets). */ +export function hasShapeOverlayProtectedSpans(fileName: string): boolean { + return protectedSpansOf(fileName).length > 0; +} + +/** Whether the declaration whose first token starts at `pos` overlaps a + * patched span: its leading comments and its header (up to its body). */ +export function declarationPatched( + sf: ts.SourceFile, + pos: number, + spans: readonly { start: number; end: number }[], +): boolean { + if (spans.length === 0) return false; + let node: ts.Node = sf; + for (;;) { + let next: ts.Node | undefined; + ts.forEachChild(node, (c) => { + if (next === undefined && c.pos <= pos && c.end > pos) next = c; + return undefined; + }); + if (next === undefined || ts.tokenStartOfNode(next) > pos) break; + node = next; + if (ts.tokenStartOfNode(node) === pos) break; + } + if (node === sf) return false; + const body = (node as { body?: ts.Node }).body; + const end = body !== undefined ? ts.tokenStartOfNode(body) : node.end; + return spans.some((span) => span.start < end && span.end > node.pos); } /** Marks the overlay's own comments (an unknown JSDoc tag, ignored). */ @@ -114,13 +141,24 @@ export function planShapeOverlay( packageFiles: readonly ts.SourceFile[], exclusions: OverlayExclusions = NO_EXCLUSIONS, ): ShapeOverlayPlan { + // Debugging: SCRIPTC_DEBUG_SHAPE_FIELD=name prints that field's values per atom. + const debugField = process.env["SCRIPTC_DEBUG_SHAPE_FIELD"]; + if (debugField !== undefined) + for (const [atom, fields] of analysis.fields) { + const id = fields.get(debugField); + if (id !== undefined) + process.stderr.write( + `[shape-field] ${atom.kind}:${atom.label}.${debugField} = ${[...analysis.nodes[id]!.atoms].map((a) => `${a.kind}:${a.label}`).join(" | ")}\n`, + ); + } const pSites = new Map(); const siteVerdicts = new Set(verdicts.map((v) => v.site.atom)); + const dynHeld = dynHeldSites(analysis, verdicts); // Records and arrays print only for sites whose objects never escape: a // native record that reached untyped code would be seen as a copy. const siteKeys = new Map }>(); for (const v of verdicts) { - if (v.verdict !== "P-local") continue; + if (v.verdict !== "P-local" || dynHeld.has(v.site.atom)) continue; if (v.site.kind === "array") { pSites.set(v.site.atom, v); continue; @@ -148,6 +186,19 @@ export function planShapeOverlay( list.push(branch); } } + // Classes whose instances meet as the receivers of one member read, by + // member name: their same-named methods need one signature. + const cooccurring = new Map[]>(); + for (const op of analysis.ops) { + if (op.kind !== "read" || op.name === null) continue; + const classes = new Set(); + for (const a of analysis.nodes[op.recv]!.atoms) + if (a.kind === "instance" && a.node !== undefined) classes.add(a.node); + if (classes.size < 2) continue; + let l = cooccurring.get(op.name); + if (l === undefined) cooccurring.set(op.name, (l = [])); + l.push(classes); + } for (const sf of packageFiles) { if (exclusions.files.has(sf.fileName)) continue; const typedefs = new Map(); @@ -156,21 +207,10 @@ export function planShapeOverlay( // One annotation: a JSDoc comment at `pos` carrying MARK and the // annotation's id (its position), so a refused lowering can name it and // the next load drop it. `close`: the text that closes a type cast. - const docRanges: DocRange[] = []; - const annotate = ( - pos: number, - tags: string, - close?: number, - from?: number, - replaces?: FunctionTags, - ): void => { - if (from !== undefined && from < pos) - docRanges.push({ - lo: from, - hi: pos, - id: pos, - ...(replaces !== undefined ? { replaces } : {}), - }); + const patchedSpans = protectedSpansOf(sf.fileName); + const annotate = (pos: number, tags: string, close?: number): void => { + // A declaration a project type patch edited keeps the patch alone. + if (declarationPatched(sf, pos, patchedSpans)) return; edits.push({ pos, id: pos, @@ -295,6 +335,9 @@ export function planShapeOverlay( // Arms, not atoms: per-site allocations of one record shape are one // typedef. if (arms.size > MAX_ARMS) return inner ? "any" : null; + // Arithmetic on unknown values may answer a number or a bigint: the + // lowering has no useful mixed representation, so those stay untyped. + if (arms.has("number") && arms.has("bigint")) return inner ? "any" : null; if (arms.has("any")) return inner ? "any" : null; return [...arms].join(" | "); }; @@ -435,9 +478,14 @@ export function planShapeOverlay( if (method.body !== undefined) for (const st of method.body.statements.nodes) scan(st); }; - // Same-named instance methods of the file's classes form a family: a - // call on a union of those classes needs one signature, so each member's - // parameters and result join the whole family's values. + // Same-named instance methods of the file's classes form a family when + // a call can reach several of them: a call on a union of those classes + // needs one signature, so each member's parameters and result join the + // whole family's values. Two members join when their classes' instances + // meet as the receivers of one read of the method, or when one class + // extends the other (an override keeps its base's signature). Members + // no union reaches keep their own values: ZodString's `_addCheck` + // returns a ZodString, not every schema class's `_addCheck` result. const families = new Map(); const familyVisit = (node: ts.Node): void => { if ( @@ -457,6 +505,84 @@ export function planShapeOverlay( }); }; familyVisit(sf); + const familyParent = new Map(); + const familyFind = (m: ts.MethodDeclaration): ts.MethodDeclaration => { + let r = m; + for (let p = familyParent.get(r); p !== undefined && p !== r; p = familyParent.get(r)) r = p; + return r; + }; + const familyUnion = (a: ts.MethodDeclaration, b: ts.MethodDeclaration): void => { + const ra = familyFind(a); + const rb = familyFind(b); + if (ra !== rb) familyParent.set(rb, ra); + }; + const classNameOf = (c: ts.Node): string | null => { + const named = c as { name?: ts.Node }; + if (named.name !== undefined && ts.isIdentifier(named.name)) return named.name.text; + const p = c.parent; + return p !== undefined && ts.isVariableDeclaration(p) && ts.isIdentifier(p.name) + ? p.name.text + : null; + }; + const baseNameOf = (c: ts.Node): string | null => { + const clauses = (c as { heritageClauses?: { nodes: readonly ts.HeritageClause[] } }) + .heritageClauses; + for (const h of list(clauses)) { + if (h.token !== ts.SyntaxKind.ExtendsKeyword) continue; + const t = list(h.types)[0]; + if (t !== undefined && ts.isIdentifier(t.expression)) return t.expression.text; + } + return null; + }; + for (const [name, members] of families) { + if (members.length < 2) continue; + // The member a class's instances run: its own, or its nearest base's + // (by name, within the file). + const byClass = new Map(); + for (const m of members) if (m.parent !== undefined) byClass.set(m.parent, m); + const classesByName = new Map(); + const collect = (node: ts.Node): void => { + if (ts.isClassDeclaration(node) || ts.isClassExpression(node)) { + const n = classNameOf(node); + if (n !== null && !classesByName.has(n)) classesByName.set(n, node); + } + ts.forEachChild(node, (c) => { + collect(c); + return undefined; + }); + }; + const memberOf = (c: ts.Node): ts.MethodDeclaration | undefined => { + let cur: ts.Node | undefined = c; + for (let depth = 0; cur !== undefined && depth < 16; depth++) { + const own = byClass.get(cur); + if (own !== undefined) return own; + const base = baseNameOf(cur); + if (base === null) return undefined; + if (classesByName.size === 0) collect(sf); + cur = classesByName.get(base); + } + return undefined; + }; + // Overrides join their base's member. + for (const m of members) { + const base = m.parent === undefined ? null : baseNameOf(m.parent); + if (base === null) continue; + if (classesByName.size === 0) collect(sf); + const baseClass = classesByName.get(base); + const inherited = baseClass === undefined ? undefined : memberOf(baseClass); + if (inherited !== undefined) familyUnion(inherited, m); + } + for (const classes of cooccurring.get(name) ?? []) { + let first: ts.MethodDeclaration | undefined; + for (const c of classes) { + if (c.getSourceFile() !== sf) continue; + const m = memberOf(c); + if (m === undefined) continue; + if (first === undefined) first = m; + else familyUnion(first, m); + } + } + } // Members the program runs (they have analysis contexts); members it // never reaches would only widen the family's types. const reached = new Set(); @@ -465,7 +591,10 @@ export function planShapeOverlay( const familyOf = (fn: ts.FunctionLikeDeclaration): readonly ts.MethodDeclaration[] => { if (!ts.isMethodDeclaration(fn) || !ts.isIdentifier(fn.name)) return [fn as ts.MethodDeclaration]; - const members = (families.get(fn.name.text) ?? [fn]).filter((m) => reached.has(m)); + const root = familyFind(fn as ts.MethodDeclaration); + const members = (families.get(fn.name.text) ?? [fn]).filter( + (m) => reached.has(m) && familyFind(m) === root, + ); return members.length > 0 ? members : [fn]; }; const paramValues = (fn: ts.FunctionLikeDeclaration, index: number): Set => { @@ -488,13 +617,13 @@ export function planShapeOverlay( return out; }; - const paramTags = ( - fn: ts.FunctionLikeDeclaration, - ): { text: string; replaces: FunctionTags } | null => { + const paramTags = (fn: ts.FunctionLikeDeclaration): string | null => { const tags: string[] = []; - const params: string[] = []; - let returns = false; - if (!ts.isConstructorDeclaration(fn) && !constructed.has(fn)) { + // The package's JSDoc declares types: the overlay adds none where it + // has one, and none at all for a signature it types as a whole. + const doc = packageDocOf(fn); + if (doc.whole) return null; + if (!ts.isConstructorDeclaration(fn) && !constructed.has(fn) && !doc.returns) { const ret = returnValues(fn); const t = ret !== undefined && ret.size > 0 ? typeOf(ret, false, 0) : null; const why = process.env["SCRIPTC_DEBUG_SHAPE_OVERLAY_WHY"]; @@ -509,10 +638,7 @@ export function planShapeOverlay( process.stderr.write( `[shape-overlay-why] ${fn.name.text}: ${[...ret].map((a) => `${a.kind}:${a.label}${analysis.escaped.has(a) ? "(escaped)" : ""}`).join(" | ")}\n`, ); - if (t !== null) { - tags.push(`@returns {${t}}`); - returns = true; - } + if (t !== null) tags.push(`@returns {${t}}`); } list(fn.parameters).forEach((p, index) => { if ( @@ -528,12 +654,10 @@ export function planShapeOverlay( // narrow the typed parameter to never inside the branch, which the // lowering refuses: such a parameter stays untyped. if (t !== null && typeofTestsOutside(fn, p.name.text, values)) return; - if (t !== null) { - tags.push(`@param {${t}} ${p.name.text}`); - params.push(p.name.text); - } + if (doc.params.has(p.name.text)) return; + if (t !== null) tags.push(`@param {${t}} ${p.name.text}`); }); - return tags.length > 0 ? { text: tags.join(" "), replaces: { returns, params } } : null; + return tags.length > 0 ? tags.join(" ") : null; }; const visit = (node: ts.Node): void => { @@ -551,8 +675,7 @@ export function planShapeOverlay( ) { if (node.body !== undefined) { const tags = paramTags(node); - if (tags !== null) - annotate(ts.tokenStartOfNode(node), tags.text, undefined, node.pos, tags.replaces); + if (tags !== null) annotate(ts.tokenStartOfNode(node), tags); } // `this.f = …` fields of a constructor's top-level statements. if (ts.isConstructorDeclaration(node) && node.body !== undefined) { @@ -575,7 +698,8 @@ export function planShapeOverlay( if (seen.has(name)) continue; seen.add(name); const t = cls !== undefined ? fieldType(cls, name) : null; - if (t !== null) annotate(ts.tokenStartOfNode(st), `@type {${t}}`, undefined, st.pos); + if (t !== null && !packageDocOf(st).type) + annotate(ts.tokenStartOfNode(st), `@type {${t}}`); } } } @@ -591,7 +715,12 @@ export function planShapeOverlay( const init = d.initializer; if (ts.isIdentifier(d.name) && init !== undefined && !literalInitializer(init)) { const t = typeOf(analysis.declValues(d), false, 0); - if (t !== null) annotate(ts.tokenStartOfNode(node), `@type {${t}}`, undefined, node.pos); + if (process.env["SCRIPTC_DEBUG_SHAPE_OVERLAY_WHY"] === d.name.text) + process.stderr.write( + `[shape-overlay-why] ${d.name.text}@${sf.getLineAndCharacterOfPosition(ts.tokenStartOfNode(d)).line + 1}: ${t} <- ${[...analysis.declValues(d)].map((a) => `${a.kind}:${a.label}`).join(" | ")}\n`, + ); + if (t !== null && !packageDocOf(node).type) + annotate(ts.tokenStartOfNode(node), `@type {${t}}`); } } ts.forEachChild(node, (c) => { @@ -667,8 +796,8 @@ export function planShapeOverlay( edits.length = 0; edits.push(...kept); } - if (edits.length === 0) continue; - parts.set(sf.fileName, { text: sf.text, edits, suffix: typedefText.join(""), docRanges }); + if (edits.length === 0 && typedefText.length === 0) continue; + parts.set(sf.fileName, { text: sf.text, edits, suffix: typedefText.join("") }); } // Type aliases other files name these files' classes through. for (const [other, names] of exportedClasses) { @@ -684,18 +813,56 @@ export function planShapeOverlay( text: other.text, edits: [], suffix: `\n${aliases}`, - docRanges: [], }); } return { parts }; } +/** The plan moved onto texts that structural rewrites changed: each + * annotation and dead-branch edit maps through the rewrite's edits, and an + * annotation whose position the rewrite replaced is left out (with its + * closing parenthesis). `rewritten` gives a rewritten file's text and its + * position map. */ +export function planOnRewrittenTexts( + plan: ShapeOverlayPlan, + rewritten: ReadonlyMap number | null }>, +): ShapeOverlayPlan { + const parts = new Map(); + for (const [file, part] of plan.parts) { + const r = rewritten.get(file); + if (r === undefined) { + parts.set(file, part); + continue; + } + const lost = new Set(); + const moved: Edit[] = []; + for (const e of part.edits) { + const pos = r.map(e.pos); + const end = e.end === undefined ? undefined : r.map(e.end); + if (pos === null || end === null || (end !== undefined && end < pos)) { + if (e.id !== undefined) lost.add(e.id); + continue; + } + if (e.end !== undefined && end !== undefined && end - pos !== e.end - e.pos) { + // A rewrite inside a replaced dead branch. + if (e.id !== undefined) lost.add(e.id); + continue; + } + moved.push(end === undefined ? { ...e, pos } : { ...e, pos, end }); + } + const edits = moved.filter((e) => e.id === undefined || !lost.has(e.id)); + parts.set(file, { text: r.text, edits, suffix: part.suffix }); + } + return { parts }; +} + /** The overlay's files with the annotations a build dropped left out. */ export function renderShapeOverlay( plan: ShapeOverlayPlan, exclusions: OverlayExclusions = NO_EXCLUSIONS, ): ShapeOverlay { const files = new Map(); + const typed = new Set(); let annotations = 0; let deadBranches = 0; for (const [file, part] of plan.parts) { @@ -707,84 +874,201 @@ export function renderShapeOverlay( else annotations++; } if (edits.length === 0 && part.suffix === "") continue; - files.set(file, applyEdits(neutralizedDocRanges(part, dropped), edits) + part.suffix); + const text = applyEdits(part.text, edits) + part.suffix; + files.set(file, text); + if (edits.some((e) => e.id !== undefined)) typed.add(file); } - return { files, annotations, deadBranches }; + return { files, typed, annotations, deadBranches }; } -/** The part's text with the package JSDoc neutralized in the leading trivia - * of the annotations a build kept. Each range is one declaration's leading - * trivia, so two ranges are equal or disjoint; the text is built once, with - * the tags of all the annotations of a range neutralized. */ -function neutralizedDocRanges(part: OverlayPart, dropped: ReadonlySet | undefined): string { - const ranges = part.docRanges - .filter((r) => dropped?.has(r.id) !== true) - .sort((a, b) => a.lo - b.lo || a.hi - b.hi); - if (ranges.length === 0) return part.text; - let out = ""; - let at = 0; - for (let i = 0; i < ranges.length; i++) { - const r = ranges[i]!; - let replaces = r.replaces; - for (; i + 1 < ranges.length && ranges[i + 1]!.lo === r.lo && ranges[i + 1]!.hi === r.hi; i++) { - const next = ranges[i + 1]!.replaces; - replaces = - replaces === undefined || next === undefined - ? undefined - : { - returns: replaces.returns || next.returns, - params: [...replaces.params, ...next.params], - }; +/** Literal sites whose objects some holder keeps in the lowering's dynamic + * representation: a module-level binding of a JavaScript file (the overlay + * does not type those, and the lowering keeps their objects dynamic), and, + * transitively, the fields and elements of those objects and of every + * object or array that is not P-local. A typed position that receives such + * an object converts it to a native record, which is a copy: identity + * comparisons and writes through either reference would diverge. */ +function dynHeldSites(analysis: ShapeAnalysis, verdicts: readonly SiteVerdict[]): Set { + const held = new Set(); + const work: Atom[] = []; + const add = (a: Atom): void => { + if ((a.kind === "object" || a.kind === "array") && !held.has(a)) { + held.add(a); + work.push(a); + } + }; + for (const sf of analysis.files) { + if (!/\.[cm]?jsx?$/.test(sf.fileName)) continue; + for (const st of sf.statements.nodes) { + if (!ts.isVariableStatement(st)) continue; + for (const d of st.declarationList.declarations.nodes) + for (const a of analysis.declValues(d)) add(a); } - const lo = Math.max(r.lo, at); - if (r.hi <= lo) continue; - out += part.text.slice(at, lo) + neutralizeCommentTags(part.text.slice(lo, r.hi), replaces); - at = r.hi; } - return out + part.text.slice(at); + const pLocal = new Set(); + for (const v of verdicts) if (v.verdict === "P-local") pLocal.add(v.site.atom); + for (const a of analysis.atoms.atoms) + if ((a.kind === "object" || a.kind === "array") && !pLocal.has(a)) work.push(a); + const seen = new Set(); + while (work.length > 0) { + const a = work.pop()!; + if (seen.has(a)) continue; + seen.add(a); + for (const id of analysis.fields.get(a)?.values() ?? []) + for (const x of analysis.nodes[id]!.atoms) add(x); + const el = analysis.elements.get(a); + if (el !== undefined) for (const x of analysis.nodes[el]!.atoms) add(x); + } + return held; +} + +interface PackageDoc { + /** Documented parameter names. */ + readonly params: ReadonlySet; + readonly returns: boolean; + readonly type: boolean; + /** A tag that types the whole declaration (`@type`, `@template`, + * `@this`, `@overload`, `@callback`). */ + readonly whole: boolean; +} + +/** The type tags of the package's own JSDoc for a declaration: in its + * leading trivia and in that of the statement or property holding it + * (`/** … *\/ const f = function (…) {}`, `exports.f = function …`). */ +function packageDocOf(node: ts.Node): PackageDoc { + let text = ""; + for (let n: ts.Node | undefined = node; n !== undefined; n = n.parent) { + const sf = n.getSourceFile(); + for (const m of sf.text.slice(n.pos, ts.tokenStartOfNode(n)).matchAll(/\/\*\*[\s\S]*?\*\//g)) + if (!m[0].includes(MARK)) text += m[0]; + const p: ts.Node | undefined = n.parent; + if ( + p === undefined || + !( + ts.isVariableDeclaration(p) || + ts.isVariableDeclarationList(p) || + ts.isVariableStatement(p) || + ts.isPropertyAssignment(p) || + ts.isParenthesizedExpression(p) || + ts.isExpressionStatement(p) || + ts.isExportAssignment(p) || + (ts.isBinaryExpression(p) && + p.operatorToken.kind === ts.SyntaxKind.EqualsToken && + p.right === n) + ) + ) + break; + } + const params = new Set(); + for (const m of text.matchAll(/@(?:param|arg|argument)\b/g)) { + let i = m.index + m[0].length; + while (i < text.length && /\s/.test(text[i]!)) i++; + if (text[i] === "{") { + let depth = 0; + for (; i < text.length; i++) { + if (text[i] === "{") depth++; + else if (text[i] === "}" && --depth === 0) { + i++; + break; + } + } + } + const name = /^\s*\[?\s*([A-Za-z_$][\w$]*)/.exec(text.slice(i)); + if (name !== null) params.add(name[1]!); + } + return { + params, + returns: /@returns?\b/.test(text), + type: /@type\b/.test(text), + whole: /@(?:type|template|this|overload|callback)\b/.test(text), + }; } /** A comment range's JSDoc type tags made plain text (the range is * trivia, so it holds no literal). The overlay's own comments carry MARK - * and are kept. With `replaces`, only the tags a function's annotation - * replaces are. Lengths are unchanged. */ -export function neutralizeCommentTags(text: string, replaces?: FunctionTags): string { + * and are kept. Lengths are unchanged. */ +export function neutralizeCommentTags(text: string): string { return text.replace(/\/\*\*[\s\S]*?\*\//g, (comment) => { if (comment.includes(MARK)) return comment; - if (replaces === undefined) - return comment.replace( - /@(param|type|typedef|returns?|template|this|callback|property|prop|arg|argument|enum|implements|extends|augments|satisfies|overload|import)\b/g, - "-$1", - ); return comment.replace( - /@(param|arg|argument|returns?|type|overload)\b/g, - (tag: string, name: string, at: number) => { - if (name === "type" || name === "overload") return `-${name}`; - if (name === "returns" || name === "return") return replaces.returns ? `-${name}` : tag; - const param = paramTagName(comment, at + tag.length); - return param !== null && replaces.params.includes(param) ? `-${name}` : tag; - }, + /@(param|type|typedef|returns?|template|this|callback|property|prop|arg|argument|enum|implements|extends|augments|satisfies|overload|import)\b/g, + "-$1", ); }); } -/** The parameter a `@param` tag ending at `from` names (`{type} name`, - * `[name=default]`, or `name.member` of a documented object), or null. */ -function paramTagName(comment: string, from: number): string | null { - let i = from; - while (i < comment.length && /\s/.test(comment[i]!)) i++; - if (comment[i] === "{") { - let depth = 0; - for (; i < comment.length; i++) { - if (comment[i] === "{") depth++; - else if (comment[i] === "}" && --depth === 0) break; +/** The `var` names a dead branch declares that its function (or file) + * declares nowhere else outside the `removed` branches: replacing the + * branch must keep them bound, as undefined. Null when the branch declares + * a function in a block, whose binding outside the block depends on the + * file's strictness; such a branch is kept. */ +function hoistedNames(branch: ts.Node, removed: ReadonlySet): readonly string[] | null { + let scope: ts.Node | undefined = branch.parent; + while (scope !== undefined && !ts.isSourceFile(scope) && !functionScope(scope)) + scope = scope.parent; + if (scope === undefined) return null; + const own = new Set(); + let blockFunction = false; + ts.walkPreorder(branch, (n) => { + if (n !== branch && ts.isFunctionDeclaration(n)) { + blockFunction = true; + return "skip"; } - i++; - while (i < comment.length && /\s/.test(comment[i]!)) i++; + if (n !== branch && (functionScope(n) || ts.isClassDeclaration(n) || ts.isClassExpression(n))) + return "skip"; + if (ts.isVariableDeclarationList(n) && (n.flags & ts.NodeFlags.BlockScoped) === 0) + for (const d of n.declarations.nodes) addBindingNames(d.name, own); + return undefined; + }); + if (blockFunction) return null; + if (own.size === 0) return []; + // Names the scope binds outside the branch: parameters, a function + // expression's name, other var declarations and function declarations. + const elsewhere = new Set(); + if (!ts.isSourceFile(scope)) { + for (const p of list( + (scope as { parameters?: { readonly nodes: readonly ts.ParameterDeclaration[] } }).parameters, + )) + addBindingNames(p.name, elsewhere); + const name = (scope as { name?: ts.Node }).name; + if (name !== undefined && ts.isIdentifier(name)) elsewhere.add(name.text); } - if (comment[i] === "[") i++; - const name = /^[A-Za-z_$][\w$]*/.exec(comment.slice(i, i + 256)); - return name !== null ? name[0] : null; + const body = ts.isSourceFile(scope) ? scope : (scope as { body?: ts.Node }).body; + if (body !== undefined) + ts.walkPreorder(body, (n) => { + if (n === branch || removed.has(n)) return "skip"; + if (n !== body && ts.isFunctionDeclaration(n) && n.name !== undefined) + elsewhere.add(n.name.text); + if (n !== body && (functionScope(n) || ts.isClassDeclaration(n) || ts.isClassExpression(n))) + return "skip"; + if (ts.isVariableDeclarationList(n) && (n.flags & ts.NodeFlags.BlockScoped) === 0) + for (const d of n.declarations.nodes) addBindingNames(d.name, elsewhere); + return undefined; + }); + return [...own].filter((name) => !elsewhere.has(name)); +} + +function functionScope(n: ts.Node): boolean { + return ( + ts.isFunctionDeclaration(n) || + ts.isFunctionExpression(n) || + ts.isArrowFunction(n) || + ts.isMethodDeclaration(n) || + ts.isGetAccessorDeclaration(n) || + ts.isSetAccessorDeclaration(n) || + ts.isConstructorDeclaration(n) || + ts.isClassStaticBlockDeclaration(n) + ); +} + +function addBindingNames(name: ts.BindingName, out: Set): void { + if (ts.isIdentifier(name)) { + out.add(name.text); + return; + } + for (const element of name.elements.nodes) + if (!ts.isOmittedExpression(element) && element.name !== undefined) + addBindingNames(element.name, out); } /** The `typeof` results the values can have. */ @@ -869,160 +1153,6 @@ function applyEdits(text: string, edits: readonly Edit[]): string { return out + text.slice(at); } -/** The file's text with the JSDoc type tags of its comments made plain - * text: package JSDoc is a hint, not a declaration. String, template and - * regular expression literals and JSX text are copied unchanged, so - * comment-like text in them keeps its runtime value. Lengths are unchanged, - * so edit positions in the source text stay valid. */ -export function neutralizeDocTags(sf: ts.SourceFile): string { - const text = sf.text; - // Literal token ranges, in source order. - const literals: [number, number][] = []; - const visit = (node: ts.Node): void => { - if (ts.isJsxText(node)) { - literals.push([node.pos, node.end]); - return; - } - if ( - ts.isStringLiteral(node) || - ts.isNoSubstitutionTemplateLiteral(node) || - ts.isTemplateHead(node) || - ts.isTemplateMiddle(node) || - ts.isTemplateTail(node) || - ts.isRegularExpressionLiteral(node) - ) { - literals.push([ts.tokenStartOfNode(node), node.end]); - return; - } - ts.forEachChild(node, (c) => { - visit(c); - return undefined; - }); - }; - visit(sf); - literals.sort((a, b) => a[0] - b[0]); - let out = ""; - let copied = 0; - let next = 0; - let pos = 0; - // Outside literals, `//` and `/*` only start comments. - while (pos < text.length) { - while (next < literals.length && literals[next]![1] <= pos) next++; - const literal = literals[next]; - if (literal !== undefined && literal[0] <= pos) { - pos = literal[1]; - continue; - } - if (text.charCodeAt(pos) !== 47) { - pos++; - continue; - } - const second = text.charCodeAt(pos + 1); - if (second === 47) { - while (pos < text.length && !lineBreak(text.charCodeAt(pos))) pos++; - continue; - } - if (second !== 42) { - pos++; - continue; - } - const close = text.indexOf("*/", pos + 2); - const end = close < 0 ? text.length : close + 2; - const comment = text.slice(pos, end); - if (comment.startsWith("/**") && !comment.includes(MARK)) { - out += text.slice(copied, pos) + neutralizeComment(comment); - copied = end; - } - pos = end; - } - return out + text.slice(copied); -} - -function lineBreak(code: number): boolean { - return code === 10 || code === 13 || code === 0x2028 || code === 0x2029; -} - -function neutralizeComment(comment: string): string { - return comment.replace( - /@(param|type|typedef|returns?|template|this|callback|property|prop|arg|argument|enum|implements|extends|augments|satisfies|overload|import)\b/g, - "-$1", - ); -} - -/** The `var` names a dead branch declares that its function (or file) - * declares nowhere else outside the `removed` branches: replacing the - * branch must keep them bound, as undefined. Null when the branch declares - * a function in a block, whose binding outside the block depends on the - * file's strictness; such a branch is kept. */ -function hoistedNames(branch: ts.Node, removed: ReadonlySet): readonly string[] | null { - let scope: ts.Node | undefined = branch.parent; - while (scope !== undefined && !ts.isSourceFile(scope) && !functionScope(scope)) - scope = scope.parent; - if (scope === undefined) return null; - const own = new Set(); - let blockFunction = false; - ts.walkPreorder(branch, (n) => { - if (n !== branch && ts.isFunctionDeclaration(n)) { - blockFunction = true; - return "skip"; - } - if (n !== branch && (functionScope(n) || ts.isClassDeclaration(n) || ts.isClassExpression(n))) - return "skip"; - if (ts.isVariableDeclarationList(n) && (n.flags & ts.NodeFlags.BlockScoped) === 0) - for (const d of n.declarations.nodes) addBindingNames(d.name, own); - return undefined; - }); - if (blockFunction) return null; - if (own.size === 0) return []; - // Names the scope binds outside the branch: parameters, a function - // expression's name, other var declarations and function declarations. - const elsewhere = new Set(); - if (!ts.isSourceFile(scope)) { - for (const p of list( - (scope as { parameters?: { readonly nodes: readonly ts.ParameterDeclaration[] } }).parameters, - )) - addBindingNames(p.name, elsewhere); - const name = (scope as { name?: ts.Node }).name; - if (name !== undefined && ts.isIdentifier(name)) elsewhere.add(name.text); - } - const body = ts.isSourceFile(scope) ? scope : (scope as { body?: ts.Node }).body; - if (body !== undefined) - ts.walkPreorder(body, (n) => { - if (n === branch || removed.has(n)) return "skip"; - if (n !== body && ts.isFunctionDeclaration(n) && n.name !== undefined) - elsewhere.add(n.name.text); - if (n !== body && (functionScope(n) || ts.isClassDeclaration(n) || ts.isClassExpression(n))) - return "skip"; - if (ts.isVariableDeclarationList(n) && (n.flags & ts.NodeFlags.BlockScoped) === 0) - for (const d of n.declarations.nodes) addBindingNames(d.name, elsewhere); - return undefined; - }); - return [...own].filter((name) => !elsewhere.has(name)); -} - -function functionScope(n: ts.Node): boolean { - return ( - ts.isFunctionDeclaration(n) || - ts.isFunctionExpression(n) || - ts.isArrowFunction(n) || - ts.isMethodDeclaration(n) || - ts.isGetAccessorDeclaration(n) || - ts.isSetAccessorDeclaration(n) || - ts.isConstructorDeclaration(n) || - ts.isClassStaticBlockDeclaration(n) - ); -} - -function addBindingNames(name: ts.BindingName, out: Set): void { - if (ts.isIdentifier(name)) { - out.add(name.text); - return; - } - for (const element of name.elements.nodes) - if (!ts.isOmittedExpression(element) && element.name !== undefined) - addBindingNames(element.name, out); -} - /** A record site's keys: its static keys, or for a literal with spreads its * own keys plus every key a spread source contributes (optional unless * every value of that source has it). Null when a computed key or an diff --git a/packages/compiler/src/frontend/shape-inference/static-objects.ts b/packages/compiler/src/frontend/shape-inference/static-objects.ts index 99df9fd19..bf8835b19 100644 --- a/packages/compiler/src/frontend/shape-inference/static-objects.ts +++ b/packages/compiler/src/frontend/shape-inference/static-objects.ts @@ -56,14 +56,38 @@ import { resolveCjsRuntime } from "../cjs-resolve.js"; type Node = ts.Node; +/** One read, call or write site, with the receivers of every context it + * was analyzed in. `nullish`: a receiver may also be null or undefined, + * which `atoms` leaves out (see planStaticObjectRewrites). */ +interface MergedOp { + op: ShapeAnalysis["ops"][number]; + atoms: Set; + nullish: boolean; +} + +/** The replacement text of the read `node`: `text`, or, when the receiver + * may be nullish, the read as written followed by `text` in a comma + * expression, so a nullish receiver still throws Node's TypeError and any + * other receiver yields `text`'s value. */ +function keepRead(node: ts.Node, text: string, nullish: boolean): string { + if (!nullish) return text; + const sf = node.getSourceFile(); + return `(${sf.text.slice(ts.tokenStartOfNode(node), node.end)}, ${text})`; +} + interface Edit { readonly start: number; readonly end: number; readonly text: string; } +export type RewriteEdit = Edit; + export interface StaticObjectRewrites { readonly files: Map; + /** Per rewritten file, the edits its text applied (in order; edits nested + * in a replacement are left out), in the text's original offsets. */ + readonly edits: Map; readonly classes: number; readonly functions: number; readonly directCalls: number; @@ -214,6 +238,149 @@ function ownLiteralMember(literal: Node, key: string): string | null { return found; } +/** Whether a top-level statement runs no code but its own. Its expressions + * are literals, identifiers, functions (not run), object and array literals + * without spreads or computed keys, `=` stores, `var` bindings without + * patterns, `&&`, `||`, `??`, `?:`, `,`, `!`, `typeof`, `void`, `===` and + * `!==`, none of which calls a function, and member reads and stores that + * `plainMember` accepts (no getter or setter runs). Every other form can + * run code: a call, `new` or tagged template, a class (static blocks and + * field initializers run), coercion (arithmetic, `+`, templates with + * substitutions, relational and loose comparisons, `in`, `instanceof`, + * computed keys), iteration (spreads, destructuring), `delete`, `await` and + * `yield`. */ +function runsNoCode(stmt: ts.Statement, plainMember: (n: Node) => boolean): boolean { + if (ts.isFunctionDeclaration(stmt) || ts.isEmptyStatement(stmt)) return true; + if (ts.isExpressionStatement(stmt)) return quietExpression(stmt.expression, plainMember); + if (!ts.isVariableStatement(stmt)) return false; + return stmt.declarationList.declarations.nodes.every( + (d) => + ts.isIdentifier(d.name) && + (d.initializer === undefined || quietExpression(d.initializer, plainMember)), + ); +} + +function quietExpression(e: ts.Expression, plainMember: (n: Node) => boolean): boolean { + const quiet = (x: ts.Expression): boolean => quietExpression(x, plainMember); + if (ts.isParenthesizedExpression(e)) return quiet(e.expression); + if ( + ts.isIdentifier(e) || + ts.isNumericLiteral(e) || + ts.isBigIntLiteral(e) || + ts.isStringLiteral(e) || + ts.isNoSubstitutionTemplateLiteral(e) || + ts.isRegularExpressionLiteral(e) || + ts.isFunctionExpression(e) || + ts.isArrowFunction(e) || + e.kind === ts.SyntaxKind.TrueKeyword || + e.kind === ts.SyntaxKind.FalseKeyword || + e.kind === ts.SyntaxKind.NullKeyword || + e.kind === ts.SyntaxKind.ThisKeyword + ) + return true; + if (ts.isObjectLiteralExpression(e)) + return e.properties.nodes.every((m) => { + if (ts.isShorthandPropertyAssignment(m)) return m.objectAssignmentInitializer === undefined; + if (!ts.isPropertyAssignment(m)) return false; + const name = m.name; + if (!ts.isIdentifier(name) && !ts.isStringLiteral(name) && !ts.isNumericLiteral(name)) + return false; + return quiet(m.initializer); + }); + if (ts.isArrayLiteralExpression(e)) + return e.elements.nodes.every( + (x) => ts.isOmittedExpression(x) || (!ts.isSpreadElement(x) && quiet(x)), + ); + if (ts.isTypeOfExpression(e) || ts.isVoidExpression(e)) return quiet(e.expression); + if (ts.isPrefixUnaryExpression(e)) + return e.operator === ts.SyntaxKind.ExclamationToken && quiet(e.operand); + if (ts.isConditionalExpression(e)) + return quiet(e.condition) && quiet(e.whenTrue) && quiet(e.whenFalse); + if (ts.isPropertyAccessExpression(e) || ts.isElementAccessExpression(e)) + return quietMember(e, plainMember); + if (!ts.isBinaryExpression(e)) return false; + switch (e.operatorToken.kind) { + case ts.SyntaxKind.EqualsToken: + if (ts.isIdentifier(e.left)) return quiet(e.right); + return ( + (ts.isPropertyAccessExpression(e.left) || ts.isElementAccessExpression(e.left)) && + quietMember(e.left, plainMember) && + quiet(e.right) + ); + case ts.SyntaxKind.AmpersandAmpersandToken: + case ts.SyntaxKind.BarBarToken: + case ts.SyntaxKind.QuestionQuestionToken: + case ts.SyntaxKind.CommaToken: + case ts.SyntaxKind.EqualsEqualsEqualsToken: + case ts.SyntaxKind.ExclamationEqualsEqualsToken: + return quiet(e.left) && quiet(e.right); + default: + return false; + } +} + +/** A member read or store target with a literal key on a receiver that + * runs no code, which `plainMember` accepts. */ +function quietMember( + e: ts.PropertyAccessExpression | ts.ElementAccessExpression, + plainMember: (n: Node) => boolean, +): boolean { + if (e.questionDotToken !== undefined) return false; + if ( + ts.isElementAccessExpression(e) && + !ts.isStringLiteral(e.argumentExpression) && + !ts.isNumericLiteral(e.argumentExpression) + ) + return false; + return plainMember(e) && quietExpression(e.expression, plainMember); +} + +/** Whether an identifier is written where it appears: an assignment or + * update target, a destructuring assignment or loop target (through array + * and object patterns, defaults and rests), or a `var`, `let`, `const` or + * parameter binding of the name (a nested binding shadows it, which only + * makes the answer conservative). */ +function writesBinding(id: ts.Identifier): boolean { + let n: Node = id; + for (let p = n.parent; p !== undefined; n = p, p = p.parent) { + if (ts.isPrefixUnaryExpression(p) || ts.isPostfixUnaryExpression(p)) + return ( + p.operator === ts.SyntaxKind.PlusPlusToken || p.operator === ts.SyntaxKind.MinusMinusToken + ); + if (ts.isBinaryExpression(p)) { + const op = p.operatorToken.kind; + if (op >= ts.SyntaxKind.FirstAssignment && op <= ts.SyntaxKind.LastAssignment) + return p.left === n; + return false; + } + if (ts.isForOfStatement(p) || ts.isForInStatement(p)) return p.initializer === n; + if (ts.isVariableDeclaration(p) || ts.isParameter(p) || ts.isBindingElement(p)) + return p.name === n; + if ( + ts.isParenthesizedExpression(p) || + ts.isArrayLiteralExpression(p) || + ts.isObjectLiteralExpression(p) || + ts.isSpreadElement(p) || + ts.isSpreadAssignment(p) || + ts.isShorthandPropertyAssignment(p) || + ts.isArrayBindingPattern(p) || + ts.isObjectBindingPattern(p) + ) + continue; + if (ts.isPropertyAssignment(p)) { + if (p.initializer !== n) return false; + continue; + } + return false; + } + return false; +} + +function enclosingFunctionLike(n: Node): Node | null { + for (let p = n.parent; p !== undefined; p = p.parent) if (ts.isFunctionLike(p)) return p; + return null; +} + function primitiveLiteralText(e: ts.Expression): string | null { if ( ts.isNumericLiteral(e) || @@ -414,9 +581,9 @@ export function planStaticObjectRewrites( // One entry per (kind, syntax node): a node the analysis walked in // several contexts (receiver classes, per-site clones) is rewritten only // when the receivers of every context together satisfy the conditions. - const mergedOps: { op: (typeof analysis.ops)[number]; atoms: Set }[] = []; + const mergedOps: MergedOp[] = []; { - const byKey = new Map }>(); + const byKey = new Map(); const ids = new Map(); for (const op of analysis.ops) { let id = ids.get(op.node); @@ -424,10 +591,21 @@ export function planStaticObjectRewrites( const key = `${op.kind}:${op.name ?? ""}:${id}`; let entry = byKey.get(key); if (entry === undefined) { - byKey.set(key, (entry = { op, atoms: new Set() })); + byKey.set(key, (entry = { op, atoms: new Set(), nullish: false })); mergedOps.push(entry); } - for (const a of analysis.nodes[op.recv]!.atoms) entry.atoms.add(a); + // A nullish receiver makes the read throw, so the rewrites decide on + // the other receivers (absent keys read undefined in the analysis: + // `ns.util` before `ns.util = {…}` runs), and `nullish` keeps the + // throw: a rewrite of such a read evaluates the read first. In an + // optional chain a nullish receiver short-circuits instead, so it + // counts. + const optional = (op.node.flags & ts.NodeFlags.OptionalChain) !== 0; + for (const a of analysis.nodes[op.recv]!.atoms) { + if (optional || !(a.kind === "prim" && (a.label === "undefined" || a.label === "null"))) + entry.atoms.add(a); + else entry.nullish = true; + } } } const forIns: ts.Expression[] = []; @@ -569,11 +747,42 @@ export function planStaticObjectRewrites( } return false; }; + /** Whether the read `e` of a `prototype` member may yield one of `atoms` + * through a receiver that is not a function. An object's field holds the + * values stored to it by name (writes through a ⊤ receiver reach every + * object); ⊤, a ⊤ field and an object written by computed key may hold + * anything. */ + const prototypeFieldMayBe = ( + e: ts.PropertyAccessExpression, + atoms: ReadonlySet, + ): boolean => { + const op = readOps.get(e); + if (op === undefined) return true; + for (const a of analysis.nodes[op.recv]!.atoms) { + if (analysis.protoOf.get(a) !== undefined) continue; + if (a === analysis.top || computedRecv.has(a)) return true; + const field = analysis.fields.get(a)?.get("prototype"); + if (field === undefined) continue; + for (const v of analysis.nodes[field]!.atoms) + if (v === analysis.top || atoms.has(v)) return true; + } + return false; + }; /** Whether a reflective call may reach one of `atoms`: as an argument, or * as the object `propertyIsEnumerable` is asked about. */ const reflectsOn = (atoms: ReadonlySet, ctorPath: string): boolean => enumerabilityEscapes || - enumerabilityQueries.some((recv) => mayBeValue(recv, atoms, ctorPath)) || + enumerabilityQueries.some((recv) => { + // `F.prototype` is one of `atoms` (a class's prototype) when F may be + // that class's function, which `atoms` holds too: the analysis reads + // a prototype as ⊤ beside the class prototype objects it tracks. It + // also is when F is an object whose `prototype` field holds it. + let e: ts.Expression = recv; + while (ts.isParenthesizedExpression(e)) e = e.expression; + if (ts.isPropertyAccessExpression(e) && e.name.text === "prototype") + return mayBeValue(e.expression, atoms, ctorPath) || prototypeFieldMayBe(e, atoms); + return mayBeValue(recv, atoms, ctorPath); + }) || reflective.some((call) => call.arguments.nodes.some((arg) => mayBe(arg, atoms, ctorPath))); /** Whether a call that freezes, seals or redefines an object may reach one of `atoms`. */ const reshapes = (atoms: ReadonlySet, ctorPath: string): boolean => @@ -1190,7 +1399,7 @@ export function planStaticObjectRewrites( const stmts = node.getSourceFile().statements.nodes; return stmts.indexOf(top) > stmts.indexOf(store.stmt); }; - for (const { op, atoms } of mergedOps) { + for (const { op, atoms, nullish } of mergedOps) { if (op.kind !== "read" || op.name === null) continue; if (atoms.size !== 1) continue; const recv = [...atoms][0]!; @@ -1226,14 +1435,22 @@ export function planStaticObjectRewrites( if (text === null) continue; callEdits.push({ sf: node.getSourceFile(), - edit: { start: ts.tokenStartOfNode(node), end: node.end, text }, + edit: { + start: ts.tokenStartOfNode(node), + end: node.end, + text: keepRead(node, text, nullish), + }, }); directCalls++; continue; } callEdits.push({ sf: node.getSourceFile(), - edit: { start: ts.tokenStartOfNode(node), end: node.end, text: name }, + edit: { + start: ts.tokenStartOfNode(node), + end: node.end, + text: keepRead(node, name, nullish), + }, }); directCalls++; } @@ -1245,16 +1462,84 @@ export function planStaticObjectRewrites( // the object does and nothing writes it, so every read sees it. (A // member stored after the object was created could be read before the // store; those stay reads.) + // + // A member stored once by an unconditional top-level statement of the + // object's own file (`app.math.EPSILON = 1e-9;`) folds too, at reads that + // provably run after the store: reads inside functions, when no code can + // run before the store (the statements before it call nothing, so no + // function runs and no other module starts), and top-level reads after + // it in the same file. + const writesOf = new Map>(); + for (const op of analysis.ops) { + if (op.kind !== "write" || op.name === null) continue; + for (const a of analysis.nodes[op.recv]!.atoms) { + let byKey = writesOf.get(a); + if (byKey === undefined) writesOf.set(a, (byKey = new Map())); + let l = byKey.get(op.name); + if (l === undefined) byKey.set(op.name, (l = [])); + if (!l.includes(op.node)) l.push(op.node); + } + } + // Member reads and stores whose receiver can only be constant objects or + // primitives: neither has accessors of its own, so no getter or setter + // runs (a primitive receiver of a store throws, or reads its prototype). + const memberReceivers = new Map>(); + for (const op of analysis.ops) { + if ((op.kind !== "read" && op.kind !== "write") || op.name === null) continue; + let set = memberReceivers.get(op.node); + if (set === undefined) memberReceivers.set(op.node, (set = new Set())); + for (const a of analysis.nodes[op.recv]!.atoms) set.add(a); + } + const plainMember = (n: Node): boolean => { + const atoms = memberReceivers.get(n); + return ( + atoms !== undefined && + atoms.size > 0 && + [...atoms].every((a) => a.kind === "prim" || constantAtoms.has(a)) + ); + }; + const quietPrefix = new Map(); + const storedLiteral = ( + atom: Atom, + site: Node, + key: string, + ): { literal: string; stmt: ts.ExpressionStatement; statements: readonly Node[] } | null => { + const writes = writesOf.get(atom)?.get(key) ?? []; + if (writes.length !== 1) return null; + const target = writes[0]!; + const assign = target.parent; + if ( + assign === undefined || + !ts.isBinaryExpression(assign) || + assign.left !== target || + assign.operatorToken.kind !== ts.SyntaxKind.EqualsToken + ) + return null; + const stmt = assign.parent; + if (stmt === undefined || !ts.isExpressionStatement(stmt) || !ts.isSourceFile(stmt.parent)) + return null; + if (stmt.getSourceFile() !== site.getSourceFile()) return null; + const literal = primitiveLiteralText(assign.right); + if (literal === null) return null; + const statements = stmt.parent.statements.nodes; + let quiet = quietPrefix.get(stmt); + if (quiet === undefined) { + quiet = statements + .slice(0, statements.indexOf(stmt)) + .every((s) => runsNoCode(s, plainMember)); + quietPrefix.set(stmt, quiet); + } + return quiet ? { literal, stmt, statements } : null; + }; let foldedReads = 0; - for (const { op, atoms } of mergedOps) { + for (const { op, atoms, nullish } of mergedOps) { if (op.kind !== "read" || op.name === null) continue; if (atoms.size !== 1) continue; - const o = constantAtoms.get([...atoms][0]!); + const atom = [...atoms][0]!; + const o = constantAtoms.get(atom); if (o === undefined || o.site.kind !== "literal") continue; const field = o.fields.find((f) => f.key === op.name); if (field === undefined || field.stores !== 1) continue; - const literal = ownLiteralMember(o.site.node, op.name); - if (literal === null) continue; const node = op.node; if (!ts.isPropertyAccessExpression(node) && !ts.isElementAccessExpression(node)) continue; if (!isPlainRead(node) || topStatement(node) === null) continue; @@ -1266,9 +1551,32 @@ export function planStaticObjectRewrites( kept.push(`constant read ${node.getText()}: it may run before the object is created`); continue; } + let literal = ownLiteralMember(o.site.node, op.name); + if (literal === null && process.env["SCRIPTC_SHAPE_FOLD_STORES"] !== "0") { + const stored = storedLiteral(atom, o.site.node, op.name); + if (stored === null) continue; + const inFunction = enclosingFunctionLike(node) !== null; + const top = topStatement(node)!; + const after = + top.getSourceFile() === stored.stmt.getSourceFile() && + stored.statements.indexOf(top) > stored.statements.indexOf(stored.stmt); + if (!inFunction && !after) continue; + literal = stored.literal; + } + if (literal === null) continue; + // An element key needs no parentheses (and a parenthesized literal key + // would read as a computed one). + const key = + node.parent !== undefined && + ts.isElementAccessExpression(node.parent) && + node.parent.argumentExpression === node; callEdits.push({ sf: node.getSourceFile(), - edit: { start: ts.tokenStartOfNode(node), end: node.end, text: `(${literal})` }, + edit: { + start: ts.tokenStartOfNode(node), + end: node.end, + text: nullish ? keepRead(node, literal, true) : key ? literal : `(${literal})`, + }, }); foldedReads++; } @@ -1298,8 +1606,82 @@ export function planStaticObjectRewrites( // key reads the table as written. The members do not use `this`, the // table has no accessors, and comparing a string runs no code, so only // the representation of the call changes. - const tableCalls: { sf: ts.SourceFile; call: ts.CallExpression; arms: [string, string][] }[] = []; - for (const { op, atoms } of mergedOps) { + const tableCalls: { + sf: ts.SourceFile; + call: ts.CallExpression; + arms: [string, string][]; + nullish: boolean; + exhaustive: boolean; + }[] = []; + // Whether a key parameter only ever holds one of the table's keys: its + // function never escapes, every call of it passes a string literal key + // of the table, and its body neither assigns the parameter nor uses + // `arguments` (which aliases parameters in sloppy code). The dispatch + // then needs no arm for other keys. + const literalKeysOnly = (keyDecl: Node, keys: ReadonlySet): boolean => { + if (process.env["SCRIPTC_SHAPE_EXHAUSTIVE_TABLES"] === "0") return false; + if (!ts.isParameter(keyDecl) || !ts.isIdentifier(keyDecl.name)) return false; + const fn = keyDecl.parent; + if (fn === undefined || !ts.isFunctionLike(fn) || escapedFns.has(fn)) return false; + for (const a of analysis.atoms.atoms) + if (a.kind === "function" && a.node === fn && analysis.escaped.has(a)) return false; + const index = (fn as ts.FunctionLikeDeclaration).parameters.nodes.indexOf(keyDecl); + const name = keyDecl.name.text; + // Any write of the name in the function, its other parameters included, + // or a direct eval or `arguments` (which aliases parameters in sloppy + // code) can give the key another value. + let touched = false; + const visit = (n: Node): void => { + if (touched) return; + if (ts.isIdentifier(n) && (n.text === "arguments" || n.text === "eval")) touched = true; + else if (ts.isIdentifier(n) && n.text === name && n !== keyDecl.name && writesBinding(n)) + touched = true; + ts.forEachChild(n, (c) => { + visit(c); + return undefined; + }); + }; + for (const p of (fn as ts.FunctionLikeDeclaration).parameters.nodes) visit(p); + const body = (fn as { body?: Node }).body; + if (body === undefined) return false; + visit(body); + if (touched) return false; + const edges = analysis.calls.filter((c) => c.callee.node === fn); + if (edges.length === 0) return false; + // A call lines its arguments up with the parameters only when its callee + // is the function itself: a binding, or a constant object's member, that + // holds nothing else. `call` and `apply` pass the this value first, and + // a builtin that calls the function back chooses its own arguments. + const isFn = (a: Atom): boolean => a.kind === "function" && a.node === fn; + const onlyFn = (values: ReadonlySet): boolean => + values.size > 0 && [...values].every(isFn); + const callsFn = (callee: ts.Expression): boolean => { + while (ts.isParenthesizedExpression(callee)) callee = callee.expression; + if (ts.isIdentifier(callee)) { + const d = declOf(callee); + return d !== undefined && onlyFn(analysis.declValues(d)); + } + if (!ts.isPropertyAccessExpression(callee)) return false; + const op = readOps.get(callee); + if (op === undefined || op.name === null) return false; + const recv = analysis.nodes[op.recv]!.atoms; + if (recv.size === 0) return false; + for (const a of recv) { + if (!constantAtoms.has(a)) return false; + const field = analysis.fields.get(a)?.get(op.name); + if (field === undefined || !onlyFn(analysis.nodes[field]!.atoms)) return false; + } + return true; + }; + return edges.every((e) => { + if (!ts.isCallExpression(e.site) || !callsFn(e.site.expression)) return false; + const args = e.site.arguments.nodes; + if (args.slice(0, index + 1).some((a) => ts.isSpreadElement(a))) return false; + const arg = args[index]; + return arg !== undefined && ts.isStringLiteral(arg) && keys.has(arg.text); + }); + }; + for (const { op, atoms, nullish } of mergedOps) { if (op.kind !== "computedRead") continue; const node = op.node; if (!ts.isElementAccessExpression(node) || node.questionDotToken !== undefined) continue; @@ -1349,22 +1731,44 @@ export function planStaticObjectRewrites( kept.push(`${where}: it may run before the table is created`); continue; } - tableCalls.push({ sf: call.getSourceFile(), call, arms }); + tableCalls.push({ + sf: call.getSourceFile(), + call, + arms, + nullish, + exhaustive: literalKeysOnly(keyDecl, new Set(arms.map(([k]) => k))), + }); } - for (const { sf, call, arms } of tableCalls) { + for (const { sf, call, arms, nullish, exhaustive } of tableCalls) { const node = call.expression as ts.ElementAccessExpression; const key = node.argumentExpression.getText(); const args = call.arguments.nodes.map((a) => sliceWithCalls(sf, a)).join(", "); const table = sliceWithCalls(sf, node.expression); - const text = arms + const tested = exhaustive ? arms.slice(0, -1) : arms; + const text = tested .map(([k, name]) => `${key} === ${JSON.stringify(k)} ? ${name}(${args}) : `) .join(""); + if (exhaustive) { + callEdits.push({ + sf, + edit: { + start: ts.tokenStartOfNode(call), + end: call.end, + text: `(${nullish ? `${table}[${key}], ` : ""}${text}${arms[arms.length - 1]![1]}(${args}))`, + }, + }); + tableDispatches++; + continue; + } callEdits.push({ sf, edit: { start: ts.tokenStartOfNode(call), end: call.end, - text: `(${text}${table}[${key}](${args}))`, + // A table that may be nullish is read first, where the call reads + // it, so the read throws before the arguments run; on the constant + // object it reads a plain member with a string key. + text: `(${nullish ? `${table}[${key}], ` : ""}${text}${table}[${key}](${args}))`, }, }); tableDispatches++; @@ -1413,8 +1817,53 @@ export function planStaticObjectRewrites( edit(sf, { start: at, end: at, text: props.length > 0 ? `, ${text}` : ` ${text} ` }); } const out = new Map(); - for (const [sf, edits] of editsByFile) out.set(sf.fileName, applyEdits(sf.text, edits)); - return { files: out, classes, functions, directCalls, tableDispatches, foldedReads, kept }; + const applied = new Map(); + for (const [sf, edits] of editsByFile) { + out.set(sf.fileName, applyEdits(sf.text, edits)); + applied.set(sf.fileName, appliedEdits(edits)); + } + return { + files: out, + edits: applied, + classes, + functions, + directCalls, + tableDispatches, + foldedReads, + kept, + }; +} + +/** The edits applyEdits applies, in order. */ +function appliedEdits(edits: readonly Edit[]): Edit[] { + const sorted = [...edits].sort((a, b) => a.start - b.start || b.end - a.end); + const out: Edit[] = []; + let at = 0; + for (const e of sorted) { + if (e.start < at) continue; + out.push(e); + at = e.end > e.start ? e.end : e.start; + } + return out; +} + +/** A position of a rewritten file's original text in its rewritten text, + * or null inside a replaced range or at an insertion (the text there + * changed). */ +export function rewrittenPosition(edits: readonly RewriteEdit[], pos: number): number | null { + let shift = 0; + for (const e of edits) { + if (e.start > pos) break; + if (e.end === e.start) { + if (e.start === pos) return null; + shift += e.text.length; + continue; + } + if (pos === e.start) break; + if (pos < e.end) return null; + shift += e.text.length - (e.end - e.start); + } + return pos + shift; } /** Applies replacements and insertions; an edit inside an earlier diff --git a/packages/compiler/src/frontend/ts7/checker.ts b/packages/compiler/src/frontend/ts7/checker.ts index eb2b9e20b..44839bc2f 100644 --- a/packages/compiler/src/frontend/ts7/checker.ts +++ b/packages/compiler/src/frontend/ts7/checker.ts @@ -345,6 +345,15 @@ export class CheckerFacade { this.prefetchSymbols(sf); } + /** The symbols at a file's identifiers only, for consumers that resolve + * names but never ask for types (shape inference). The file stays open + * to a later whole-file prefetch, which skips the answered nodes. */ + prefetchSourceFileSymbols(sf: SourceFile): void { + this.ensureActive(); + if (this.cache.prefetchedSymbols.has(sf)) return; + this.prefetchSymbolNodes(collectNodes([sf]), false, false); + } + /** Batches the non-body structure of many files as ONE logical wave. * Top-level executable statements, class field initializers/static blocks, * and every declaration header are included; function/method/constructor diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index ce11c0202..bd974a98d 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -1,4 +1,4 @@ -import { shapeOverlayEnabled } from "./frontend/shape-inference/index.js"; +import { setShapeOverlayMemoDir, shapeOverlayEnabled } from "./frontend/shape-inference/index.js"; import { nodeFrontendServices } from "./frontend/services-node.js"; import { runTypePatchCommand, @@ -1119,6 +1119,16 @@ async function stageDriverBuildSources( * (`runFrontendProcess`). Node's heap keeps the pages a build's frontend used for the life of the * process, so running them in a process of their own leaves the native helper the memory they took. */ +/** The shape overlay's memo directory for the current executable build + * (frontend/shape-inference/memo.ts): per compiler, in the build cache. */ +let shapeOverlayMemo: string | null = null; + +function shapeOverlayMemoDir(cacheRoot: string | null, implementation: string): string | null { + return cacheRoot === null + ? null + : join(cacheRoot, "shape-overlay-v1", implementation.slice(0, 24)); +} + const frontendProcessEntry = fileURLToPath(new URL("./frontend-process.js", import.meta.url)); /** @@ -1198,6 +1208,8 @@ interface FrontendProcessRequest { timing: TimingClock; /** The compile's --provenance-sources mappings, which the frontend process installs. */ provenance: ProvenanceSources | null; + /** The shape overlay's memo directory (shapeOverlayMemoDir). */ + shapeOverlayMemo: string | null; } /** What the frontend process sends back. A thrown error comes back as `failed`. */ @@ -1251,6 +1263,7 @@ async function prepareExecutableInputInChild( buildPlatform, timing: timingClock, provenance: provenanceSources(), + shapeOverlayMemo, }; await writeFile(requestPath, JSON.stringify(request), { mode: 0o600 }); const exitCode = await frontendProcess.run(requestPath, resultPath); @@ -1318,6 +1331,8 @@ export async function runFrontendProcess(requestPath: string, resultPath: string process.env["SCRIPTC_SHAPE_OVERLAY"] = request.opts.shapeOverlay ? "1" : "0"; clearCompileSessionCaches(); setProvenanceSources(request.provenance); + shapeOverlayMemo = request.shapeOverlayMemo; + setShapeOverlayMemoDir(request.shapeOverlayMemo); // The compile that started this process prepared the cache root. configureLibDiagnostics( (request.opts.outputKind ?? "exe") === "exe" && provenanceSources() === null @@ -1387,6 +1402,8 @@ async function compileTracked( const timing = compilationTiming(); const timingStart = Date.now(); entryPath = resolve(entryPath); + shapeOverlayMemo = null; + setShapeOverlayMemoDir(null); const outputKind = opts.outputKind ?? "exe"; if ( (opts.backend !== undefined && opts.backend !== "llvm") || @@ -1704,6 +1721,8 @@ async function compileTracked( implementation: implementation.digest, implementationDependencies: implementation.dependencies, }; + shapeOverlayMemo = shapeOverlayMemoDir(cacheRoot, implementation.digest); + setShapeOverlayMemoDir(shapeOverlayMemo); } const earlyHit = earlyCacheOptions === null diff --git a/packages/compiler/test/constant-graphs.test.ts b/packages/compiler/test/constant-graphs.test.ts index 748c6cb20..478932341 100644 --- a/packages/compiler/test/constant-graphs.test.ts +++ b/packages/compiler/test/constant-graphs.test.ts @@ -89,9 +89,12 @@ test("constant graphs: namespaces, revealing modules and prototypes, with near m const text = lines.join("\n"); // The namespace graph: guarded declarations, the revealing module's // table, and the constructor function. + // `lib.math` is stored after `lib` was created, so a read of it may see + // undefined first; its object is a graph of its own. expect(text).toMatch( - /graph ns-fixture\/index\.js:\d+ \{lib: \{math: \{TAU: number, half: fn\}, text: \{upper: fn\}/, + /graph ns-fixture\/index\.js:\d+ \{lib: \{math: undefined\|\{\}@ns-fixture\/index\.js:\d+, text: \{upper: fn\}/, ); + expect(text).toMatch(/graph ns-fixture\/index\.js:\d+ \{TAU: number, half: fn\}/); // The constructor's prototype object, built by member stores at init. expect(text).toMatch(/graph ns-fixture\/index\.js:\d+ \{constructor: \{\}, norm: fn\}/); // A field written by a function called from a loop. diff --git a/packages/compiler/test/shape-overlay-cost.test.ts b/packages/compiler/test/shape-overlay-cost.test.ts new file mode 100644 index 000000000..9a3229aeb --- /dev/null +++ b/packages/compiler/test/shape-overlay-cost.test.ts @@ -0,0 +1,127 @@ +/* The overlay's compile-cost pieces: the memo that carries a build's + * verdicts to the next build of the same JavaScript (memo.ts), and method + * families that join only the methods a call can reach together + * (overlay.ts), so a call on one class keeps that class's result type. */ +import { mkdir, mkdtemp, readdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, expect, test, vi } from "vitest"; +import { compile } from "../src/index.js"; +import { + overlayMemoKey, + readOverlayMemo, + setShapeOverlayMemoDir, + writeOverlayMemo, +} from "../src/frontend/shape-inference/memo.js"; + +const packageSource = `export class Str { + constructor(checks) { this.checks = checks; } + _addCheck(c) { return new Str([...this.checks, c]); } + min(n) { return this._addCheck({ kind: "min", n }); } + max(n) { return this._addCheck({ kind: "max", n }); } +} +export class Num { + constructor(checks) { this.checks = checks; } + _addCheck(c) { return new Num([...this.checks, c]); } + int() { return this._addCheck({ kind: "int" }); } +} +export const str = () => new Str([]); +export const num = () => new Num([]); +`; + +let dir = ""; +const saved: Record = {}; +beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), "scriptc-overlay-cost-")); + const pkg = join(dir, "node_modules/families-fixture"); + await mkdir(pkg, { recursive: true }); + await writeFile( + join(pkg, "package.json"), + JSON.stringify({ + name: "families-fixture", + version: "1.0.0", + type: "module", + main: "index.js", + }), + ); + await writeFile(join(pkg, "index.js"), packageSource); + await writeFile( + join(dir, "package.json"), + JSON.stringify({ type: "module", dependencies: { "families-fixture": "1.0.0" } }), + ); + await writeFile( + join(dir, "main.ts"), + `import { str, num } from "families-fixture";\n` + + `console.log(str().min(1).max(2).checks.length, num().int().checks.length);\n`, + ); + for (const k of ["SCRIPTC_SHAPE_OVERLAY", "SCRIPTC_DEBUG_SHAPE_OVERLAY", "SCRIPTC_NO_CACHE"]) + saved[k] = process.env[k]; +}); +afterEach(async () => { + for (const [k, v] of Object.entries(saved)) { + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + setShapeOverlayMemoDir(null); + vi.restoreAllMocks(); + await rm(dir, { recursive: true, force: true }); +}); + +test("same-named methods no call reaches together keep their own result types", async () => { + process.env["SCRIPTC_SHAPE_OVERLAY"] = "1"; + process.env["SCRIPTC_DEBUG_SHAPE_OVERLAY"] = "1"; + process.env["SCRIPTC_NO_CACHE"] = "1"; + let text = ""; + vi.spyOn(process.stderr, "write").mockImplementation((chunk: string | Uint8Array) => { + text += typeof chunk === "string" ? chunk : Buffer.from(chunk).toString("utf8"); + return true; + }); + const result = await compile(join(dir, "main.ts"), { + outDir: dir, + outPath: join(dir, "main.ir.json"), + outputKind: "ir", + npmStatic: ["families-fixture"], + }); + vi.restoreAllMocks(); + if (!result.ok) throw new Error(result.diagnostics.map((d) => d.message).join("\n")); + expect(text).toMatch(/@returns \{Str\}[^\n]*\*\/ _addCheck\(c\) \{ return new Str/); + expect(text).toMatch(/@returns \{Num\}[^\n]*\*\/ _addCheck\(c\) \{ return new Num/); + expect(text).not.toContain("Str | Num"); +}); + +test("a memo round-trips and is keyed by every JavaScript file's text", async () => { + const memoDir = join(dir, "memo"); + setShapeOverlayMemoDir(memoDir); + const key = overlayMemoKey([ + ["/p/a.js", "a"], + ["/p/b.js", "b"], + ]); + expect( + overlayMemoKey([ + ["/p/b.js", "b"], + ["/p/a.js", "a"], + ]), + ).toBe(key); + expect( + overlayMemoKey([ + ["/p/a.js", "a"], + ["/p/b.js", "b2"], + ]), + ).not.toBe(key); + expect(readOverlayMemo(key)).toBeNull(); + const memo = { + off: false, + annotations: [{ file: "/p/a.js", hash: "h", ids: [3, 7] }], + files: [], + plainFences: ["SC2002|/p/a.js|4"], + }; + writeOverlayMemo(key, memo); + expect(readOverlayMemo(key)).toEqual(memo); + expect((await readdir(memoDir)).filter((f) => f.endsWith(".tmp"))).toEqual([]); + // A malformed memo reads as none. + await writeFile(join(memoDir, `${key}.json`), `{"off": 1}`); + expect(readOverlayMemo(key)).toBeNull(); + // Without a memo directory (SCRIPTC_NO_CACHE) nothing is read or written. + setShapeOverlayMemoDir(null); + expect(readOverlayMemo(key)).toBeNull(); +}); diff --git a/packages/compiler/test/shape-overlay-patches.test.ts b/packages/compiler/test/shape-overlay-patches.test.ts new file mode 100644 index 000000000..bbd6c24eb --- /dev/null +++ b/packages/compiler/test/shape-overlay-patches.test.ts @@ -0,0 +1,86 @@ +/* A declaration a project type patch edited keeps the patch alone: the + * shape overlay writes no annotation on it (overlay.ts + * setShapeOverlayProtectedSpans; the type-patches layer supplies the + * spans). Other declarations of the same file are typed as usual. */ +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, expect, test, vi } from "vitest"; +import { compile } from "../src/index.js"; +import { setShapeOverlayProtectedSpans } from "../src/frontend/shape-inference/index.js"; + +const patchedDoc = "/** @param {number} n patched by the project */"; +const packageSource = `export function twice(n) { return n * 2; } +${patchedDoc} +export function thrice(n) { return n * 3; } +`; + +let dir = ""; +const saved: Record = {}; +beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), "scriptc-overlay-patch-")); + const pkg = join(dir, "node_modules/patched-fixture"); + await mkdir(pkg, { recursive: true }); + await writeFile( + join(pkg, "package.json"), + JSON.stringify({ name: "patched-fixture", version: "1.0.0", type: "module", main: "index.js" }), + ); + await writeFile(join(pkg, "index.js"), packageSource); + await writeFile( + join(dir, "package.json"), + JSON.stringify({ type: "module", dependencies: { "patched-fixture": "1.0.0" } }), + ); + await writeFile( + join(dir, "main.ts"), + `import { twice, thrice } from "patched-fixture";\nconsole.log(twice(2), thrice(3));\n`, + ); + for (const k of ["SCRIPTC_SHAPE_OVERLAY", "SCRIPTC_DEBUG_SHAPE_OVERLAY", "SCRIPTC_NO_CACHE"]) + saved[k] = process.env[k]; +}); +afterEach(async () => { + for (const [k, v] of Object.entries(saved)) { + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + setShapeOverlayProtectedSpans(null); + vi.restoreAllMocks(); + await rm(dir, { recursive: true, force: true }); +}); + +async function overlayText(): Promise { + process.env["SCRIPTC_SHAPE_OVERLAY"] = "1"; + process.env["SCRIPTC_DEBUG_SHAPE_OVERLAY"] = "1"; + process.env["SCRIPTC_NO_CACHE"] = "1"; + let text = ""; + vi.spyOn(process.stderr, "write").mockImplementation((chunk: string | Uint8Array) => { + text += typeof chunk === "string" ? chunk : Buffer.from(chunk).toString("utf8"); + return true; + }); + const result = await compile(join(dir, "main.ts"), { + outDir: dir, + outPath: join(dir, "main.ir.json"), + outputKind: "ir", + npmStatic: ["patched-fixture"], + }); + vi.restoreAllMocks(); + if (!result.ok) throw new Error(result.diagnostics.map((d) => d.message).join("\n")); + const start = text.indexOf("[shape-overlay] "); + return start < 0 ? "" : text.slice(start); +} + +test("the overlay types both functions without a patch", async () => { + const text = await overlayText(); + expect(text).toMatch(/@scriptcShape #\d+ \*\/ export function twice/); + expect(text).toMatch(/@scriptcShape #\d+ \*\/ export function thrice/); +}); + +test("a patched declaration gets no overlay annotation", async () => { + const start = packageSource.indexOf(patchedDoc); + setShapeOverlayProtectedSpans((file) => + file.endsWith("patched-fixture/index.js") ? [{ start, end: start + patchedDoc.length }] : [], + ); + const text = await overlayText(); + expect(text).toMatch(/@scriptcShape #\d+ \*\/ export function twice/); + expect(text).toContain(`${patchedDoc}\nexport function thrice`); + expect(text).not.toMatch(/@scriptcShape #\d+ \*\/ export function thrice/); +}); diff --git a/packages/compiler/test/shape-overlay-render.test.ts b/packages/compiler/test/shape-overlay-render.test.ts index 53b47092d..b1982311b 100644 --- a/packages/compiler/test/shape-overlay-render.test.ts +++ b/packages/compiler/test/shape-overlay-render.test.ts @@ -1,83 +1,75 @@ -/* renderShapeOverlay neutralizes the package JSDoc in the leading trivia of - * the annotations a build keeps, and leaves the trivia of dropped ones. */ +/* renderShapeOverlay keeps the package's JSDoc as written (package JSDoc is + * a declaration) and writes the annotations a build keeps. */ import { expect, test } from "vitest"; import { - neutralizeCommentTags, OVERLAY_MARK, + planOnRewrittenTexts, renderShapeOverlay, } from "../src/frontend/shape-inference/overlay.js"; +import { rewrittenPosition } from "../src/frontend/shape-inference/static-objects.js"; import type { ShapeOverlayPlan } from "../src/frontend/shape-inference/overlay.js"; -test("each kept annotation's leading JSDoc is neutralized, once per range", () => { +test("package JSDoc stays as written; dropped annotations are left out", () => { const decls = [ - "/** @param {string} a */\nfunction f(a) {}\n", + "/** @param {string} a */\nfunction f(a, b) {}\n", "/** @type {number} */\nvar x = 1;\n", - "/** @returns {void} */\nfunction g() {}\n", + "function g(c) {}\n", ]; const text = decls.join(""); - const docRanges: { lo: number; hi: number; id: number }[] = []; const edits: { pos: number; id: number; text: string }[] = []; + const at: number[] = []; let lo = 0; for (const decl of decls) { - const hi = lo + decl.indexOf("\n") + 1; - docRanges.push({ lo, hi, id: hi }); - // A declaration's first child starts at the same position: its range is - // the same trivia. - docRanges.push({ lo, hi, id: hi }); - edits.push({ pos: hi, id: hi, text: `/** @type {any} ${OVERLAY_MARK} #${hi} */ ` }); + const pos = lo + (decl.startsWith("/**") ? decl.indexOf("\n") + 1 : 0); + at.push(pos); + edits.push({ pos, id: pos, text: `/** @param {number} b ${OVERLAY_MARK} #${pos} */ ` }); lo += decl.length; } - const plan: ShapeOverlayPlan = { - parts: new Map([["a.js", { text, edits, suffix: "", docRanges }]]), - }; - const dropped = docRanges[2]!.id; + const plan: ShapeOverlayPlan = { parts: new Map([["a.js", { text, edits, suffix: "" }]]) }; const overlay = renderShapeOverlay(plan, { - annotations: new Map([["a.js", new Set([dropped])]]), + annotations: new Map([["a.js", new Set([at[1]!])]]), files: new Set(), }); - const annotated = (decl: string, keep: boolean, at: number): string => { - const head = decl.slice(0, decl.indexOf("\n") + 1); - const rest = decl.slice(head.length); - return keep - ? `${neutralizeCommentTags(head)}/** @type {any} ${OVERLAY_MARK} #${at} */ ${rest}` - : decl; - }; - expect(overlay.files.get("a.js")).toBe( - decls - .map((decl, i) => annotated(decl, docRanges[2 * i]!.id !== dropped, docRanges[2 * i]!.id)) - .join(""), - ); - expect(overlay.files.get("a.js")).toContain("/** @type {number} */"); - expect(overlay.files.get("a.js")).toContain("/** -param {string} a */"); + const out = overlay.files.get("a.js")!; + expect(out).toContain("/** @param {string} a */\n/** @param {number} b"); + expect(out).toContain("/** @type {number} */\nvar x = 1;"); + expect(out).not.toContain(`#${at[1]}`); + expect(out).toContain(`#${at[2]} */ function g(c) {}`); expect(overlay.annotations).toBe(2); }); -test("a function's annotation neutralizes only the tags it replaces", () => { - const head = - "/**\n * @template T\n * @param {{ a?: boolean }} options\n * @param {T} [b]\n * @param {string} options.c\n * @param {number} n\n * @returns {T}\n */\n"; - expect(neutralizeCommentTags(head, { returns: true, params: ["n"] })).toBe( - "/**\n * @template T\n * @param {{ a?: boolean }} options\n * @param {T} [b]\n * @param {string} options.c\n * -param {number} n\n * -returns {T}\n */\n", - ); - expect(neutralizeCommentTags(head, { returns: false, params: ["options", "b"] })).toBe( - "/**\n * @template T\n * -param {{ a?: boolean }} options\n * -param {T} [b]\n * -param {string} options.c\n * @param {number} n\n * @returns {T}\n */\n", - ); - const text = `${head}function f(options, b, n) {}\n`; +test("a plan moves onto a text whose constant reads were replaced", () => { + // `NS.K` (6..10) folds to `1`; the annotation on `g` follows the shift, + // and an annotation inside the folded read is left out with its closer. + const text = "f(a, NS.K);\nfunction g(c) {}\n"; + const fold = { start: text.indexOf("NS.K"), end: text.indexOf("NS.K") + 4, text: "1" }; + const rewrittenText = text.replace("NS.K", "1"); + const g = text.indexOf("function g"); + const inside = fold.start + 3; + expect(rewrittenPosition([fold], g)).toBe(g - 3); + expect(rewrittenPosition([fold], fold.start)).toBe(fold.start); + expect(rewrittenPosition([fold], inside)).toBeNull(); + expect(rewrittenPosition([{ start: 0, end: 0, text: "x" }], 0)).toBeNull(); const plan: ShapeOverlayPlan = { parts: new Map([ [ "a.js", { text, - edits: [{ pos: head.length, id: head.length, text: "/** @returns {undefined} */ " }], - suffix: "", - docRanges: [ - { lo: 0, hi: head.length, id: head.length, replaces: { returns: true, params: [] } }, + edits: [ + { pos: g, id: g, text: `/** @param {number} c ${OVERLAY_MARK} #${g} */ ` }, + { pos: inside, id: inside, text: `/** @type {X} ${OVERLAY_MARK} #${inside} */ (` }, + { pos: fold.end, id: inside, text: ")" }, ], + suffix: "", }, ], ]), }; - const out = renderShapeOverlay(plan).files.get("a.js")!; - expect(out).toContain("@param {{ a?: boolean }} options"); - expect(out).toContain("-returns {T}"); + const moved = planOnRewrittenTexts( + plan, + new Map([["a.js", { text: rewrittenText, map: (p: number) => rewrittenPosition([fold], p) }]]), + ); + const out = renderShapeOverlay(moved).files.get("a.js")!; + expect(out).toBe(`f(a, 1);\n/** @param {number} c ${OVERLAY_MARK} #${g} */ function g(c) {}\n`); }); diff --git a/packages/compiler/test/static-object-guards.test.ts b/packages/compiler/test/static-object-guards.test.ts new file mode 100644 index 000000000..e14793c9a --- /dev/null +++ b/packages/compiler/test/static-object-guards.test.ts @@ -0,0 +1,165 @@ +/* Static-object specialization (shape-inference/static-objects.ts) keeps two + * reads dynamic when the facts it would fold on can change: + * - a member stored after a statement that can run code (a coercion such as + * `+` can call toString or valueOf) keeps its read in functions, which + * that code could call before the store (the statements checked are all + * the file's top-level statements before the store); + * - a table call whose key parameter the function rebinds (destructuring, + * a loop target, another parameter's default or a `var` redeclaration) + * keeps reading the table for other keys, though every caller passes one + * of the table's keys; + * - so does one whose function is called through `call` or `apply`, where + * the first argument is the this value and not the key. + * The checks read the rewritten package text SCRIPTC_DEBUG_SHAPE_OVERLAY + * prints. */ +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, expect, test, vi } from "vitest"; +import { compile } from "../src/index.js"; + +const packageSource = `var quiet = quiet || {}; +quiet.cfg = { U: 1 }; +quiet.read = function () { + return quiet.cfg.V; +}; +quiet.label = quiet.cfg.U; +quiet.cfg.V = 6; + +var late = late || {}; +late.cfg = { U: 1 }; +late.read = function () { + return late.cfg.V; +}; +late.label = "v" + late.cfg.U; +late.cfg.V = 5; + +var t = t || {}; +t.ops = { + a: function (v) { + return v + 1; + }, + b: function (v) { + return v * 2; + }, +}; +t.plain = function (op, v) { + return t.ops[op](v); +}; +t.pattern = function (op, v) { + [op] = ["c"]; + return t.ops[op](v); +}; +t.loop = function (op, v) { + for (op of ["c"]); + return t.ops[op](v); +}; +t.initializer = function (op, v, w = (op = "c")) { + return t.ops[op](v); +}; +t.redeclared = function (op, v) { + var op = "c"; + return t.ops[op](v); +}; +t.viaCall = function (op, v) { + return t.ops[op](v); +}; +t.viaApply = function (op, v) { + return t.ops[op](v); +}; +module.exports = { late, quiet, t }; +`; + +const consumer = `const { late, quiet, t } = require("guard-fixture"); +console.log(late.read(), late.label, quiet.read(), quiet.label); +const tryCall = (call) => { + try { + return call(); + } catch (e) { + return e.name; + } +}; +console.log(t.plain("a", 1), t.plain("b", 2)); +console.log(tryCall(() => t.pattern("a", 1)), tryCall(() => t.loop("b", 2))); +console.log(tryCall(() => t.initializer("a", 1)), tryCall(() => t.redeclared("b", 2))); +console.log(tryCall(() => t.viaCall.call("a", "c", 1)), tryCall(() => t.viaApply.apply("b", ["c", 2]))); +`; + +let dir = ""; +const saved: Record = {}; +beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), "scriptc-static-guards-")); + const pkg = join(dir, "node_modules/guard-fixture"); + await mkdir(pkg, { recursive: true }); + await writeFile( + join(pkg, "package.json"), + JSON.stringify({ name: "guard-fixture", version: "1.0.0", main: "index.js" }), + ); + await writeFile(join(pkg, "index.js"), packageSource); + await writeFile( + join(dir, "package.json"), + JSON.stringify({ dependencies: { "guard-fixture": "1.0.0" } }), + ); + await writeFile(join(dir, "main.cjs"), consumer); + for (const k of ["SCRIPTC_SHAPE_OVERLAY", "SCRIPTC_DEBUG_SHAPE_OVERLAY", "SCRIPTC_NO_CACHE"]) + saved[k] = process.env[k]; +}); +afterEach(async () => { + for (const [k, v] of Object.entries(saved)) { + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + vi.restoreAllMocks(); + await rm(dir, { recursive: true, force: true }); +}); + +/** The package text after the structural rewrites. */ +async function rewrittenPackage(): Promise { + process.env["SCRIPTC_SHAPE_OVERLAY"] = "1"; + process.env["SCRIPTC_DEBUG_SHAPE_OVERLAY"] = "1"; + process.env["SCRIPTC_NO_CACHE"] = "1"; + let text = ""; + vi.spyOn(process.stderr, "write").mockImplementation((chunk: string | Uint8Array) => { + text += typeof chunk === "string" ? chunk : Buffer.from(chunk).toString("utf8"); + return true; + }); + const result = await compile(join(dir, "main.cjs"), { + outDir: dir, + outPath: join(dir, "main.ir.json"), + outputKind: "ir", + npmStatic: ["guard-fixture"], + }); + vi.restoreAllMocks(); + if (!result.ok) throw new Error(result.diagnostics.map((d) => d.message).join("\n")); + const start = text.indexOf("[static-objects] "); + const end = text.indexOf("[shape-overlay] ", start); + return start < 0 ? "" : text.slice(start, end < 0 ? undefined : end); +} + +/** The body of the function the rewritten text assigns to or hoists as `name`. */ +function body(text: string, name: string): string { + const at = text.search(new RegExp(`function [\\w$]*${name}\\b|${name} = function`)); + return at < 0 ? "" : text.slice(at, text.indexOf("\n}", at)); +} + +test("a store after code that can run keeps the read; a quiet prefix folds it", async () => { + const text = await rewrittenPackage(); + expect(body(text, "quiet_read")).toMatch(/return \(.*\b6\);/); + expect(body(text, "late_read")).toContain("return late.cfg.V"); +}); + +test("a table call whose key parameter is rebound keeps reading the table", async () => { + const text = await rewrittenPackage(); + // Every caller passes "a" or "b": the plain call needs no arm for other keys. + expect(body(text, "t_plain")).not.toContain("t.ops[op](v)"); + for (const name of ["t_pattern", "t_loop", "t_initializer", "t_redeclared"]) + expect(body(text, name), name).toContain("t.ops[op](v)"); +}); + +test("a table call whose function is called through call or apply keeps reading the table", async () => { + // The first argument of those calls, a key of the table, is the this + // value; the key each call passes, "c", is not one. + const text = await rewrittenPackage(); + for (const name of ["t_viaCall", "t_viaApply"]) + expect(body(text, name), name).toContain("t.ops[op](v)"); +}); diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/concat-dynamic-args/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/concat-dynamic-args/main.ts.json new file mode 100644 index 000000000..0fbf33bad --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/concat-dynamic-args/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/concat-dynamic-args/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/evolving-let-index/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/evolving-let-index/main.ts.json new file mode 100644 index 000000000..8c503a074 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/evolving-let-index/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/evolving-let-index/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/shape-absent-reads/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/shape-absent-reads/main.ts.json new file mode 100644 index 000000000..9a8706ad0 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/shape-absent-reads/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/shape-absent-reads/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/shape-bigint-arith/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/shape-bigint-arith/main.ts.json new file mode 100644 index 000000000..5df93d3c8 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/shape-bigint-arith/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/shape-bigint-arith/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/shape-module-identity/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/shape-module-identity/main.ts.json new file mode 100644 index 000000000..1940110e8 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/shape-module-identity/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/shape-module-identity/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/static-stores/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/static-stores/main.ts.json new file mode 100644 index 000000000..10384a91b --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/static-stores/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/static-stores/main.ts" + ], + "diags": [] +} diff --git a/tests/fixtures/npm/cases/concat-dynamic-args/main.ts b/tests/fixtures/npm/cases/concat-dynamic-args/main.ts new file mode 100644 index 000000000..35d6e0268 --- /dev/null +++ b/tests/fixtures/npm/cases/concat-dynamic-args/main.ts @@ -0,0 +1,3 @@ +// concat of checked-dynamic values decides spreading at run time. +import { run } from "concat-dynamic"; +console.log(run(JSON.stringify({ text: "desc", list: ["a", "b"] }))); diff --git a/tests/fixtures/npm/cases/concat-dynamic-args/node_modules/concat-dynamic/index.d.ts b/tests/fixtures/npm/cases/concat-dynamic-args/node_modules/concat-dynamic/index.d.ts new file mode 100644 index 000000000..615876884 --- /dev/null +++ b/tests/fixtures/npm/cases/concat-dynamic-args/node_modules/concat-dynamic/index.d.ts @@ -0,0 +1 @@ +export declare function run(raw: string): string; diff --git a/tests/fixtures/npm/cases/concat-dynamic-args/node_modules/concat-dynamic/index.js b/tests/fixtures/npm/cases/concat-dynamic-args/node_modules/concat-dynamic/index.js new file mode 100644 index 000000000..e2bc8066f --- /dev/null +++ b/tests/fixtures/npm/cases/concat-dynamic-args/node_modules/concat-dynamic/index.js @@ -0,0 +1,12 @@ +'use strict'; +// Array.prototype.concat spreads an argument that is an array at run time and +// appends any other value, whatever the static type says. +function run(raw) { + const parsed = JSON.parse(raw); + let output = ['Usage:', '']; + output = output.concat([parsed.text, '']); + output = output.concat(parsed.list); + output = output.concat(parsed.text); + return output.join('|') + ' ' + output.length; +} +module.exports = { run }; diff --git a/tests/fixtures/npm/cases/concat-dynamic-args/node_modules/concat-dynamic/package.json b/tests/fixtures/npm/cases/concat-dynamic-args/node_modules/concat-dynamic/package.json new file mode 100644 index 000000000..afd15f22b --- /dev/null +++ b/tests/fixtures/npm/cases/concat-dynamic-args/node_modules/concat-dynamic/package.json @@ -0,0 +1 @@ +{ "name": "concat-dynamic", "version": "1.0.0", "main": "index.js", "types": "index.d.ts" } diff --git a/tests/fixtures/npm/cases/concat-dynamic-args/package.json b/tests/fixtures/npm/cases/concat-dynamic-args/package.json new file mode 100644 index 000000000..55b029ac8 --- /dev/null +++ b/tests/fixtures/npm/cases/concat-dynamic-args/package.json @@ -0,0 +1 @@ +{ "name": "concat-dynamic-args-case", "private": true, "type": "module" } diff --git a/tests/fixtures/npm/cases/evolving-let-index/main.ts b/tests/fixtures/npm/cases/evolving-let-index/main.ts new file mode 100644 index 000000000..f34197343 --- /dev/null +++ b/tests/fixtures/npm/cases/evolving-let-index/main.ts @@ -0,0 +1,4 @@ +// Reads of a `let` binding whose checker type comes from string index reads +// see undefined past the end. +import { countUntilBracket, skipSpaces } from "evolving-let"; +console.log(skipSpaces("ab ", 2), skipSpaces("a b", 1), countUntilBracket("abc"), countUntilBracket("ab]c")); diff --git a/tests/fixtures/npm/cases/evolving-let-index/node_modules/evolving-let/index.d.ts b/tests/fixtures/npm/cases/evolving-let-index/node_modules/evolving-let/index.d.ts new file mode 100644 index 000000000..51158faab --- /dev/null +++ b/tests/fixtures/npm/cases/evolving-let-index/node_modules/evolving-let/index.d.ts @@ -0,0 +1 @@ +export declare function skipSpaces(str: string, ptr: number): number; export declare function countUntilBracket(str: string): number; diff --git a/tests/fixtures/npm/cases/evolving-let-index/node_modules/evolving-let/index.js b/tests/fixtures/npm/cases/evolving-let-index/node_modules/evolving-let/index.js new file mode 100644 index 000000000..37cbcac70 --- /dev/null +++ b/tests/fixtures/npm/cases/evolving-let-index/node_modules/evolving-let/index.js @@ -0,0 +1,24 @@ +'use strict'; +// `let c;` takes what an index read past the end answers: undefined. +/** + * @param {string} str + * @param {number} ptr + * @returns {number} + */ +function skipSpaces(str, ptr) { + let c; + while ((c = str[ptr]) === ' ' || c === '\t') ptr++; + return ptr; +} +/** + * @param {string} str + * @returns {number} + */ +function countUntilBracket(str) { + let c; + let ptr = 0; + let n = 0; + while ((c = str[ptr++]) !== ']' && c) n++; + return n; +} +module.exports = { skipSpaces, countUntilBracket }; diff --git a/tests/fixtures/npm/cases/evolving-let-index/node_modules/evolving-let/package.json b/tests/fixtures/npm/cases/evolving-let-index/node_modules/evolving-let/package.json new file mode 100644 index 000000000..02f3315c7 --- /dev/null +++ b/tests/fixtures/npm/cases/evolving-let-index/node_modules/evolving-let/package.json @@ -0,0 +1 @@ +{ "name": "evolving-let", "version": "1.0.0", "main": "index.js", "types": "index.d.ts" } diff --git a/tests/fixtures/npm/cases/evolving-let-index/package.json b/tests/fixtures/npm/cases/evolving-let-index/package.json new file mode 100644 index 000000000..ec244a62b --- /dev/null +++ b/tests/fixtures/npm/cases/evolving-let-index/package.json @@ -0,0 +1 @@ +{ "name": "evolving-let-index-case", "private": true, "type": "module" } diff --git a/tests/fixtures/npm/cases/shape-absent-reads/main.ts b/tests/fixtures/npm/cases/shape-absent-reads/main.ts new file mode 100644 index 000000000..a7351b0c4 --- /dev/null +++ b/tests/fixtures/npm/cases/shape-absent-reads/main.ts @@ -0,0 +1,3 @@ +// Values shape inference must include undefined (or inherited members) for. +import { run } from "absent-reads"; +console.log(run()); diff --git a/tests/fixtures/npm/cases/shape-absent-reads/node_modules/absent-reads/index.d.ts b/tests/fixtures/npm/cases/shape-absent-reads/node_modules/absent-reads/index.d.ts new file mode 100644 index 000000000..da7f0a141 --- /dev/null +++ b/tests/fixtures/npm/cases/shape-absent-reads/node_modules/absent-reads/index.d.ts @@ -0,0 +1 @@ +export declare function run(): string; diff --git a/tests/fixtures/npm/cases/shape-absent-reads/node_modules/absent-reads/index.js b/tests/fixtures/npm/cases/shape-absent-reads/node_modules/absent-reads/index.js new file mode 100644 index 000000000..4f76a10bc --- /dev/null +++ b/tests/fixtures/npm/cases/shape-absent-reads/node_modules/absent-reads/index.js @@ -0,0 +1,48 @@ +'use strict'; +// Reads that can answer undefined: a member inherited through Object.create, +// an absent key under a computed name, an index past the end, a key written +// after the object was created, and a short-circuited optional chain. +function Schema() {} +Schema.prototype.extend = function extend(extra) { + const result = Object.create(Schema.prototype); + result.names = (this.names || []).concat(extra); + return result; +}; +const base = new Schema().extend(['a']); +const grown = base.extend(['b']); +const ESCAPES = {}; +ESCAPES[10] = '\\n'; +ESCAPES[9] = '\\t'; +function escape(text) { + let out = ''; + for (let i = 0; i < text.length; i++) { + const seq = ESCAPES[text.charCodeAt(i)]; + out += seq || text[i]; + } + return out + ' ' + String(ESCAPES[text.charCodeAt(0)]); +} +function lastOf(list) { + const last = list[list.length]; + return last === undefined ? 'past the end' : last; +} +function later() { + const state = { count: 0 }; + const label = state.label; + state.label = 'set'; + return String(label) + ' ' + state.label; +} +function tail(value, key) { + const t = value?.[key].child; + return t === undefined ? 'none' : 'child ' + t; +} +function run() { + return [ + grown.names.join(','), + escape('a\tb\n'), + lastOf([1, 2]), + later(), + tail(null, 'k'), + tail({ k: { child: 3 } }, 'k'), + ].join(' | '); +} +module.exports = { run }; diff --git a/tests/fixtures/npm/cases/shape-absent-reads/node_modules/absent-reads/package.json b/tests/fixtures/npm/cases/shape-absent-reads/node_modules/absent-reads/package.json new file mode 100644 index 000000000..9126f0f10 --- /dev/null +++ b/tests/fixtures/npm/cases/shape-absent-reads/node_modules/absent-reads/package.json @@ -0,0 +1 @@ +{ "name": "absent-reads", "version": "1.0.0", "main": "index.js", "types": "index.d.ts" } diff --git a/tests/fixtures/npm/cases/shape-absent-reads/package.json b/tests/fixtures/npm/cases/shape-absent-reads/package.json new file mode 100644 index 000000000..ff39d6da5 --- /dev/null +++ b/tests/fixtures/npm/cases/shape-absent-reads/package.json @@ -0,0 +1 @@ +{ "name": "shape-absent-reads-case", "private": true, "type": "module" } diff --git a/tests/fixtures/npm/cases/shape-bigint-arith/main.ts b/tests/fixtures/npm/cases/shape-bigint-arith/main.ts new file mode 100644 index 000000000..a3dd19cbb --- /dev/null +++ b/tests/fixtures/npm/cases/shape-bigint-arith/main.ts @@ -0,0 +1,3 @@ +// Values computed by arithmetic on bigints are bigints. +import { run } from "bigint-arith"; +console.log(run()); diff --git a/tests/fixtures/npm/cases/shape-bigint-arith/node_modules/bigint-arith/index.d.ts b/tests/fixtures/npm/cases/shape-bigint-arith/node_modules/bigint-arith/index.d.ts new file mode 100644 index 000000000..da7f0a141 --- /dev/null +++ b/tests/fixtures/npm/cases/shape-bigint-arith/node_modules/bigint-arith/index.d.ts @@ -0,0 +1 @@ +export declare function run(): string; diff --git a/tests/fixtures/npm/cases/shape-bigint-arith/node_modules/bigint-arith/index.js b/tests/fixtures/npm/cases/shape-bigint-arith/node_modules/bigint-arith/index.js new file mode 100644 index 000000000..2949f4c5e --- /dev/null +++ b/tests/fixtures/npm/cases/shape-bigint-arith/node_modules/bigint-arith/index.js @@ -0,0 +1,15 @@ +'use strict'; +// Arithmetic on bigints answers bigints, also through closures. +const nowNanos = (function () { + const scale = BigInt(1000); + return () => BigInt(Date.now()) * scale; +})(); +const elapsed = (function () { + const clock = typeof process === 'object' ? process.hrtime : undefined; + const origin = nowNanos() - clock.bigint(); + return () => origin + clock.bigint(); +})(); +function run() { + return typeof nowNanos() + ' ' + typeof elapsed(); +} +module.exports = { run }; diff --git a/tests/fixtures/npm/cases/shape-bigint-arith/node_modules/bigint-arith/package.json b/tests/fixtures/npm/cases/shape-bigint-arith/node_modules/bigint-arith/package.json new file mode 100644 index 000000000..04084c0a3 --- /dev/null +++ b/tests/fixtures/npm/cases/shape-bigint-arith/node_modules/bigint-arith/package.json @@ -0,0 +1 @@ +{ "name": "bigint-arith", "version": "1.0.0", "main": "index.js", "types": "index.d.ts" } diff --git a/tests/fixtures/npm/cases/shape-bigint-arith/package.json b/tests/fixtures/npm/cases/shape-bigint-arith/package.json new file mode 100644 index 000000000..ce66e72db --- /dev/null +++ b/tests/fixtures/npm/cases/shape-bigint-arith/package.json @@ -0,0 +1 @@ +{ "name": "shape-bigint-arith-case", "private": true, "type": "module" } diff --git a/tests/fixtures/npm/cases/shape-module-identity/main.ts b/tests/fixtures/npm/cases/shape-module-identity/main.ts new file mode 100644 index 000000000..2590da8fc --- /dev/null +++ b/tests/fixtures/npm/cases/shape-module-identity/main.ts @@ -0,0 +1,4 @@ +// Shape inference must not type objects held by module-level bindings as +// native records: a typed return would copy them (identity and writes lost). +import { run } from "module-identity"; +console.log(run()); diff --git a/tests/fixtures/npm/cases/shape-module-identity/node_modules/module-identity/index.d.ts b/tests/fixtures/npm/cases/shape-module-identity/node_modules/module-identity/index.d.ts new file mode 100644 index 000000000..da7f0a141 --- /dev/null +++ b/tests/fixtures/npm/cases/shape-module-identity/node_modules/module-identity/index.d.ts @@ -0,0 +1 @@ +export declare function run(): string; diff --git a/tests/fixtures/npm/cases/shape-module-identity/node_modules/module-identity/index.js b/tests/fixtures/npm/cases/shape-module-identity/node_modules/module-identity/index.js new file mode 100644 index 000000000..a036f74de --- /dev/null +++ b/tests/fixtures/npm/cases/shape-module-identity/node_modules/module-identity/index.js @@ -0,0 +1,21 @@ +'use strict'; +// Objects held by module-level bindings stay the same object when functions +// return them: identity holds and writes through either reference are seen. +const expected = { label: 'same' }; +function wrapper() { return expected; } +function identity(value) { return value; } +const top = { inner: { n: 1 } }; +function topInner() { return top.inner; } +function run() { + const out = []; + out.push(String(wrapper() === expected), String(identity(wrapper()) === expected)); + const t = topInner(); + t.n = 9; + out.push(String(top.inner.n), String(topInner() === top.inner)); + const local = { inner: { n: 1 } }; + const getInner = () => local.inner; + getInner().n = 5; + out.push(String(local.inner.n)); + return out.join(' '); +} +module.exports = { run }; diff --git a/tests/fixtures/npm/cases/shape-module-identity/node_modules/module-identity/package.json b/tests/fixtures/npm/cases/shape-module-identity/node_modules/module-identity/package.json new file mode 100644 index 000000000..2ca8cea54 --- /dev/null +++ b/tests/fixtures/npm/cases/shape-module-identity/node_modules/module-identity/package.json @@ -0,0 +1 @@ +{ "name": "module-identity", "version": "1.0.0", "main": "index.js", "types": "index.d.ts" } diff --git a/tests/fixtures/npm/cases/shape-module-identity/package.json b/tests/fixtures/npm/cases/shape-module-identity/package.json new file mode 100644 index 000000000..1ecd1d81d --- /dev/null +++ b/tests/fixtures/npm/cases/shape-module-identity/package.json @@ -0,0 +1 @@ +{ "name": "shape-module-identity-case", "private": true, "type": "module" } diff --git a/tests/fixtures/npm/cases/static-objects/node_modules/static-ns/index.js b/tests/fixtures/npm/cases/static-objects/node_modules/static-ns/index.js index b1884948c..92ab5dda2 100644 --- a/tests/fixtures/npm/cases/static-objects/node_modules/static-ns/index.js +++ b/tests/fixtures/npm/cases/static-objects/node_modules/static-ns/index.js @@ -171,6 +171,16 @@ ns.Listed.prototype.item = function () { return this.l; }; +// Its prototype is reached through a plain object's field, and that field +// is asked whether a method is enumerable. +ns.Held = function () { + this.h = 2; +}; +ns.Held.prototype.hold = function () { + return this.h; +}; +var protoHolder = { prototype: ns.Held.prototype }; + ns.run = function () { var out = []; var p = new ns.Point(3, 4); @@ -197,6 +207,7 @@ ns.run = function () { out.push(uninitialized, aliasLater.twice(2), tdzAlias.twice(3)); out.push(probeLog.join("|"), new ns.Gadget().spin()); out.push(ns.Listed.prototype.propertyIsEnumerable("item"), new ns.Listed().item()); + out.push(protoHolder.prototype.propertyIsEnumerable("hold"), new ns.Held().hold()); return out.join(" "); }; diff --git a/tests/fixtures/npm/cases/static-stores/js-main.cjs b/tests/fixtures/npm/cases/static-stores/js-main.cjs new file mode 100644 index 000000000..9925086f2 --- /dev/null +++ b/tests/fixtures/npm/cases/static-stores/js-main.cjs @@ -0,0 +1,10 @@ +// The JavaScript consumer of static-stores (main.ts is the TypeScript one). +const { lib, other } = require("static-stores"); + +let near = 0; +for (let i = 0; i < 20; i++) if (lib.near((i - 10) / 10)) near++; +console.log(near, lib.name(), lib.early, lib.after); +console.log(lib.counter.inc(), lib.counter.inc(), lib.twice(21), lib.tagOf.call({ tag: "t" })); +const key = process.argv.length > 99 ? "dbl" : "inc"; +console.log(lib.apply("inc", 1), lib.apply("dbl", 4), lib.applyAny("dbl", 5), lib.applyAny(key, 3)); +console.log(other.seen, other.check()); diff --git a/tests/fixtures/npm/cases/static-stores/main.ts b/tests/fixtures/npm/cases/static-stores/main.ts new file mode 100644 index 000000000..ee05b1adf --- /dev/null +++ b/tests/fixtures/npm/cases/static-stores/main.ts @@ -0,0 +1,11 @@ +// Primitive members stored after their namespace was created, and functions +// with and without a receiver (js-main.cjs is the JavaScript consumer). +import { lib, other } from "static-stores"; + +let near = 0; +for (let i = 0; i < 20; i++) if (lib.near((i - 10) / 10)) near++; +console.log(near, lib.name(), lib.early, lib.after); +console.log(lib.counter.inc(), lib.counter.inc(), lib.twice(21), lib.tagOf.call({ tag: "t" })); +const key = process.argv.length > 99 ? "dbl" : "inc"; +console.log(lib.apply("inc", 1), lib.apply("dbl", 4), lib.applyAny("dbl", 5), lib.applyAny(key, 3)); +console.log(other.seen, other.check()); diff --git a/tests/fixtures/npm/cases/static-stores/node_modules/static-stores/index.d.ts b/tests/fixtures/npm/cases/static-stores/node_modules/static-stores/index.d.ts new file mode 100644 index 000000000..88a4d6aa8 --- /dev/null +++ b/tests/fixtures/npm/cases/static-stores/node_modules/static-stores/index.d.ts @@ -0,0 +1,12 @@ +export declare const lib: { + early: number | undefined; + after: number; + near(x: number): boolean; + name(): string; + counter: { n: number; inc(): number }; + twice(a: number): number; + tagOf(this: { tag: string }): string; + apply(op: "inc" | "dbl", v: number): number; + applyAny(op: string, v: number): number | string; +}; +export declare const other: { seen: number; check(): boolean }; diff --git a/tests/fixtures/npm/cases/static-stores/node_modules/static-stores/index.js b/tests/fixtures/npm/cases/static-stores/node_modules/static-stores/index.js new file mode 100644 index 000000000..8e65f2fc7 --- /dev/null +++ b/tests/fixtures/npm/cases/static-stores/node_modules/static-stores/index.js @@ -0,0 +1,64 @@ +// Primitive members stored after their namespace object was created. Reads +// that provably run after the store read the stored literal; a read that can +// run before it (top-level code before the store, or any read once code has +// run before the store) stays a read. +var lib = lib || {}; +lib.cfg = lib.cfg || {}; +lib.cfg.EPS = 0.5; +lib.cfg.NAME = "v1"; +lib.early = lib.cfg.LATE; +lib.cfg.LATE = 7; +lib.after = lib.cfg.LATE + 1; +lib.near = function (x) { + return Math.abs(x) <= lib.cfg.EPS; +}; +lib.name = function () { + return lib.cfg.NAME + ":" + lib.cfg.LATE; +}; +// Functions that read their own receiver, through an arrow, and not at all. +lib.counter = { + n: 0, + inc: function () { + this.n++; + return this.n; + }, +}; +lib.twice = function (a) { + return a * 2; +}; +lib.tagOf = function () { + const read = () => this.tag; + return read(); +}; +// A constant function table: `apply` is only ever called with literal keys +// of the table, so its dispatch needs no arm for other keys; `applyAny` is +// called with a computed key and keeps reading the table for other keys. +lib.ops = { + inc: function (v) { + return v + 1; + }, + dbl: function (v) { + return v * 2; + }, +}; +lib.apply = function (op, v) { + return lib.ops[op](v); +}; +lib.applyAny = function (op, v) { + return lib.ops[op](v); +}; + +// A store after a call: code ran before it, so no read folds. +var other = {}; +other.log = []; +other.note = function (s) { + other.log.push(s); + return other.log.length; +}; +other.seen = other.note("start"); +other.flag = true; +other.check = function () { + return other.flag; +}; + +module.exports = { lib, other }; diff --git a/tests/fixtures/npm/cases/static-stores/node_modules/static-stores/package.json b/tests/fixtures/npm/cases/static-stores/node_modules/static-stores/package.json new file mode 100644 index 000000000..2d202dde0 --- /dev/null +++ b/tests/fixtures/npm/cases/static-stores/node_modules/static-stores/package.json @@ -0,0 +1 @@ +{ "name": "static-stores", "version": "1.0.0", "main": "index.js", "types": "index.d.ts" } diff --git a/tests/fixtures/npm/cases/static-stores/package.json b/tests/fixtures/npm/cases/static-stores/package.json new file mode 100644 index 000000000..38edc4090 --- /dev/null +++ b/tests/fixtures/npm/cases/static-stores/package.json @@ -0,0 +1 @@ +{ "name": "static-stores-case", "private": true, "type": "module" } diff --git a/tests/fixtures/npm/cases/static-tables/js-main.cjs b/tests/fixtures/npm/cases/static-tables/js-main.cjs index 8433deecd..b1ec4d116 100644 --- a/tests/fixtures/npm/cases/static-tables/js-main.cjs +++ b/tests/fixtures/npm/cases/static-tables/js-main.cjs @@ -20,3 +20,12 @@ try { } catch (e) { console.log(e instanceof TypeError); } +for (const holder of lib.holders) { + for (const read of [() => lib.modeOn(holder), () => lib.addVia(holder), () => lib.callVia(holder, "mul")]) { + try { + console.log(read()); + } catch (e) { + console.log(e.constructor.name, e.message); + } + } +} diff --git a/tests/fixtures/npm/cases/static-tables/node_modules/static-tables/index.js b/tests/fixtures/npm/cases/static-tables/node_modules/static-tables/index.js index 1150aff5f..c8ea71e5f 100644 --- a/tests/fixtures/npm/cases/static-tables/node_modules/static-tables/index.js +++ b/tests/fixtures/npm/cases/static-tables/node_modules/static-tables/index.js @@ -113,4 +113,20 @@ lib.Pair.prototype.sum = function () { return this.a + this.b; }; +// Receivers that may be a constant object or undefined: each read keeps the +// TypeError a nullish receiver throws. +lib.holders = [ + { mode: lib.Mode, ops: lib.ops, table: lib.ops }, + { mode: undefined, ops: undefined, table: undefined }, +]; +lib.modeOn = function (holder) { + return holder.mode.ON; +}; +lib.addVia = function (holder) { + return holder.ops.add(2, 3); +}; +lib.callVia = function (holder, op) { + return holder.table[op](4, 5); +}; + module.exports = { lib }; diff --git a/tests/harness/ci-stack-scope.test.ts b/tests/harness/ci-stack-scope.test.ts index 4abd321d0..a2186c512 100644 --- a/tests/harness/ci-stack-scope.test.ts +++ b/tests/harness/ci-stack-scope.test.ts @@ -16,8 +16,11 @@ const workflow = load( ) as { jobs: Record }; const scopeJob = workflow.jobs.validation_scope!; const scopeScript = scopeJob.steps!.find((step) => step.run)!.run!; +// Validation jobs that run only nightly and on manual runs, outside the +// required check (the shape overlay over the real-package suites). +const nightlyJobs = ["shape_overlay_native", "shape_overlay"]; const heavyJobs = Object.keys(workflow.jobs).filter( - (name) => !["validation_scope", "code_quality", "test"].includes(name), + (name) => !["validation_scope", "code_quality", "test", ...nightlyJobs].includes(name), ); const repository = "vercel-labs/scriptc"; @@ -160,6 +163,16 @@ const full = "needs.validation_scope.outputs.full == 'true'"; const laneOf = (name: string) => /needs\.lanes\.outputs\.(\w+) == 'true'/.exec(workflow.jobs[name]!.if ?? "")?.[1]; +test("nightly validation jobs run only on schedule or manual runs, outside the required check", () => { + for (const name of nightlyJobs) { + expect(workflow.jobs[name], name).toBeDefined(); + expect(workflow.jobs[name]!.if, name).toBe( + "${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}", + ); + expect(workflow.jobs.test!.needs, name).not.toContain(name); + } +}); + test("every expensive CI job is gated and included in the required test check", () => { for (const name of heavyJobs) { const job = workflow.jobs[name]!; diff --git a/tests/harness/npm-static.test.ts b/tests/harness/npm-static.test.ts index 368d4257f..959dfa2fd 100644 --- a/tests/harness/npm-static.test.ts +++ b/tests/harness/npm-static.test.ts @@ -24,7 +24,10 @@ const cacheDir = join(repoRoot, "node_modules/.cache/scriptc-tests"); const sanitize = process.env["SCRIPTC_SAN"] === "1"; /* Sanitized builds of the published Effect pilots alone take 290 to 330 s on a 16-vCPU host, most of it in * the sanitized native link: near or past the suite's 300 s default. */ -const effectBuildTimeout = sanitize ? 900_000 : undefined; +// The shape overlay (SCRIPTC_SHAPE_OVERLAY=1, validation runs) analyzes the +// program and may lower it a second time without the overlay. +const effectBuildTimeout = + sanitize || process.env["SCRIPTC_SHAPE_OVERLAY"] === "1" ? 900_000 : undefined; // Measured milliseconds per test on a CI runner (scripts/ci-test-costs.mjs). const testCosts = ( JSON.parse(readFileSync(new URL("./npm-static-costs.json", import.meta.url), "utf8")) as { diff --git a/tests/harness/static-stores.test.ts b/tests/harness/static-stores.test.ts new file mode 100644 index 000000000..7296580fd --- /dev/null +++ b/tests/harness/static-stores.test.ts @@ -0,0 +1,46 @@ +/* Static-object specialization (shape-inference/static-objects.ts): with the + * shape overlay on, the npm case static-stores (through its JavaScript + * consumer, js-main.cjs) reads primitive members stored after their + * namespace object was created as literals where the read provably runs + * after the store, keeps the reads that can run before it, and prints what + * Node prints. Its functions with and without a receiver read `this` as + * Node does. The npm suites run the TypeScript consumer with and without + * the overlay. */ +import { execFile } from "node:child_process"; +import { mkdirSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { promisify } from "node:util"; +import { expect, test } from "vitest"; +import { compile } from "@scriptc/compiler"; + +const execFileAsync = promisify(execFile); +const repoRoot = join(import.meta.dirname, "../.."); +const entry = join(repoRoot, "tests/fixtures/npm/cases/static-stores/js-main.cjs"); +const outDir = join(repoRoot, "node_modules/.cache/scriptc-tests/static-stores"); +const sanitize = process.env["SCRIPTC_SAN"] === "1"; + +test( + "members stored after their namespace was created read as literals after the store", + { timeout: 600_000 }, + async () => { + mkdirSync(outDir, { recursive: true }); + const result = await compile(entry, { + outPath: join(outDir, "main"), + outDir, + sanitize, + shapeOverlay: true, + }); + if (!result.ok) + throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); + const report = (result.warnings ?? []).find((w) => w.code === "SC6103"); + expect(report?.message).toMatch(/2 function table calls, 4 constant reads\)/); + const pending = execFileAsync(result.binaryPath, [], { cwd: dirname(entry), encoding: "utf8" }); + pending.child.stdin?.end(); + const { stdout } = await pending; + const node = await execFileAsync(process.execPath, [entry], { + cwd: dirname(entry), + encoding: "utf8", + }); + expect(stdout).toBe(node.stdout); + }, +); diff --git a/tests/harness/static-tables.test.ts b/tests/harness/static-tables.test.ts index 2d1e88c65..9f7782e2b 100644 --- a/tests/harness/static-tables.test.ts +++ b/tests/harness/static-tables.test.ts @@ -3,7 +3,8 @@ * consumer, js-main.cjs) calls its constant function table directly, reads * constant primitive members as literals, refers to namespace members of * another module through that module's exports, keeps every form whose - * dynamic read is observable, and prints what Node prints. The npm suites + * dynamic read is observable, keeps the TypeError of a receiver that may + * be undefined, and prints what Node prints. The npm suites * run the TypeScript consumer with and without the overlay. */ import { execFile } from "node:child_process"; import { mkdirSync } from "node:fs"; @@ -32,7 +33,7 @@ test( if (!result.ok) throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); const report = (result.warnings ?? []).find((w) => w.code === "SC6103"); - expect(report?.message).toMatch(/1 function table call, 8 constant reads\)/); + expect(report?.message).toMatch(/2 function table calls, 9 constant reads\)/); const pending = execFileAsync(result.binaryPath, [], { cwd: dirname(entry), encoding: "utf8" }); pending.child.stdin?.end(); const { stdout } = await pending;