From 6c07441c243310e6beac9f84031eafc0e1177f9d Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 18 Oct 2026 19:55:52 -0500 Subject: [PATCH 01/35] Check programs with tsc-ts by default scriptc's frontend talks to a TypeScript 7 API server. tsc-ts serves the same API as tsgo, and it is now the default checker; SCRIPTC_CHECKER=tsgo selects tsgo, which scriptc still includes. - @scriptc/compiler depends on tsc-ts 0.0.3. The native distribution ships its platform package as lib/tsc-ts, named by a new tsc_ts field of the toolchain manifest; a manifest without it takes the executable of an installed tsc-ts package. - The executable cache key includes the tsc-ts executable's identity, in the compiler and in the CLI's startup route. - tsc-ts keeps the syntax trees it builds for the life of its process, so the source parser starts a new server after 256 KB of parsed source instead of letting one server grow with every file. - A CI job compiles every corpus program with both checkers, in release and dev, and requires the same LLVM IR and diagnostics (scripts/checker-identity.mjs). - Tests that compare with the TypeScript 7 SDK, the checker-panic diagnostics, and the protocol tests run against tsgo. --- .github/workflows/ci.yml | 48 ++++++- docs/content/docs/cli.mdx | 3 + package.json | 1 + packages/cli/scripts/install-native.mjs | 2 + packages/cli/src/bootstrap.ts | 8 ++ packages/compiler/package.json | 1 + .../src/executable/executable-cache.test.ts | 3 +- .../src/executable/executable-cache.ts | 5 + .../src/frontend/ts7/checker-backend.test.ts | 75 ++++++++++ .../src/frontend/ts7/checker-backend.ts | 105 ++++++++++++++ packages/compiler/src/frontend/ts7/rpc-api.ts | 10 +- .../src/frontend/ts7/semantic-checker.test.ts | 5 +- .../compiler/src/frontend/ts7/session.test.ts | 5 +- .../src/frontend/ts7/source-parser.test.ts | 27 ++++ .../src/frontend/ts7/source-parser.ts | 33 ++++- packages/compiler/src/index.ts | 8 ++ .../compiler/src/native/toolchain.test.ts | 50 ++++++- packages/compiler/src/native/toolchain.ts | 22 ++- packages/compiler/src/startup-cache.ts | 1 + pnpm-lock.yaml | 78 +++++++++++ pnpm-workspace.yaml | 8 ++ scripts/build-native-cli.mts | 4 +- scripts/checker-identity.mjs | 130 ++++++++++++++++++ scripts/native-cli-assets.mts | 20 ++- scripts/package-native-cli.mjs | 1 + scripts/verify-native-cli.mjs | 1 + tests/harness/diagnostics.test.ts | 12 +- tests/harness/native-cli-packaging.test.ts | 9 +- .../self-hosting-native-toolchain.test.ts | 1 + .../harness/self-hosting-ts7-process.test.ts | 7 +- tests/harness/ts7-client-harness.ts | 6 +- 31 files changed, 665 insertions(+), 24 deletions(-) create mode 100644 packages/compiler/src/frontend/ts7/checker-backend.test.ts create mode 100644 packages/compiler/src/frontend/ts7/checker-backend.ts create mode 100644 scripts/checker-identity.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6f2a6d51d..9e2345e29 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -496,6 +496,30 @@ jobs: - name: TypeScript 7 full parity sweep run: pnpm test:ts7 + # tsc-ts, the default checker, and tsgo (SCRIPTC_CHECKER=tsgo) serve the same TypeScript 7 API. Every corpus + # program must compile to the same LLVM IR, with the same diagnostics, with either checker. + checker_identity: + needs: [validation_scope, lanes] + if: ${{ needs.validation_scope.outputs.full == 'true' && needs.lanes.outputs.linux == 'true' }} + name: test (tsgo and tsc-ts give the same IR) + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + with: + version: 11.1.3 + - uses: actions/setup-node@v4 + with: + node-version-file: .node-version + cache: pnpm + - run: pnpm install --frozen-lockfile + - run: pnpm build + - name: Corpus LLVM IR with tsgo and with tsc-ts (release) + run: pnpm test:checker-identity --optimization release + - name: Corpus LLVM IR with tsgo and with tsc-ts (dev) + run: pnpm test:checker-identity --optimization dev + llvm_artifacts_linux_x64: needs: [validation_scope, lanes] if: ${{ needs.validation_scope.outputs.full == 'true' && needs.lanes.outputs.linux == 'true' }} @@ -861,7 +885,28 @@ jobs: # Require this check on the stack base. Upper layers must not pass before # full validation, so untested groups cannot merge. Merge bottom-up. test: - needs: [validation_scope, lanes, code_quality, native_macos, tests, tests_macos, native_linux, effects, bootstrap_seed, bootstrap, linux_host_clang, ts7_parity, llvm_artifacts_linux_x64, llvm_artifacts_macos_arm64, llvm_artifacts_macos_x64, windows_cli, llvm_artifacts_windows_x64, mobile_runtime_packs] + needs: + [ + validation_scope, + lanes, + code_quality, + native_macos, + tests, + tests_macos, + native_linux, + effects, + bootstrap_seed, + bootstrap, + linux_host_clang, + ts7_parity, + checker_identity, + llvm_artifacts_linux_x64, + llvm_artifacts_macos_arm64, + llvm_artifacts_macos_x64, + windows_cli, + llvm_artifacts_windows_x64, + mobile_runtime_packs, + ] # Run even after failed or cancelled lanes: skipped required jobs allow merging. if: ${{ always() }} runs-on: ubuntu-latest @@ -885,6 +930,7 @@ jobs: lane bootstrap "${{ needs.bootstrap.result }}" "${{ needs.lanes.outputs.bootstrap }}" lane linux_host_clang "${{ needs.linux_host_clang.result }}" "${{ needs.lanes.outputs.linux }}" lane ts7_parity "${{ needs.ts7_parity.result }}" "${{ needs.lanes.outputs.linux }}" + lane checker_identity "${{ needs.checker_identity.result }}" "${{ needs.lanes.outputs.linux }}" lane llvm_artifacts_linux_x64 "${{ needs.llvm_artifacts_linux_x64.result }}" "${{ needs.lanes.outputs.linux }}" lane llvm_artifacts_macos_arm64 "${{ needs.llvm_artifacts_macos_arm64.result }}" "${{ needs.lanes.outputs.macos }}" lane llvm_artifacts_macos_x64 "${{ needs.llvm_artifacts_macos_x64.result }}" "${{ needs.lanes.outputs.macos }}" diff --git a/docs/content/docs/cli.mdx b/docs/content/docs/cli.mdx index c4fdf7876..dda2f8d74 100644 --- a/docs/content/docs/cli.mdx +++ b/docs/content/docs/cli.mdx @@ -178,6 +178,9 @@ Verifies and loads the installed precompiled runtime objects and vendored librar
SCRIPTC_RUNTIME_PACK
A runtime pack directory to use instead of an installed @scriptc/runtime-<target> package.
+ +
SCRIPTC_CHECKER
+
The TypeScript 7 API server that checks the program: tsc-ts (the default), which scriptc includes, or tsgo, from the typescript package scriptc also includes. Both give the same diagnostics and generated code.
```console diff --git a/package.json b/package.json index 7318e04f5..03a763372 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "bench:backend-analysis": "node scripts/bench-backend-analysis.mjs", "test": "vitest run", "test:ts7": "node scripts/test-ts7.mjs", + "test:checker-identity": "node scripts/checker-identity.mjs", "generate:ts7-ast": "node scripts/generate-ts7-ast-schema.mjs", "test:fetch-conformance": "vitest run tests/harness/fetch-conformance.test.ts", "test:test262": "tsx tests/test262/run.ts", diff --git a/packages/cli/scripts/install-native.mjs b/packages/cli/scripts/install-native.mjs index 0d125970d..84eb846bc 100644 --- a/packages/cli/scripts/install-native.mjs +++ b/packages/cli/scripts/install-native.mjs @@ -42,6 +42,7 @@ export function relocateToolchain(manifest, sourceDirectory, destinationDirector const pathFrom = (value) => relative(destinationDirectory, resolve(sourceDirectory, value)); for (const name of [ "ts7", + "tsc_ts", "llvm_package", "runtime_pack", "runtime_sources", @@ -96,6 +97,7 @@ export function installNativeCli( if (process.platform !== "win32") { for (const path of [ manifest.ts7, + ...(manifest.tsc_ts === undefined ? [] : [manifest.tsc_ts]), manifest.comptime, join(manifest.llvm_package, "bin/scriptc-llvm-codegen"), ]) { diff --git a/packages/cli/src/bootstrap.ts b/packages/cli/src/bootstrap.ts index 2d12883fa..1466707d6 100644 --- a/packages/cli/src/bootstrap.ts +++ b/packages/cli/src/bootstrap.ts @@ -116,6 +116,13 @@ async function tryFastPath(): Promise { // the whole compiler graph to rediscover it. const helperRuntimePackTarget = !values.sanitize ? startup.precompiledRuntimePackTarget() : null; const helperObjectRoute = helperRuntimePackTarget !== null; + // The full compiler reports a checker it cannot find. + let checker: string | undefined; + try { + checker = startup.checkerCacheIdentity(); + } catch { + return null; + } const hit = await startup.readRoutedExecutableCache(root, { entryPath: input, outDir, @@ -126,6 +133,7 @@ async function tryFastPath(): Promise { ...(optimization === "dev" || optimization === "speed" ? { optimization } : {}), ...(values.strip ? { strip: true as const } : {}), ...(values["detached-exit"] ? { detachedExit: true as const } : {}), + ...(checker === undefined ? {} : { checker }), npmStatic, ffiProfile: ffiPath === null ? null : { path: ffiPath, bytes: ffiBytes! }, target: `${process.env["SCRIPTC_TARGET"] ?? "native"}:${buildPlatform}:${process.arch}:${ diff --git a/packages/compiler/package.json b/packages/compiler/package.json index 6e5442dc6..b6a18db06 100644 --- a/packages/compiler/package.json +++ b/packages/compiler/package.json @@ -39,6 +39,7 @@ }, "dependencies": { "@scriptc/runtime": "workspace:*", + "tsc-ts": "0.0.3", "typescript": "7.0.2", "typescript5": "npm:typescript@5.9.3" }, diff --git a/packages/compiler/src/executable/executable-cache.test.ts b/packages/compiler/src/executable/executable-cache.test.ts index e5887e855..10c29f2bc 100644 --- a/packages/compiler/src/executable/executable-cache.test.ts +++ b/packages/compiler/src/executable/executable-cache.test.ts @@ -198,7 +198,7 @@ test("early executable cache misses on source and resolution changes", async () expect(await readEarlyExecutableCache(f.root, f.options)).toBeNull(); }); -test("early executable keys isolate compile modes, paths, implementation, and FFI manifests", async () => { +test("early executable keys isolate compile modes, paths, implementation, checker, and FFI manifests", async () => { const f = await fixture(); const tracker = new FrontendInputTracker(); tracker.run(() => trackedReadFile(f.source)); @@ -215,6 +215,7 @@ test("early executable keys isolate compile modes, paths, implementation, and FF { ...f.options, optimization: "dev" }, { ...f.options, optimization: "speed" }, { ...f.options, detachedExit: true }, + { ...f.options, checker: "tsc-ts-executable-identity" }, { ...f.options, npmStatic: "auto" }, { ...f.options, npmStatic: ["commander"] }, { ...f.options, outPath: join(f.options.outDir, "other") }, diff --git a/packages/compiler/src/executable/executable-cache.ts b/packages/compiler/src/executable/executable-cache.ts index fbd7e4edd..a060e72ae 100644 --- a/packages/compiler/src/executable/executable-cache.ts +++ b/packages/compiler/src/executable/executable-cache.ts @@ -111,6 +111,9 @@ export interface EarlyExecutableCacheOptions { windowsSubsystem?: "gui"; /** Omitted unless --detached-exit is set. */ detachedExit?: true; + /** Omitted for the default tsgo checker; with SCRIPTC_CHECKER=tsc-ts, the + * tsc-ts executable's file identity. */ + checker?: string; npmStatic: readonly string[] | "auto" | null; /** Raw manifest identity: path and bytes. Native archives remain under the * stricter native cache's independent dependency validation. */ @@ -221,6 +224,7 @@ function cacheKey(options: EarlyExecutableCacheOptions): string { ...(options.strip ? ["strip"] : []), ...(options.windowsSubsystem === "gui" ? ["windows-subsystem-gui"] : []), ...(options.detachedExit ? ["detached-exit"] : []), + ...(options.checker !== undefined ? [`checker:${options.checker}`] : []), options.npmStatic === null ? "" : options.npmStatic === "auto" @@ -260,6 +264,7 @@ function routeKey(options: Omit { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); + +function tempRoot(): string { + const root = realpathSync(mkdtempSync(join(tmpdir(), "scriptc-checker-"))); + roots.push(root); + return root; +} + +function writePackage(directory: string, name: string, executable?: string): void { + mkdirSync(directory, { recursive: true }); + writeFileSync(join(directory, "package.json"), JSON.stringify({ name })); + if (executable === undefined) return; + mkdirSync(join(directory, "lib"), { recursive: true }); + writeFileSync(join(directory, "lib", executable), ""); +} + +test("tsc-ts is the default checker and SCRIPTC_CHECKER=tsgo selects tsgo", () => { + expect(checkerBackend({})).toBe("tsc-ts"); + expect(checkerBackend({ SCRIPTC_CHECKER: "" })).toBe("tsc-ts"); + expect(checkerBackend({ SCRIPTC_CHECKER: "tsc-ts" })).toBe("tsc-ts"); + expect(checkerBackend({ SCRIPTC_CHECKER: "tsgo" })).toBe("tsgo"); + expect(() => checkerBackend({ SCRIPTC_CHECKER: "tsc" })).toThrow( + 'SCRIPTC_CHECKER must be tsgo or tsc-ts, not "tsc"', + ); +}); + +test("tsc-ts platform packages are named for the host", () => { + expect(tscTsPlatformPackage("linux", "x64", false)).toBe("@tsc-ts/linux-x64-gnu"); + expect(tscTsPlatformPackage("linux", "arm64", true)).toBe("@tsc-ts/linux-arm64-musl"); + expect(tscTsPlatformPackage("darwin", "arm64", false)).toBe("@tsc-ts/darwin-arm64"); + expect(tscTsPlatformPackage("win32", "x64", false)).toBe("@tsc-ts/win32-x64-msvc"); +}); + +test("the tsc-ts executable is found next to an npm install of tsc-ts", () => { + const root = tempRoot(); + writePackage(join(root, "node_modules/tsc-ts"), "tsc-ts"); + writePackage(join(root, "node_modules/@tsc-ts/linux-x64-gnu"), "@tsc-ts/linux-x64-gnu", "tsc-ts"); + mkdirSync(join(root, "src/deep"), { recursive: true }); + expect(tscTsExecutable([join(root, "src/deep")], "@tsc-ts/linux-x64-gnu", "linux")).toBe( + join(root, "node_modules/@tsc-ts/linux-x64-gnu/lib/tsc-ts"), + ); +}); + +test("the tsc-ts executable is found in a pnpm install of tsc-ts", () => { + const root = tempRoot(); + const store = join(root, "node_modules/.pnpm/tsc-ts@0.0.3/node_modules"); + writePackage(join(store, "tsc-ts"), "tsc-ts"); + writePackage(join(store, "@tsc-ts/win32-x64-msvc"), "@tsc-ts/win32-x64-msvc", "tsc-ts.exe"); + mkdirSync(join(root, "app/node_modules"), { recursive: true }); + symlinkSync(join(store, "tsc-ts"), join(root, "app/node_modules/tsc-ts")); + expect(tscTsExecutable([join(root, "app")], "@tsc-ts/win32-x64-msvc", "win32")).toBe( + join(store, "@tsc-ts/win32-x64-msvc/lib/tsc-ts.exe"), + ); +}); + +test("a missing tsc-ts install or platform package is an error", () => { + const root = tempRoot(); + expect(() => tscTsExecutable([root], "@tsc-ts/linux-x64-gnu", "linux")).toThrow( + "the tsc-ts package is not installed", + ); + writePackage(join(root, "node_modules/tsc-ts"), "tsc-ts"); + expect(() => tscTsExecutable([root], "@tsc-ts/linux-x64-gnu", "linux")).toThrow( + "has no @tsc-ts/linux-x64-gnu executable", + ); +}); diff --git a/packages/compiler/src/frontend/ts7/checker-backend.ts b/packages/compiler/src/frontend/ts7/checker-backend.ts new file mode 100644 index 000000000..a2c989edb --- /dev/null +++ b/packages/compiler/src/frontend/ts7/checker-backend.ts @@ -0,0 +1,105 @@ +import { existsSync, realpathSync, statSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +/** The TypeScript 7 API servers a compile can check with: tsc-ts, the default, + * and tsgo from the `typescript` package, selected with `SCRIPTC_CHECKER=tsgo`. + * Both serve the same API and give the same results. */ +export type CheckerBackend = "tsgo" | "tsc-ts"; + +export function checkerBackend(env: NodeJS.ProcessEnv = process.env): CheckerBackend { + const value = env["SCRIPTC_CHECKER"]; + if (value === undefined || value === "" || value === "tsc-ts") return "tsc-ts"; + if (value === "tsgo") return "tsgo"; + throw new Error(`SCRIPTC_CHECKER must be tsgo or tsc-ts, not ${JSON.stringify(value)}`); +} + +/** tsc-ts's platform package for a host: `@tsc-ts/linux-x64-gnu`, + * `@tsc-ts/darwin-arm64`, `@tsc-ts/win32-x64-msvc`, and so on. */ +export function tscTsPlatformPackage( + platform: string, + architecture: string, + musl: boolean, +): string { + const abi = + platform === "linux" ? (musl ? "-musl" : "-gnu") : platform === "win32" ? "-msvc" : ""; + return `@tsc-ts/${platform}-${architecture}${abi}`; +} + +let hostMusl: boolean | undefined; + +/** Whether this process runs on musl, the libc of tsc-ts's `-musl` Linux packages: Node reports no glibc + * runtime version there. The libc cannot change within a process, and a report is costly to generate. */ +export function hostIsMusl(): boolean { + if (process.platform !== "linux") return false; + if (hostMusl === undefined) { + const report = process.report?.getReport() as + | { header?: { glibcVersionRuntime?: unknown } } + | undefined; + hostMusl = typeof report?.header?.glibcVersionRuntime !== "string"; + } + return hostMusl; +} + +/** The checker's part of an executable cache key: the identity of the tsc-ts + * executable, or undefined for tsgo, whose keys predate the choice. */ +export function checkerCacheIdentity(env: NodeJS.ProcessEnv = process.env): string | undefined { + if (checkerBackend(env) === "tsgo") return undefined; + const path = hostTscTsExecutable(); + const info = statSync(path); + return JSON.stringify({ + path, + canonical: realpathSync(path), + dev: info.dev, + ino: info.ino, + size: info.size, + mtime: info.mtimeMs, + ctime: info.ctimeMs, + }); +} + +/** The Node host's tsc-ts executable: that of the `tsc-ts` package installed + * with this compiler, or else for the working directory. */ +export function hostTscTsExecutable(): string { + return tscTsExecutable( + [dirname(fileURLToPath(import.meta.url)), process.cwd()], + tscTsPlatformPackage(process.platform, process.arch, hostIsMusl()), + process.platform, + ); +} + +/** The tsc-ts executable of the `tsc-ts` package in the nearest + * `node_modules` above each of `starts`, in order. The platform package is + * installed next to `tsc-ts` (npm) or in the directory that holds it (pnpm). */ +export function tscTsExecutable( + starts: readonly string[], + platformPackage: string, + platform: string, +): string { + const name = platform === "win32" ? "tsc-ts.exe" : "tsc-ts"; + for (const start of starts) { + const packageRoot = installedPackage(start, "tsc-ts"); + if (packageRoot === undefined) continue; + const root = realpathSync(packageRoot); + for (const candidate of [ + join(root, "node_modules", platformPackage), + join(dirname(root), platformPackage), + ]) { + const executable = join(candidate, "lib", name); + if (existsSync(executable)) return executable; + } + throw new Error(`tsc-ts checker: ${root} has no ${platformPackage} executable`); + } + throw new Error("tsc-ts checker: the tsc-ts package is not installed"); +} + +function installedPackage(start: string, name: string): string | undefined { + let directory = resolve(start); + for (;;) { + const candidate = join(directory, "node_modules", name); + if (existsSync(join(candidate, "package.json"))) return candidate; + const parent = dirname(directory); + if (parent === directory) return undefined; + directory = parent; + } +} diff --git a/packages/compiler/src/frontend/ts7/rpc-api.ts b/packages/compiler/src/frontend/ts7/rpc-api.ts index a6ae609d4..a21ba6279 100644 --- a/packages/compiler/src/frontend/ts7/rpc-api.ts +++ b/packages/compiler/src/frontend/ts7/rpc-api.ts @@ -1,6 +1,7 @@ import { createRequire } from "node:module"; import { dirname, join } from "node:path"; import { Ts7Api as ConnectedTs7Api } from "./api.js"; +import { checkerBackend, hostTscTsExecutable } from "./checker-backend.js"; import { Ts7RpcClient } from "./rpc-client.js"; import { TS7_FILE_SYSTEM_CALLBACKS, type Ts7FileSystem } from "./rpc-filesystem.js"; import { spawnTs7Wire } from "./rpc-process.js"; @@ -12,8 +13,15 @@ const { default: getExePath } = require(join(packageRoot, "lib/getExePath.js")) }; /** Platform-package discovery and process creation remain at the host - * boundary. The session itself uses no SDK implementation or Node process. */ + * boundary. The session itself uses no SDK implementation or Node process. + * The executable is tsc-ts's (`hostTscTsExecutable`), or tsgo's with + * `SCRIPTC_CHECKER=tsgo`. */ export function ts7Executable(): string { + return checkerBackend() === "tsgo" ? getExePath() : hostTscTsExecutable(); +} + +/** tsgo's executable, from the `typescript` package, whichever checker compiles. */ +export function tsgoExecutable(): string { return getExePath(); } diff --git a/packages/compiler/src/frontend/ts7/semantic-checker.test.ts b/packages/compiler/src/frontend/ts7/semantic-checker.test.ts index c917a6f66..6eac5fcce 100644 --- a/packages/compiler/src/frontend/ts7/semantic-checker.test.ts +++ b/packages/compiler/src/frontend/ts7/semantic-checker.test.ts @@ -6,7 +6,7 @@ import { API } from "typescript/unstable/sync"; import type { Node as SdkNode } from "typescript/unstable/ast"; import { AstNode } from "./ast-node.js"; import { Ts7RpcClient } from "./rpc-client.js"; -import { ts7Executable } from "./rpc-api.js"; +import { tsgoExecutable } from "./rpc-api.js"; import { spawnTs7Wire } from "./rpc-process.js"; import { Ts7Session } from "./session.js"; import { SyntaxKind } from "./enums.js"; @@ -45,8 +45,9 @@ beforeAll(() => { files: [file], }), ); + // Compared with the pinned SDK, which runs tsgo. session = new Ts7Session( - new Ts7RpcClient(spawnTs7Wire(ts7Executable(), ["--api", "--cwd", directory])), + new Ts7RpcClient(spawnTs7Wire(tsgoExecutable(), ["--api", "--cwd", directory])), ); oracle = new API({ cwd: directory }); }); diff --git a/packages/compiler/src/frontend/ts7/session.test.ts b/packages/compiler/src/frontend/ts7/session.test.ts index 47c6d4a82..96149621b 100644 --- a/packages/compiler/src/frontend/ts7/session.test.ts +++ b/packages/compiler/src/frontend/ts7/session.test.ts @@ -11,7 +11,7 @@ import { type Ts7FileSystem, } from "./rpc-filesystem.js"; import { spawnTs7Wire } from "./rpc-process.js"; -import { ts7Executable } from "./rpc-api.js"; +import { tsgoExecutable } from "./rpc-api.js"; import { Ts7Session, Ts7SessionProject, Ts7SessionSnapshot } from "./session.js"; import { AstNode } from "./ast-node.js"; @@ -25,9 +25,10 @@ const fallback: Ts7FileSystem = { getAccessibleEntries: () => undefined, }; +// These tests compare the session with the pinned SDK, which runs tsgo, so the session runs tsgo too. function transport(cwd: string, fs: Ts7FileSystem = fallback, timing = false) { const rpc = new Ts7RpcClient( - spawnTs7Wire(ts7Executable(), [ + spawnTs7Wire(tsgoExecutable(), [ "--api", "--cwd", cwd, diff --git a/packages/compiler/src/frontend/ts7/source-parser.test.ts b/packages/compiler/src/frontend/ts7/source-parser.test.ts index e416d2920..2001f11fb 100644 --- a/packages/compiler/src/frontend/ts7/source-parser.test.ts +++ b/packages/compiler/src/frontend/ts7/source-parser.test.ts @@ -206,3 +206,30 @@ test("JSDoc callable types cannot replace method body views", () => { expect(member.body?.parent).toBe(member); } }); + +test("a parser starts a new server once it has parsed its source budget, and earlier results stay usable", () => { + const directory = mkdtempSync( + join(process.platform === "win32" ? tmpdir() : "/tmp", "scriptc-source-parser-"), + ); + directories.push(directory); + const servers: Ts7Api[] = []; + const parser = new Ts7SourceParser( + (options) => { + const api = new Ts7Api(options); + servers.push(api); + return api; + }, + directory, + 64, + ); + parsers.push(parser); + const first = parser.parse("a.ts", `export const a = ${"1 + ".repeat(20)}1;`, "ts"); + expect(servers).toHaveLength(1); + const second = parser.parse("b.ts", "export const b = 2;", "ts"); + expect(servers).toHaveLength(2); + const third = parser.parse("b.ts", "export const b = 3;", "ts"); + expect(servers).toHaveLength(2); + expect(first.statements[0]!.getText()).toBe(`export const a = ${"1 + ".repeat(20)}1;`); + expect(second.statements[0]!.getText()).toBe("export const b = 2;"); + expect(third.statements[0]!.getText()).toBe("export const b = 3;"); +}); diff --git a/packages/compiler/src/frontend/ts7/source-parser.ts b/packages/compiler/src/frontend/ts7/source-parser.ts index 38605cd64..f9029774c 100644 --- a/packages/compiler/src/frontend/ts7/source-parser.ts +++ b/packages/compiler/src/frontend/ts7/source-parser.ts @@ -7,12 +7,21 @@ import { tsgoPath } from "./session-path.js"; export type Ts7SourceKind = "js" | "jsx" | "ts" | "tsx"; export type Ts7SourceApiFactory = (options: { cwd: string; fs: Ts7FileSystem }) => Ts7Api; +/** The source text, in UTF-16 code units, one parser server parses before a + * new server replaces it. tsc-ts keeps the syntax trees it builds for the + * life of its process, so a server that parses many files grows with them; a + * new one starts empty. Results are detached ASTs and outlive the server that + * made them. */ +const SERVER_SOURCE_BUDGET = 256 * 1024; + /** A syntax-only session with a closed, in-memory filesystem. Parsing a * source string never reads imports, project configuration, or libraries * from disk. The returned AST owns its wire data and outlives the session. * Only one project and one input are retained between requests. */ export class Ts7SourceParser { - private readonly api: Ts7Api; + private api: Ts7Api; + private readonly createApi: Ts7SourceApiFactory; + private parsedSource = 0; private readonly files = new Map(); private readonly configPath: string; private previousPath: string | undefined; @@ -21,10 +30,19 @@ export class Ts7SourceParser { private closed = false; private readonly cwd: string; - constructor(createApi: Ts7SourceApiFactory, cwd = process.cwd()) { + constructor( + createApi: Ts7SourceApiFactory, + cwd = process.cwd(), + private readonly sourceBudget = SERVER_SOURCE_BUDGET, + ) { this.cwd = resolve(cwd); this.configPath = tsgoPath(resolve(this.cwd, "__scriptc-source-parser.tsconfig.json")); - this.api = createApi({ + this.createApi = createApi; + this.api = this.connect(); + } + + private connect(): Ts7Api { + return this.createApi({ cwd: this.cwd, fs: { readFile: (path) => this.files.get(tsgoPath(path)) ?? null, @@ -61,7 +79,14 @@ export class Ts7SourceParser { this.previousFile !== undefined ) return this.previousFile; - const previousPath = this.previousPath; + let previousPath = this.previousPath; + if (this.parsedSource >= this.sourceBudget) { + this.api.close(); + this.api = this.connect(); + this.parsedSource = 0; + previousPath = undefined; + } + this.parsedSource += source.length; this.files.clear(); this.files.set(path, source); this.files.set( diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index 56b787e18..3c16bdf9c 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -8,6 +8,7 @@ import { libraryWasmRefusal, } from "./library/prepare.js"; import { analyzeWithFrontend } from "./frontend/analysis.js"; +import { checkerCacheIdentity } from "./frontend/ts7/checker-backend.js"; import { analyzeWithVerdictCache, fileVerdictStore } from "./coverage/verdict-cache.js"; import { coverageCompilerIdentity } from "./coverage/verdict-startup.js"; import { commentOnlyReplay } from "./coverage/verdict-semantic.js"; @@ -378,6 +379,12 @@ export async function analyzeCached( const nodeFrontend: FrontendFactory = (entry, npmStatic, externalTypes, libraryNpmStatic) => runFrontend(entry, loadProgram, npmStatic, externalTypes, libraryNpmStatic); +/** The checker's part of the executable cache key (`checkerCacheIdentity`), as the CLI's startup route computes it. */ +function checkerField(): { checker?: string } { + const checker = checkerCacheIdentity(); + return checker === undefined ? {} : { checker }; +} + /** The whole pipeline: load → preflight → lower → validate → emit LLVM → link. */ function clearCompileSessionCaches(): void { clearResolveCaches(); @@ -1046,6 +1053,7 @@ async function compileTracked( ...(opts.strip ? { strip: true as const } : {}), ...(opts.windowsSubsystem === "gui" ? { windowsSubsystem: "gui" as const } : {}), ...(opts.detachedExit === true ? { detachedExit: true as const } : {}), + ...checkerField(), npmStatic: opts.npmStatic ?? null, ffiProfile: opts.ffiProfilePath === undefined || ffiProfileBytes === null diff --git a/packages/compiler/src/native/toolchain.test.ts b/packages/compiler/src/native/toolchain.test.ts index 9afccdc18..78d2c9c7e 100644 --- a/packages/compiler/src/native/toolchain.test.ts +++ b/packages/compiler/src/native/toolchain.test.ts @@ -14,6 +14,7 @@ test("native toolchain installation paths relocate with the manifest", () => { compiler_version: "1.2.3", target: "macos-arm64", ts7: "typescript/tsc", + tsc_ts: "tsc-ts/lib/tsc-ts", llvm_package: "llvm", runtime_pack: "runtime", linker: "tools/linker", @@ -23,7 +24,10 @@ test("native toolchain installation paths relocate with the manifest", () => { try { writeFileSync(path, JSON.stringify(manifest)); const toolchain = loadNativeToolchain(path); - expect(toolchain.ts7Executable).toBe(join(root, "typescript/tsc")); + expect(toolchain.ts7Executable).toBe(join(root, "tsc-ts/lib/tsc-ts")); + expect(loadNativeToolchain(path, { SCRIPTC_CHECKER: "tsgo" }).ts7Executable).toBe( + join(root, "typescript/tsc"), + ); expect(toolchain.helperExecutable).toBe(join(root, "llvm/bin/scriptc-llvm-codegen")); expect(toolchain.runtimePackRoot).toBe(join(root, "runtime")); expect(toolchain.linker).toBe(join(root, "tools/linker")); @@ -39,6 +43,7 @@ test("native toolchain installation paths relocate with the manifest", () => { { ...manifest, schema: "other" }, { ...manifest, target: "other" }, { ...manifest, ts7: "" }, + { ...manifest, tsc_ts: "" }, { ...manifest, linker_args: [1] }, ]) { writeFileSync(path, JSON.stringify(invalid)); @@ -71,6 +76,7 @@ test("native toolchain assets resolve from the physical manifest through package compiler_version: "1.2.3", target: "windows-x64-msvc", ts7: asset("typescript/lib/tsc.exe"), + tsc_ts: asset("tsc-ts/lib/tsc-ts.exe"), llvm_package: asset("llvm"), runtime_pack: asset("runtime"), runtime_packs: [{ target: "windows-x64-msvc", path: asset("runtime") }], @@ -93,7 +99,7 @@ test("native toolchain assets resolve from the physical manifest through package const direct = loadNativeToolchain(path); const viaLink = loadNativeToolchain(join(linked, "bin", "scriptc.exe.json")); expect(viaLink).toEqual(direct); - expect(viaLink.ts7Executable).toBe(join(assets, "typescript/lib/tsc.exe")); + expect(viaLink.ts7Executable).toBe(join(assets, "tsc-ts/lib/tsc-ts.exe")); expect(viaLink.helperExecutable).toBe(join(assets, "llvm/bin/scriptc-llvm-codegen.exe")); expect( loadNativeToolchain(join(linked, "bin", "scriptc.exe.json"), { @@ -126,6 +132,7 @@ test("cross-target packs can be added to a project after the compiler is install compiler_version: "1.2.3", target: "macos-arm64", ts7: "../lib/typescript/tsc", + tsc_ts: "../lib/tsc-ts/lib/tsc-ts", llvm_package: "../lib/llvm", runtime_pack: "../lib/runtime-darwin-arm64", linker: "clang", @@ -174,6 +181,7 @@ test("versioned GNU targets select Zig while ordinary host builds keep the insta compiler_version: "1.2.3", target: "linux-x64-gnu", ts7: "tsc", + tsc_ts: "tsc-ts", llvm_package: "llvm", runtime_pack: "runtime", linker: "clang", @@ -192,3 +200,41 @@ test("versioned GNU targets select Zig while ordinary host builds keep the insta rmSync(root, { recursive: true, force: true }); } }); + +test("a manifest without tsc_ts takes the host's tsc-ts executable from the installed package", () => { + const root = realpathSync( + mkdtempSync(join(process.platform === "win32" ? tmpdir() : "/tmp", "scriptc-checker-tools-")), + ); + const path = join(root, "toolchain.json"); + const manifest: NativeToolchainManifest = { + schema: "scriptc.native-toolchain.v1", + compiler_version: "1.2.3", + target: "linux-x64-musl", + ts7: "typescript/tsc", + llvm_package: "llvm", + runtime_pack: "runtime", + linker: "tools/linker", + linker_args: ["cc"], + dsymutil: "dsymutil", + }; + try { + writeFileSync(path, JSON.stringify(manifest)); + const project = join(root, "project"); + const platformPackage = join(project, "node_modules/@tsc-ts/linux-x64-musl"); + mkdirSync(join(project, "node_modules/tsc-ts"), { recursive: true }); + writeFileSync(join(project, "node_modules/tsc-ts/package.json"), '{"name":"tsc-ts"}'); + mkdirSync(join(platformPackage, "lib"), { recursive: true }); + writeFileSync(join(platformPackage, "lib/tsc-ts"), ""); + expect(loadNativeToolchain(path, {}, project).ts7Executable).toBe( + join(platformPackage, "lib/tsc-ts"), + ); + expect(loadNativeToolchain(path, { SCRIPTC_CHECKER: "tsgo" }, project).ts7Executable).toBe( + join(root, "typescript/tsc"), + ); + expect(() => loadNativeToolchain(path, {}, root)).toThrow( + "the tsc-ts package is not installed", + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/packages/compiler/src/native/toolchain.ts b/packages/compiler/src/native/toolchain.ts index 651d87027..83da0f7c2 100644 --- a/packages/compiler/src/native/toolchain.ts +++ b/packages/compiler/src/native/toolchain.ts @@ -6,6 +6,11 @@ import { type NativeTargetSpec, type NativeHelperSpec, } from "../backend/targets.js"; +import { + checkerBackend, + tscTsExecutable, + tscTsPlatformPackage, +} from "../frontend/ts7/checker-backend.js"; export interface NativeToolchain { compilerVersion: string; @@ -34,6 +39,8 @@ export interface NativeToolchainManifest { compiler_version: string; target: string; ts7: string; + /** The tsc-ts executable, the default checker; without it, an installed `tsc-ts` package's. */ + tsc_ts?: string; llvm_package: string; runtime_pack: string; runtime_sources?: string; @@ -106,6 +113,7 @@ export function loadNativeToolchain( } const manifest = raw as NativeToolchainManifest; for (const name of [ + "tsc_ts", "runtime_sources", "declarations", "archiver", @@ -169,7 +177,19 @@ export function loadNativeToolchain( : [...target.defaultLinkerArgs]; return { compilerVersion: manifest.compiler_version, - ts7Executable: pathFrom(root, manifest.ts7), + // tsc-ts, the default checker, comes with the distribution; a development + // build takes it from the tsc-ts package installed with the compiler or for + // the working directory. SCRIPTC_CHECKER=tsgo selects tsgo. + ts7Executable: + checkerBackend(env) === "tsgo" + ? pathFrom(root, manifest.ts7) + : manifest.tsc_ts !== undefined + ? pathFrom(root, manifest.tsc_ts) + : tscTsExecutable( + [root, cwd], + tscTsPlatformPackage(host.platform, host.architecture, host.name.endsWith("-musl")), + host.platform, + ), target, helper: host.helper, helperPackageRoot, diff --git a/packages/compiler/src/startup-cache.ts b/packages/compiler/src/startup-cache.ts index fedc36fd5..c2f82fdad 100644 --- a/packages/compiler/src/startup-cache.ts +++ b/packages/compiler/src/startup-cache.ts @@ -22,5 +22,6 @@ export function precompiledRuntimePackTarget( ): NativeTargetSpec | null { return nativeCodegenTarget(env); } +export { checkerCacheIdentity } from "./frontend/ts7/checker-backend.js"; export { readCoverageVerdict } from "./coverage/verdict-startup.js"; export { coverageEnvelope, coveragePasses, renderCoverage } from "./coverage/report.js"; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 41d9e02ac..cbe529577 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -106,6 +106,9 @@ importers: '@scriptc/runtime': specifier: workspace:* version: link:../runtime + tsc-ts: + specifier: 0.0.3 + version: 0.0.3 typescript: specifier: 7.0.2 version: 7.0.2 @@ -1367,6 +1370,45 @@ packages: '@ts-morph/common@0.11.1': resolution: {integrity: sha512-7hWZS0NRpEsNV8vWJzg7FEz6V8MaLNeJOmwmghqUXTpzk16V1LLZhdo+4QvE/+zv4cVci0OviuJFnqhEfoV3+g==} + '@tsc-ts/darwin-arm64@0.0.3': + resolution: {integrity: sha512-NSP4IhLx658tY21ub3ruraZwUwAC3GCbIf1eM0eOKUHfOBBXqq7LU/Loj3zvlvn/tJUyv2VdIDmzrWI/mxp5sA==} + cpu: [arm64] + os: [darwin] + + '@tsc-ts/darwin-x64@0.0.3': + resolution: {integrity: sha512-+BNvqyygkMx0/oHkjK8AmsPrE2+OcE/hGzT0wa2dndiH8lCSDBs+77C7Kmy/FSTWi9ItJShp1+AuWKgQPEOztg==} + cpu: [x64] + os: [darwin] + + '@tsc-ts/linux-arm64-gnu@0.0.3': + resolution: {integrity: sha512-I3v0J0L387TEkau0BTGiVx2GdstYfOrwVYwb0pJ/LbFURhrcGcsUODP5bdSOIYCtsp3aAX6e2afdqSIlpsxtbw==} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@tsc-ts/linux-arm64-musl@0.0.3': + resolution: {integrity: sha512-f7wD9D6fC1xL7+ifvMAbJ0TqxOuZP04R2/7c8lb7Gor5SFKe85udJoxf0gocK7mbzWzmEiOJ53J1PujJLBVIxA==} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@tsc-ts/linux-x64-gnu@0.0.3': + resolution: {integrity: sha512-BL9zownW4tHv/qqQkF8OlurHNDXi7xtGtJ6PV4qMeRYPMKsivTIGFRoZLW023KcLOr1Zptaof6X3zpv1nJEIpA==} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@tsc-ts/linux-x64-musl@0.0.3': + resolution: {integrity: sha512-g4/lE5sULNGUVLM0Px/jrsjLsQQAYSiL+Rl+JZ+l5JE2k1dXroCejT2+IRB/NucVcz6N8EOUiCRZ13zl+Wtpcg==} + cpu: [x64] + os: [linux] + libc: [musl] + + '@tsc-ts/win32-x64-msvc@0.0.3': + resolution: {integrity: sha512-oGtboSHgpuQP1dHAHqWyGjAWCpvHDpciYCi/2ywcp6MT+oOHxC99X48H/PZr29fzYVxCVeaEqWZ/VFU9qLZaKg==} + cpu: [x64] + os: [win32] + '@tybys/wasm-util@0.10.3': resolution: {integrity: sha512-F3fo1MYrRJYL3zER0OUOmkutjr1Vp23m7OsSgp7nq4SP6OqX6C/56XFIPAl5bt3zaBRjmW7SGz3u/6LwFpYcOg==} @@ -2524,6 +2566,11 @@ packages: ts-toolbelt@6.15.5: resolution: {integrity: sha512-FZIXf1ksVyLcfr7M317jbB67XFJhOO1YqdTcuGaq9q5jLUoTikukZ+98TPjKiP2jC5CgmYdWWYs0s2nLSU0/1A==} + tsc-ts@0.0.3: + resolution: {integrity: sha512-Qr0i/BxP3ULIPDfr38vRb02x7sj6Q0r70Gubgu4ty2J4CEDCUKzT8RT2KMNqLXI3ZAhl7cncJq7a4LjKYZ1+0g==} + engines: {node: '>=20'} + hasBin: true + tslib@2.8.1: resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} @@ -3394,6 +3441,27 @@ snapshots: mkdirp: 1.0.4 path-browserify: 1.0.1 + '@tsc-ts/darwin-arm64@0.0.3': + optional: true + + '@tsc-ts/darwin-x64@0.0.3': + optional: true + + '@tsc-ts/linux-arm64-gnu@0.0.3': + optional: true + + '@tsc-ts/linux-arm64-musl@0.0.3': + optional: true + + '@tsc-ts/linux-x64-gnu@0.0.3': + optional: true + + '@tsc-ts/linux-x64-musl@0.0.3': + optional: true + + '@tsc-ts/win32-x64-msvc@0.0.3': + optional: true + '@tybys/wasm-util@0.10.3': dependencies: tslib: 2.8.1 @@ -4730,6 +4798,16 @@ snapshots: ts-toolbelt@6.15.5: {} + tsc-ts@0.0.3: + optionalDependencies: + '@tsc-ts/darwin-arm64': 0.0.3 + '@tsc-ts/darwin-x64': 0.0.3 + '@tsc-ts/linux-arm64-gnu': 0.0.3 + '@tsc-ts/linux-arm64-musl': 0.0.3 + '@tsc-ts/linux-x64-gnu': 0.0.3 + '@tsc-ts/linux-x64-musl': 0.0.3 + '@tsc-ts/win32-x64-msvc': 0.0.3 + tslib@2.8.1: optional: true diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 8f10bfca8..d4d370721 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -7,6 +7,14 @@ minimumReleaseAgeExclude: - "effect@4.0.0" - "@effect/platform-node@4.0.0" - "@effect/platform-node-shared@4.0.0" + - "@tsc-ts/darwin-arm64@0.0.3" + - "@tsc-ts/darwin-x64@0.0.3" + - "@tsc-ts/linux-arm64-gnu@0.0.3" + - "@tsc-ts/linux-arm64-musl@0.0.3" + - "@tsc-ts/linux-x64-gnu@0.0.3" + - "@tsc-ts/linux-x64-musl@0.0.3" + - "@tsc-ts/win32-x64-msvc@0.0.3" + - "tsc-ts@0.0.3" allowBuilds: esbuild: true overrides: diff --git a/scripts/build-native-cli.mts b/scripts/build-native-cli.mts index fa4cb74a7..c8a14adbd 100644 --- a/scripts/build-native-cli.mts +++ b/scripts/build-native-cli.mts @@ -5,7 +5,7 @@ import { basename, dirname, join, relative, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { compile } from "../packages/compiler/src/index.js"; import { NATIVE_TARGETS, nativeCodegenTarget } from "../packages/compiler/src/backend/targets.js"; -import { ts7Executable } from "../packages/compiler/src/frontend/ts7/rpc-api.js"; +import { tsgoExecutable } from "../packages/compiler/src/frontend/ts7/rpc-api.js"; import { stageNativeCliAssets } from "./native-cli-assets.mjs"; import type { NativeToolchainManifest } from "../packages/compiler/src/native/toolchain.js"; @@ -108,7 +108,7 @@ const assets = stageNativeCliAssets({ repository: root, output, target, - ts7: ts7Executable(), + ts7: tsgoExecutable(), compilerVersion, }); // A cross-libc seed may use Zig on the build host. The installed compiler diff --git a/scripts/checker-identity.mjs b/scripts/checker-identity.mjs new file mode 100644 index 000000000..35d01cb93 --- /dev/null +++ b/scripts/checker-identity.mjs @@ -0,0 +1,130 @@ +#!/usr/bin/env node +// Compiles every corpus program to LLVM IR with each TypeScript 7 API server (tsc-ts, the default, and tsgo with +// SCRIPTC_CHECKER=tsgo) and fails unless both give the same IR and the same diagnostics. +// +// Usage: node scripts/checker-identity.mjs [--jobs ] [--optimization release|dev] [--filter ] +// +// It needs `pnpm build`. Each checker compiles in its own worker processes, which read SCRIPTC_CHECKER when they +// start their servers. +import { fork } from "node:child_process"; +import { globSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { availableParallelism, tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { parseArgs } from "node:util"; + +const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const corpusDir = join(repoRoot, "tests/corpus"); +const checkers = ["tsgo", "tsc-ts"]; + +if (process.argv[2] === "--worker") await worker(process.argv.slice(3)); +else await main(); + +async function main() { + const { values } = parseArgs({ + options: { + jobs: { + type: "string", + default: String(Math.max(1, Math.floor(availableParallelism() / 2))), + }, + optimization: { type: "string", default: "release" }, + filter: { type: "string", default: "" }, + }, + }); + if (values.optimization !== "release" && values.optimization !== "dev") + throw new Error("--optimization must be release or dev"); + // The entries llvm-differential.test.ts compiles. + const entries = ["ts", "js", "mjs", "cjs"] + .flatMap((ext) => [ + ...globSync(join(corpusDir, `*.${ext}`)), + ...globSync(join(corpusDir, `*/main.${ext}`)), + ]) + .filter((file) => file.includes(values.filter)) + .sort(); + const jobs = Math.max(1, Number(values.jobs)); + const outRoot = mkdtempSync(join(tmpdir(), "scriptc-checker-identity-")); + try { + const results = new Map(checkers.map((checker) => [checker, new Map()])); + const runs = []; + for (const checker of checkers) + for (let index = 0; index < jobs; index++) + runs.push( + new Promise((done, fail) => { + const child = fork( + fileURLToPath(import.meta.url), + [ + "--worker", + join(outRoot, checker), + values.optimization, + ...entries.filter((_, i) => i % jobs === index), + ], + { env: { ...process.env, SCRIPTC_CHECKER: checker } }, + ); + child.on("message", (record) => results.get(checker).set(record.file, record)); + child.on("error", fail); + child.on("exit", (code) => + code === 0 ? done() : fail(new Error(`${checker} worker exited with ${code}`)), + ); + }), + ); + await Promise.all(runs); + const differences = []; + for (const file of entries) { + const [a, b] = checkers.map((checker) => results.get(checker).get(file)); + const name = file.slice(corpusDir.length + 1); + if (a === undefined || b === undefined) differences.push(`${name}: not compiled`); + else if (a.ok !== b.ok || a.diagnostics !== b.diagnostics) + differences.push( + `${name}: tsgo ${a.ok ? "compiles" : a.diagnostics}; tsc-ts ${b.ok ? "compiles" : b.diagnostics}`, + ); + else if (a.ok && !readFileSync(a.llvm).equals(readFileSync(b.llvm))) + differences.push(`${name}: LLVM IR differs`); + } + if (differences.length > 0) { + console.error( + `${differences.length} of ${entries.length} programs differ between tsgo and tsc-ts:`, + ); + for (const line of differences) console.error(` ${line}`); + process.exitCode = 1; + } else { + console.log( + `${entries.length} programs: tsgo and tsc-ts give the same LLVM IR and diagnostics (${values.optimization}).`, + ); + } + } finally { + rmSync(outRoot, { recursive: true, force: true }); + } +} + +async function worker([outDir, optimization, ...files]) { + const { compile } = await import( + pathToFileURL(join(repoRoot, "packages/compiler/dist/index.js")).href + ); + for (const file of files) { + const name = file.slice(corpusDir.length + 1); + const llvm = join(outDir, `${name}.ll`); + let record; + try { + const result = await compile(file, { + outputKind: "llvm", + outPath: llvm, + outDir: join(outDir, ".work", name), + optimization, + }); + record = result.ok + ? { file, ok: true, llvm } + : { + file, + ok: false, + diagnostics: result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("; "), + }; + } catch (error) { + record = { + file, + ok: false, + diagnostics: `threw ${error instanceof Error ? error.message : String(error)}`, + }; + } + process.send(record); + } +} diff --git a/scripts/native-cli-assets.mts b/scripts/native-cli-assets.mts index 614fe52d5..b3d1436bd 100644 --- a/scripts/native-cli-assets.mts +++ b/scripts/native-cli-assets.mts @@ -2,6 +2,10 @@ import { cpSync, existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync import { basename, dirname, join, relative } from "node:path"; import type { NativeTargetSpec } from "../packages/compiler/src/backend/targets.js"; import type { NativeToolchainManifest } from "../packages/compiler/src/native/toolchain.js"; +import { + tscTsExecutable, + tscTsPlatformPackage, +} from "../packages/compiler/src/frontend/ts7/checker-backend.js"; /** Copy release payloads explicitly; development caches and build trees never ship. */ function copyPackage(source: string, destination: string, members: readonly string[]): void { @@ -28,6 +32,7 @@ export function stageNativeCliAssets(options: { }): Pick< NativeToolchainManifest, | "ts7" + | "tsc_ts" | "llvm_package" | "runtime_pack" | "runtime_packs" @@ -54,6 +59,14 @@ export function stageNativeCliAssets(options: { ); const typescript = join(lib, "typescript"); copyPackage(dirname(dirname(options.ts7)), typescript, ["lib"]); + // tsc-ts, the default checker, from the tsc-ts package the compiler depends on. + const tscTsSource = tscTsExecutable( + [join(repository, "packages/compiler")], + tscTsPlatformPackage(target.platform, target.architecture, target.name.endsWith("-musl")), + target.platform, + ); + const tscTs = join(lib, "tsc-ts"); + copyPackage(dirname(dirname(tscTsSource)), tscTs, ["lib"]); const sources = join(lib, "runtime-sources"); copyPackage(join(repository, "packages/runtime"), sources, ["src", "vendor"]); const declarations = join(lib, "declarations"); @@ -74,11 +87,15 @@ export function stageNativeCliAssets(options: { const identity = JSON.parse(readFileSync(join(typescript, "package.json"), "utf8")) as { version: string; }; + const tscTsIdentity = JSON.parse(readFileSync(join(tscTs, "package.json"), "utf8")) as { + version: string; + }; writeFileSync( join(output, "THIRD_PARTY_NOTICES"), [ - `scriptc ${options.compilerVersion} includes the existing TypeScript ${identity.version}, LLVM, and C runtime dependencies.`, + `scriptc ${options.compilerVersion} includes the existing TypeScript ${identity.version}, tsc-ts ${tscTsIdentity.version}, LLVM, and C runtime dependencies.`, "TypeScript licensing: lib/typescript/LICENSE and lib/typescript/NOTICE.txt.", + "tsc-ts licensing and notices: lib/tsc-ts/LICENSE and lib/tsc-ts/THIRD_PARTY_NOTICES.md.", "LLVM licensing and notices: lib/llvm/LICENSE and lib/llvm/THIRD_PARTY_NOTICES.", "Runtime dependency licenses: lib/runtime-sources/vendor/*/LICENSE*.", "", @@ -86,6 +103,7 @@ export function stageNativeCliAssets(options: { ); return { ts7: relative(bin, join(typescript, "lib", basename(options.ts7))), + tsc_ts: relative(bin, join(tscTs, "lib", basename(tscTsSource))), llvm_package: relative(bin, helper), runtime_pack: relative(bin, runtime), runtime_packs: [{ target: target.name, path: relative(bin, runtime) }], diff --git a/scripts/package-native-cli.mjs b/scripts/package-native-cli.mjs index dd42b8eaa..6d1c05991 100644 --- a/scripts/package-native-cli.mjs +++ b/scripts/package-native-cli.mjs @@ -44,6 +44,7 @@ for (const name of readdirSync(packages) for (const path of [ binary, resolve(bin, hostPath(manifest.ts7)), + resolve(bin, hostPath(manifest.tsc_ts)), resolve(bin, hostPath(manifest.comptime)), join( resolve(bin, hostPath(manifest.llvm_package)), diff --git a/scripts/verify-native-cli.mjs b/scripts/verify-native-cli.mjs index b1e061664..115573b5c 100644 --- a/scripts/verify-native-cli.mjs +++ b/scripts/verify-native-cli.mjs @@ -28,6 +28,7 @@ const within = (path) => { }; for (const name of [ "ts7", + "tsc_ts", "llvm_package", "runtime_pack", "runtime_sources", diff --git a/tests/harness/diagnostics.test.ts b/tests/harness/diagnostics.test.ts index 57ac8c6c0..77fd5937d 100644 --- a/tests/harness/diagnostics.test.ts +++ b/tests/harness/diagnostics.test.ts @@ -33,7 +33,17 @@ describe(`diagnostics corpus (${files.length} programs)`, () => { // any backend runs (that is the suite's contract), so the default // lane is irrelevant by construction — and riding it proves the // fallback machinery never turns a diagnostic into a build. - const result = await compile(file, { outPath: join(outDir, "never"), outDir }); + // The checker-panic programs exercise tsgo's own failures (a panic, and a + // +Infinity it cannot send), so they compile with tsgo. + const checker = process.env["SCRIPTC_CHECKER"]; + if (name.startsWith("checker-panic-")) process.env["SCRIPTC_CHECKER"] = "tsgo"; + let result: Awaited>; + try { + result = await compile(file, { outPath: join(outDir, "never"), outDir }); + } finally { + if (checker === undefined) delete process.env["SCRIPTC_CHECKER"]; + else process.env["SCRIPTC_CHECKER"] = checker; + } if (result.ok) { expect.unreachable(`${name} compiled successfully but must produce diagnostics`); } diff --git a/tests/harness/native-cli-packaging.test.ts b/tests/harness/native-cli-packaging.test.ts index a62a3c0c7..e69f9d9fe 100644 --- a/tests/harness/native-cli-packaging.test.ts +++ b/tests/harness/native-cli-packaging.test.ts @@ -42,6 +42,7 @@ function fixture(windows = true) { compiler_version: identity.version, target, ts7: asset(`../lib/typescript/lib/tsc${windows ? ".exe" : ""}`), + tsc_ts: asset(`../lib/tsc-ts/lib/tsc-ts${windows ? ".exe" : ""}`), llvm_package: asset("../lib/llvm"), runtime_pack: asset("../lib/runtime"), runtime_sources: asset("../lib/runtime-sources"), @@ -54,7 +55,12 @@ function fixture(windows = true) { write(join(distribution, "bin", executable), Buffer.alloc(2048, 1)); chmodSync(join(distribution, "bin", executable), 0o755); write(manifestPath, JSON.stringify(manifest)); - for (const path of [manifest.ts7, manifest.comptime, manifest.wasi_node_runner]) { + for (const path of [ + manifest.ts7, + manifest.tsc_ts, + manifest.comptime, + manifest.wasi_node_runner, + ]) { write(join(distribution, "bin", path.replaceAll("\\", "/")), "fixture"); } write( @@ -112,6 +118,7 @@ test.each([true, false])( readFileSync(join(unpacked, "bin", f.executable + ".json"), "utf8"), ); expect(manifest.ts7).toBe(f.manifest.ts7); + expect(manifest.tsc_ts).toBe(f.manifest.tsc_ts); const runtime = manifest.runtime_packs[0]; expect( JSON.parse( diff --git a/tests/harness/self-hosting-native-toolchain.test.ts b/tests/harness/self-hosting-native-toolchain.test.ts index c22e8a839..99668b005 100644 --- a/tests/harness/self-hosting-native-toolchain.test.ts +++ b/tests/harness/self-hosting-native-toolchain.test.ts @@ -120,6 +120,7 @@ for (const backend of ["llvm"] as const) { compiler_version: manifest.version, target: target.name, ts7: "ts7", + tsc_ts: "tsc-ts", llvm_package: "llvm", runtime_pack: "pack", linker: "ld", diff --git a/tests/harness/self-hosting-ts7-process.test.ts b/tests/harness/self-hosting-ts7-process.test.ts index b8d53dff9..d49a5c092 100644 --- a/tests/harness/self-hosting-ts7-process.test.ts +++ b/tests/harness/self-hosting-ts7-process.test.ts @@ -4,7 +4,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { analyze, compile } from "@scriptc/compiler"; import { expect, test } from "vitest"; -import { ts7Executable } from "../../packages/compiler/src/frontend/ts7/rpc-api.js"; +import { tsgoExecutable } from "../../packages/compiler/src/frontend/ts7/rpc-api.js"; import { runClient } from "./ts7-client-harness.js"; const root = join(import.meta.dirname, "../.."); @@ -96,8 +96,9 @@ for (const backend of ["llvm"] as const) { ); const report = join(dir, "native.json"); // Only ordinary stdio is inherited. The native executable starts the - // actual tsgo process; the test runner never creates its RPC pipes. - const run = spawnSync(built.binaryPath, [ts7Executable(), dir, report], { + // actual tsgo process; the test runner never creates its RPC pipes. As in + // ts7-client-harness.ts, the protocol cases run against tsgo. + const run = spawnSync(built.binaryPath, [tsgoExecutable(), dir, report], { encoding: "utf8", timeout: 45_000, }); diff --git a/tests/harness/ts7-client-harness.ts b/tests/harness/ts7-client-harness.ts index e0d0eccd6..b65dc706f 100644 --- a/tests/harness/ts7-client-harness.ts +++ b/tests/harness/ts7-client-harness.ts @@ -2,7 +2,7 @@ import { spawn, type ChildProcess } from "node:child_process"; import { readFileSync } from "node:fs"; import { fileURLToPath } from "node:url"; import { expect } from "vitest"; -import { ts7Executable } from "../../packages/compiler/src/frontend/ts7/rpc-api.js"; +import { tsgoExecutable } from "../../packages/compiler/src/frontend/ts7/rpc-api.js"; import { TS7_FILE_SYSTEM_CALLBACKS } from "../../packages/compiler/src/frontend/ts7/rpc-filesystem.js"; const root = fileURLToPath(new URL("../..", import.meta.url)); @@ -40,8 +40,10 @@ export async function runClient( directory: string, report: string, ): Promise { + // The protocol cases echo bytes that are not UTF-8 text, which tsc-ts 0.0.3 + // returns decoded, so the transport is checked against tsgo. const server = spawn( - ts7Executable(), + tsgoExecutable(), ["--api", "--timing", "--cwd", directory, `--callbacks=${TS7_FILE_SYSTEM_CALLBACKS}`], { stdio: ["pipe", "pipe", "pipe"], From 9dc45b6776c39b6645987f72f63e9f01f6a39608 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 18 Oct 2026 21:00:33 -0500 Subject: [PATCH 02/35] Check with tsgo by default on Windows --- docs/content/docs/cli.mdx | 2 +- .../src/frontend/ts7/checker-backend.test.ts | 14 +++++++++----- .../compiler/src/frontend/ts7/checker-backend.ts | 12 +++++++++--- packages/compiler/src/native/toolchain.test.ts | 6 +++++- packages/compiler/src/native/toolchain.ts | 8 ++++---- packages/compiler/test/ts7/rpc-api.test.ts | 11 ++++++++++- 6 files changed, 38 insertions(+), 15 deletions(-) diff --git a/docs/content/docs/cli.mdx b/docs/content/docs/cli.mdx index dda2f8d74..8be1a2642 100644 --- a/docs/content/docs/cli.mdx +++ b/docs/content/docs/cli.mdx @@ -180,7 +180,7 @@ Verifies and loads the installed precompiled runtime objects and vendored librar
A runtime pack directory to use instead of an installed @scriptc/runtime-<target> package.
SCRIPTC_CHECKER
-
The TypeScript 7 API server that checks the program: tsc-ts (the default), which scriptc includes, or tsgo, from the typescript package scriptc also includes. Both give the same diagnostics and generated code.
+
The TypeScript 7 API server that checks the program: tsc-ts (the default), which scriptc includes, or tsgo, from the typescript package scriptc also includes. Both give the same diagnostics and generated code. On Windows the default is tsgo, because tsc-ts's API server does not yet accept the named pipe scriptc connects over there.
```console diff --git a/packages/compiler/src/frontend/ts7/checker-backend.test.ts b/packages/compiler/src/frontend/ts7/checker-backend.test.ts index f171d1236..e37357afa 100644 --- a/packages/compiler/src/frontend/ts7/checker-backend.test.ts +++ b/packages/compiler/src/frontend/ts7/checker-backend.test.ts @@ -24,11 +24,15 @@ function writePackage(directory: string, name: string, executable?: string): voi writeFileSync(join(directory, "lib", executable), ""); } -test("tsc-ts is the default checker and SCRIPTC_CHECKER=tsgo selects tsgo", () => { - expect(checkerBackend({})).toBe("tsc-ts"); - expect(checkerBackend({ SCRIPTC_CHECKER: "" })).toBe("tsc-ts"); - expect(checkerBackend({ SCRIPTC_CHECKER: "tsc-ts" })).toBe("tsc-ts"); - expect(checkerBackend({ SCRIPTC_CHECKER: "tsgo" })).toBe("tsgo"); +test("tsc-ts is the default checker except on Windows, and SCRIPTC_CHECKER selects one", () => { + expect(checkerBackend({}, "linux")).toBe("tsc-ts"); + expect(checkerBackend({ SCRIPTC_CHECKER: "" }, "linux")).toBe("tsc-ts"); + expect(checkerBackend({}, "darwin")).toBe("tsc-ts"); + expect(checkerBackend({}, "win32")).toBe("tsgo"); + expect(checkerBackend({ SCRIPTC_CHECKER: "" }, "win32")).toBe("tsgo"); + expect(checkerBackend({ SCRIPTC_CHECKER: "tsc-ts" }, "linux")).toBe("tsc-ts"); + expect(checkerBackend({ SCRIPTC_CHECKER: "tsc-ts" }, "win32")).toBe("tsc-ts"); + expect(checkerBackend({ SCRIPTC_CHECKER: "tsgo" }, "linux")).toBe("tsgo"); expect(() => checkerBackend({ SCRIPTC_CHECKER: "tsc" })).toThrow( 'SCRIPTC_CHECKER must be tsgo or tsc-ts, not "tsc"', ); diff --git a/packages/compiler/src/frontend/ts7/checker-backend.ts b/packages/compiler/src/frontend/ts7/checker-backend.ts index a2c989edb..14418cc94 100644 --- a/packages/compiler/src/frontend/ts7/checker-backend.ts +++ b/packages/compiler/src/frontend/ts7/checker-backend.ts @@ -4,12 +4,18 @@ import { fileURLToPath } from "node:url"; /** The TypeScript 7 API servers a compile can check with: tsc-ts, the default, * and tsgo from the `typescript` package, selected with `SCRIPTC_CHECKER=tsgo`. - * Both serve the same API and give the same results. */ + * Both serve the same API and give the same results. On Windows the default is + * tsgo: the compiler connects to the server over a named pipe there, and tsc-ts's + * API server has no named-pipe transport yet. */ export type CheckerBackend = "tsgo" | "tsc-ts"; -export function checkerBackend(env: NodeJS.ProcessEnv = process.env): CheckerBackend { +export function checkerBackend( + env: NodeJS.ProcessEnv = process.env, + platform: string = process.platform, +): CheckerBackend { const value = env["SCRIPTC_CHECKER"]; - if (value === undefined || value === "" || value === "tsc-ts") return "tsc-ts"; + if (value === undefined || value === "") return platform === "win32" ? "tsgo" : "tsc-ts"; + if (value === "tsc-ts") return "tsc-ts"; if (value === "tsgo") return "tsgo"; throw new Error(`SCRIPTC_CHECKER must be tsgo or tsc-ts, not ${JSON.stringify(value)}`); } diff --git a/packages/compiler/src/native/toolchain.test.ts b/packages/compiler/src/native/toolchain.test.ts index 78d2c9c7e..0ee434300 100644 --- a/packages/compiler/src/native/toolchain.test.ts +++ b/packages/compiler/src/native/toolchain.test.ts @@ -99,7 +99,11 @@ test("native toolchain assets resolve from the physical manifest through package const direct = loadNativeToolchain(path); const viaLink = loadNativeToolchain(join(linked, "bin", "scriptc.exe.json")); expect(viaLink).toEqual(direct); - expect(viaLink.ts7Executable).toBe(join(assets, "tsc-ts/lib/tsc-ts.exe")); + expect(viaLink.ts7Executable).toBe(join(assets, "typescript/lib/tsc.exe")); + expect( + loadNativeToolchain(join(linked, "bin", "scriptc.exe.json"), { SCRIPTC_CHECKER: "tsc-ts" }) + .ts7Executable, + ).toBe(join(assets, "tsc-ts/lib/tsc-ts.exe")); expect(viaLink.helperExecutable).toBe(join(assets, "llvm/bin/scriptc-llvm-codegen.exe")); expect( loadNativeToolchain(join(linked, "bin", "scriptc.exe.json"), { diff --git a/packages/compiler/src/native/toolchain.ts b/packages/compiler/src/native/toolchain.ts index 83da0f7c2..fbcd94251 100644 --- a/packages/compiler/src/native/toolchain.ts +++ b/packages/compiler/src/native/toolchain.ts @@ -177,11 +177,11 @@ export function loadNativeToolchain( : [...target.defaultLinkerArgs]; return { compilerVersion: manifest.compiler_version, - // tsc-ts, the default checker, comes with the distribution; a development - // build takes it from the tsc-ts package installed with the compiler or for - // the working directory. SCRIPTC_CHECKER=tsgo selects tsgo. + // tsc-ts, the default checker except on Windows, comes with the distribution; + // a development build takes it from the tsc-ts package installed with the + // compiler or for the working directory. SCRIPTC_CHECKER selects a checker. ts7Executable: - checkerBackend(env) === "tsgo" + checkerBackend(env, host.platform) === "tsgo" ? pathFrom(root, manifest.ts7) : manifest.tsc_ts !== undefined ? pathFrom(root, manifest.tsc_ts) diff --git a/packages/compiler/test/ts7/rpc-api.test.ts b/packages/compiler/test/ts7/rpc-api.test.ts index e363567c2..e56b55beb 100644 --- a/packages/compiler/test/ts7/rpc-api.test.ts +++ b/packages/compiler/test/ts7/rpc-api.test.ts @@ -62,7 +62,16 @@ test("production RPC bridge preserves pinned SDK parser/checker behavior", () => JSON.stringify({ compilerOptions: { strict: true, noEmit: true, types: [] }, files: [path] }), ); writeFileSync(path, 'export const answer = 42;\nexport const invalid: number = "wrong";\n'); - const nativeClient = new Ts7Api({ cwd: dir, fs: fallback }); + // Compared with the pinned SDK, which runs tsgo, so the bridge runs tsgo too. + const checker = process.env["SCRIPTC_CHECKER"]; + process.env["SCRIPTC_CHECKER"] = "tsgo"; + let nativeClient: Ts7Api; + try { + nativeClient = new Ts7Api({ cwd: dir, fs: fallback }); + } finally { + if (checker === undefined) delete process.env["SCRIPTC_CHECKER"]; + else process.env["SCRIPTC_CHECKER"] = checker; + } const sdk = new API({ cwd: dir }); try { expect(nativeClient.parseConfigFile(config)).toEqual(sdk.parseConfigFile(config)); From a405f856bcebc68018f160f88afa6c905d543647 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Mon, 19 Oct 2026 09:06:15 -0500 Subject: [PATCH 03/35] Iterate class instances whose Symbol.iterator is a generator method A class with a parameterless *[Symbol.iterator]() generator method can now be iterated by for-of, array spread, Array.from, Math.max and Math.min spreads, new Set, new Map, Object.fromEntries and Iterator.from. Each site calls the method once and drives the generator it returns. obj[Symbol.iterator]() and the other well-known symbol methods called through element access now call the class's method, so iterator helpers apply to the result instead of failing on an untyped value. Writes that can replace a well-known symbol method (a symbol-keyed write, Object.defineProperty, a prototype write) now enter the method census, and those classes keep the existing checked or refused paths. --- .../src/frontend/lowering/class-iterables.ts | 35 ++++++++ .../lowering/class-method-mutations.ts | 27 +++++- .../lowering/containers/array-construction.ts | 6 +- .../lowering/containers/collection-methods.ts | 28 ++++++ .../lowering/containers/indexed-objects.ts | 36 +++++++- .../lowering/containers/iterator-helpers.ts | 14 ++- .../src/frontend/lowering/lower-exprs.ts | 4 + .../src/frontend/lowering/lower-stdlib.ts | 6 ++ .../src/frontend/lowering/lower-stmts.ts | 11 +++ .../compiler/src/frontend/lowering/lowerer.ts | 4 + .../src/frontend/lowering/symbol-methods.ts | 30 +++++-- .../test/class-method-mutations.test.ts | 21 +++++ .../6055-class-generator-iterables.ts.json | 6 ++ .../6056-class-symbol-method-calls.ts.json | 6 ++ .../corpus/6055-class-generator-iterables.ts | 86 +++++++++++++++++++ .../corpus/6056-class-symbol-method-calls.ts | 78 +++++++++++++++++ .../6055-class-generator-iterables.ts.json | 1 + .../6056-class-symbol-method-calls.ts.json | 1 + 18 files changed, 386 insertions(+), 14 deletions(-) create mode 100644 packages/compiler/src/frontend/lowering/class-iterables.ts create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6055-class-generator-iterables.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6056-class-symbol-method-calls.ts.json create mode 100644 tests/corpus/6055-class-generator-iterables.ts create mode 100644 tests/corpus/6056-class-symbol-method-calls.ts create mode 100644 tests/harness/corpus-costs/6055-class-generator-iterables.ts.json create mode 100644 tests/harness/corpus-costs/6056-class-symbol-method-calls.ts.json diff --git a/packages/compiler/src/frontend/lowering/class-iterables.ts b/packages/compiler/src/frontend/lowering/class-iterables.ts new file mode 100644 index 000000000..8c58fc0bd --- /dev/null +++ b/packages/compiler/src/frontend/lowering/class-iterables.ts @@ -0,0 +1,35 @@ +import type { IrExpr, IrType } from "../../ir/ir.js"; +import type { Lowerer } from "./lowerer.js"; +import { accessorCall } from "./lower-classes.js"; + +/** The generator a class instance's `[Symbol.iterator]` returns, when that + * method is a generator method taking no arguments: `obj[Symbol.iterator]()`, + * called once, as for-of, spread and Array.from call it. Null for other + * values, and for classes whose iterator the program can replace at run + * time (an instance property, a prototype write). */ +export function classGeneratorIterable(lowerer: Lowerer, value: IrExpr): IrExpr | null { + if (value.type.kind !== "object" || !iteratesThroughGenerator(lowerer, value.type)) return null; + const className = value.type.className; + const ret = lowerer.findMethodOn(lowerer.classes.get(className)!, "sym:iterator")!.sig.ret; + return accessorCall(lowerer, className, "sym:iterator", value, [], ret, value.loc); +} + +/** Whether values of `type` iterate through a generator method (the check + * classGeneratorIterable makes, from the type alone). */ +export function iteratesThroughGenerator(lowerer: Lowerer, type: IrType): boolean { + if (type.kind !== "object") return false; + const info = lowerer.classes.get(type.className); + if (!info || info.def.runtime || info.fields.has("sym:iterator")) return false; + if ( + lowerer.prototypeMethodAccesses.has("sym:iterator") || + lowerer.replacedSymbolMethods.has("sym:iterator") + ) + return false; + const found = lowerer.findMethodOn(info, "sym:iterator"); + return ( + !!found && + found.sig.params.length === 0 && + found.sig.ret.kind === "generator" && + !found.sig.ret.async + ); +} diff --git a/packages/compiler/src/frontend/lowering/class-method-mutations.ts b/packages/compiler/src/frontend/lowering/class-method-mutations.ts index 2a0e3a7eb..cef0bdf5f 100644 --- a/packages/compiler/src/frontend/lowering/class-method-mutations.ts +++ b/packages/compiler/src/frontend/lowering/class-method-mutations.ts @@ -43,9 +43,19 @@ export function collectClassMethodMutations( const jsAssignments: (ts.PropertyAccessExpression | ts.ElementAccessExpression)[] = []; const computedMethods: ts.Expression[] = []; const typedClasses: (ts.ClassDeclaration | ts.ClassExpression)[] = []; + const symbolMethods = new Set(); let unknownKey: ts.Node | undefined; let numericKey: ts.Node | undefined; let unknownOwnField = false; + // `sym:iterator` for `Symbol.iterator`: the name a class method declared + // with a well-known symbol key takes. + const wellKnownMethodName = (key: ts.Expression): string | null => { + while (ts.isParenthesizedExpression(key)) key = key.expression; + if (!ts.isPropertyAccessExpression(key)) return null; + const member = lowerer.stdlibGlobalMember(key, "Symbol"); + if (member === null || !(lowerer.typeOf(key).flags & ts.TypeFlags.ESSymbolLike)) return null; + return `sym:${member}`; + }; // JavaScript method inference and dynamic class factories keep their // existing dispatch contract. The native-call proof applies to typed // declarations (including JavaScript files TypeScript checks), while @@ -556,6 +566,19 @@ export function collectClassMethodMutations( lowerer.prototypeMethodAccesses.set(name, site); } for (const { key, receiver, site } of computedWrites) { + // A class method declared with a well-known symbol key is not a write; + // any other write with one can replace that method. + const symbolName = key ? wellKnownMethodName(unwrap(key)) : null; + if (symbolName) { + const declaration = + (ts.isMethodDeclaration(site) || + ts.isGetAccessorDeclaration(site) || + ts.isSetAccessorDeclaration(site)) && + (ts.isClassDeclaration(site.parent) || ts.isClassExpression(site.parent)); + if (declaration) symbolMethods.add(symbolName); + else lowerer.replacedSymbolMethods.add(symbolName); + continue; + } const type = key ? lowerer.typeOf(key) : undefined; const names = type && (literalValues(type) ?? (key ? literalLoopKeys(unwrap(key)) : null)); if (names) { @@ -586,8 +609,10 @@ export function collectClassMethodMutations( if (String(Number(name)) === name) markName(name, numericKey); } } - if (unknownKey) + if (unknownKey) { for (const method of methods) lowerer.prototypeMethodAccesses.set(method, unknownKey); + for (const method of symbolMethods) lowerer.replacedSymbolMethods.add(method); + } if (!unknownKey && !unknownOwnField) for (const field of arrowFields) { if (!receiverFields.has(field)) lowerer.receiverFreeCallbackFields.add(field); diff --git a/packages/compiler/src/frontend/lowering/containers/array-construction.ts b/packages/compiler/src/frontend/lowering/containers/array-construction.ts index 6072b58b7..5a61f53ce 100644 --- a/packages/compiler/src/frontend/lowering/containers/array-construction.ts +++ b/packages/compiler/src/frontend/lowering/containers/array-construction.ts @@ -44,6 +44,7 @@ import { lowerArrayCallback, callbackArrayElement, } from "./callback-arguments.js"; +import { classGeneratorIterable } from "../class-iterables.js"; /** Build the argument array for a mutating/copying operation while keeping * the receiver's scalar payload ABI. A missing read is represented as the @@ -913,7 +914,7 @@ export function lowerArrayFromCall( // and materializes holes while keeping element reference identity. if ((args.length === 1 || args.length === 2) && !ts.isObjectLiteralExpression(args[0]!)) { const source = lowerer.lowerExpr(args[0]!); - const src = + let src = source.type.kind === "string" && args.length === 2 ? strCharsCall(lowerer, source, loc) : source; @@ -1008,6 +1009,9 @@ export function lowerArrayFromCall( if (src.type.kind === "searchParams" && args.length === 1) { return lowerCheckedArrayFrom(lowerer, lowerer.coerceInto(args[0]!, src, DYN), loc); } + // A class whose [Symbol.iterator] is a generator method iterates the + // generator that method returns. + src = classGeneratorIterable(lowerer, src) ?? src; // A generator (or an iterator adapted into one) drains in order. if (src.type.kind === "generator" && args.length === 1) { const arrT = lowerer.mapTypeOf(lowerer.typeOf(call)); diff --git a/packages/compiler/src/frontend/lowering/containers/collection-methods.ts b/packages/compiler/src/frontend/lowering/containers/collection-methods.ts index a8b782b51..6a5290c0f 100644 --- a/packages/compiler/src/frontend/lowering/containers/collection-methods.ts +++ b/packages/compiler/src/frontend/lowering/containers/collection-methods.ts @@ -42,6 +42,8 @@ import { import { strCharsCall } from "./array-construction.js"; import { arrayElementRead } from "./array-iteration.js"; import { isSearchParamsIterable } from "../builtins/url.js"; +import { classGeneratorIterable } from "../class-iterables.js"; +import { generatorDrain } from "../iterator-adapters.js"; /** Ambient Map method calls. `get`/`set`/`has`/`delete`/`clear` lower to * mapIntrinsic; `forEach` desugars to a direct call of a synthetic loop @@ -1048,6 +1050,14 @@ function setFromSeedValue( } : null; } + // A generator, or a class whose [Symbol.iterator] is a generator method: + // the Set takes every value it yields, in order. + const generator = + classGeneratorIterable(lowerer, source) ?? (source.type.kind === "generator" ? source : null); + if (generator) { + const drained = generatorDrain(lowerer, generator, setT.elem, loc); + return drained ? { kind: "setNew", seed: drained, type: setT, loc } : null; + } let seed: IrExpr; if (source.type.kind === "set" && typeEquals(source.type, setT)) { return { kind: "setIntrinsic", method: "clone", receiver: source, args: [], type: setT, loc }; @@ -1246,6 +1256,24 @@ export function mapFromSeedValue( } : null; } + // A generator of entries, or a class whose [Symbol.iterator] is one: the + // Map takes every [key, value] pair it yields, in order. + const generator = + classGeneratorIterable(lowerer, seed) ?? (seed.type.kind === "generator" ? seed : null); + if (generator) { + const entry: IrType = { + kind: "record", + shapeId: lowerer.shapes.intern( + [ + { name: "0", type: mapT.key }, + { name: "1", type: mapT.value }, + ], + true, + ), + }; + const drained = generatorDrain(lowerer, generator, entry, loc); + return drained ? mapFromSeedValue(lowerer, drained, mapT) : null; + } if (seed.type.kind === "map") { if (!typeEquals(seed.type, mapT)) return null; return { kind: "mapIntrinsic", method: "clone", receiver: seed, args: [], type: mapT, loc }; diff --git a/packages/compiler/src/frontend/lowering/containers/indexed-objects.ts b/packages/compiler/src/frontend/lowering/containers/indexed-objects.ts index 38df368ac..4ba196136 100644 --- a/packages/compiler/src/frontend/lowering/containers/indexed-objects.ts +++ b/packages/compiler/src/frontend/lowering/containers/indexed-objects.ts @@ -28,6 +28,8 @@ import { tryLowerExpression } from "../expressions/try-lower-expression.js"; import { locOf } from "../../program.js"; import { typeKey } from "../../type-mapper.js"; import { type WidthLift } from "../lowerer.js"; +import { classGeneratorIterable, iteratesThroughGenerator } from "../class-iterables.js"; +import { generatorDrain } from "../iterator-adapters.js"; /** Object() and new Object() preserve known object inputs. Primitive * boxing and unproven checked-dynamic values retain their refusal. */ @@ -472,9 +474,37 @@ function lowerTypedObjectFromEntriesCall( if (call.arguments.nodes.length !== 1 || ts.isSpreadElement(call.arguments.nodes[0]!)) return null; const argNode = call.arguments.nodes[0]!; - const argIr = lowerer.mapTypeOf(lowerer.typeOf(argNode)); + let argIr = lowerer.mapTypeOf(lowerer.typeOf(argNode)); + // A generator of entries, or a class whose [Symbol.iterator] is one, + // drains into the entry array the object is built from. + let drained: IrExpr | null = null; + if (argIr?.kind === "generator" || (argIr && iteratesThroughGenerator(lowerer, argIr))) { + const value = lowerer.lowerExpr(argNode); + const generator = + classGeneratorIterable(lowerer, value) ?? (value.type.kind === "generator" ? value : null); + if (generator?.type.kind !== "generator") return null; + // The entries take the result's value type: `[string, V]` for an + // object `{ [k: string]: V }`. + const result = lowerer.mapTypeOf(lowerer.typeOf(call)); + const valueT = + result?.kind === "record" ? lowerer.shapes.get(result.shapeId)?.indexValue : undefined; + if (!valueT) return null; + const entry: IrType = { + kind: "record", + shapeId: lowerer.shapes.intern( + [ + { name: "0", type: STRING }, + { name: "1", type: valueT }, + ], + true, + ), + }; + drained = generatorDrain(lowerer, generator, entry, locOf(call)); + if (!drained) return null; + argIr = drained.type; + } if (argIr?.kind !== "array") return null; - const represented = tryLowerExpression(lowerer, argNode); + const represented = drained ?? tryLowerExpression(lowerer, argNode); if (represented && !typeEquals(represented.type, argIr)) return null; // `Object.fromEntries(rows)` over a `string[][]` VALUE — the env-line // idiom (`envArray.map((env) => env.split('='))`). The checker has no @@ -522,7 +552,7 @@ function lowerTypedObjectFromEntriesCall( // Enumeration/filtering may keep the array in checked-dynamic storage. // Validate its tuple layout before calling a helper with a native-array // ABI; opaque tuple values themselves retain their original references. - const receiver = lowerer.lowerExprExpecting(argNode, argIr); + const receiver = drained ?? lowerer.lowerExprExpecting(argNode, argIr); const key = `obj.fromEntries:${argIr.elem.shapeId}:${resultT.shapeId}`; let helper = lowerer.arrHofHelpers.get(key); if (!helper) { diff --git a/packages/compiler/src/frontend/lowering/containers/iterator-helpers.ts b/packages/compiler/src/frontend/lowering/containers/iterator-helpers.ts index aeefffe48..d439d956e 100644 --- a/packages/compiler/src/frontend/lowering/containers/iterator-helpers.ts +++ b/packages/compiler/src/frontend/lowering/containers/iterator-helpers.ts @@ -34,6 +34,7 @@ import { genValue, stepDrivable, } from "../lower-generators.js"; +import { classGeneratorIterable, iteratesThroughGenerator } from "../class-iterables.js"; import { exitBeforeReturns } from "./for-of.js"; import { iteratorSlotAdapter, @@ -317,9 +318,11 @@ function isHelperObject(lowerer: Lowerer, expr: ts.Expression): boolean { * it rather than a wrapper. A generator (including a helper object or an * array iterator) is itself; an array, string or typed array gives its * values iterator; a Set its values and a Map its entries, as a generator - * over the live collection. The result converts into the call's - * IteratorObject type. Null for other arguments (objects with their own - * next method, untyped values), which keep the Iterator fence. */ + * over the live collection; a class instance whose `[Symbol.iterator]` is + * a generator method gives the generator it returns. The result converts + * into the call's IteratorObject type. Null for other arguments (objects + * with their own next method, untyped values), which keep the Iterator + * fence. */ export function lowerIteratorFromCall( lowerer: Lowerer, call: ts.CallExpression, @@ -332,7 +335,9 @@ export function lowerIteratorFromCall( const argT = lowerer.mapTypeOf(lowerer.typeOf(argNode)); if (resultT?.kind !== "generator" || resultT.async || argT === null) return null; const iterableKinds = ["generator", "array", "string", "bytes", "set", "map"]; - if (!iterableKinds.includes(argT.kind) || (argT.kind === "generator" && argT.async)) return null; + const classIterable = iteratesThroughGenerator(lowerer, argT); + if (!classIterable && !iterableKinds.includes(argT.kind)) return null; + if (argT.kind === "generator" && argT.async) return null; const loc = locOf(call); const value = lowerer.lowerExpr(argNode); const own = (yieldT: IrType): GenType => ({ @@ -344,6 +349,7 @@ export function lowerIteratorFromCall( let iterator: IrExpr | null = null; const t = value.type; if (t.kind === "generator") iterator = value; + else if (classIterable) iterator = classGeneratorIterable(lowerer, value); else if (t.kind === "array") iterator = lowerArrayIteratorValue(lowerer, value, "values", own(t.elem), loc); else if (t.kind === "string") diff --git a/packages/compiler/src/frontend/lowering/lower-exprs.ts b/packages/compiler/src/frontend/lowering/lower-exprs.ts index 09407f71f..654133b5e 100644 --- a/packages/compiler/src/frontend/lowering/lower-exprs.ts +++ b/packages/compiler/src/frontend/lowering/lower-exprs.ts @@ -224,6 +224,7 @@ import { } from "./expressions/optional-chains.js"; import { foldedStringKeyOf } from "./expressions/object-literals.js"; import { tryLowerExpression } from "./expressions/try-lower-expression.js"; +import { classGeneratorIterable } from "./class-iterables.js"; import { classToNumberConversion, classToStringConversion, @@ -6244,6 +6245,9 @@ export function lowerArrayLiteral( loc: locOf(el), }; } + // `[...obj]` over a class whose [Symbol.iterator] is a generator + // method spreads the generator that method returns. + src = classGeneratorIterable(lowerer, src) ?? src; // `[...it]` over a generator (or an iterator adapted into one): // drain it once, in order, into a fresh element array. if (src.type.kind === "generator") { diff --git a/packages/compiler/src/frontend/lowering/lower-stdlib.ts b/packages/compiler/src/frontend/lowering/lower-stdlib.ts index 92d9f0690..01e02c8ff 100644 --- a/packages/compiler/src/frontend/lowering/lower-stdlib.ts +++ b/packages/compiler/src/frontend/lowering/lower-stdlib.ts @@ -42,6 +42,8 @@ import { STATIC_RESPONSE_CALLS, STATIC_RESPONSE_READS, } from "../../compat/fetch-profile.js"; +import { classGeneratorIterable } from "./class-iterables.js"; +import { generatorDrain } from "./iterator-adapters.js"; export function refuseUnsupportedApi( lowerer: Lowerer, @@ -2438,6 +2440,10 @@ export function lowerStdlibMethodCall( src = lowerer.coerceInto(spread.expression, src, inferred); } } + // A generator, or a class whose [Symbol.iterator] is a generator + // method, drains into the number[] the fold reads. + src = classGeneratorIterable(lowerer, src) ?? src; + if (src.type.kind === "generator") src = generatorDrain(lowerer, src, F64, loc) ?? src; if (src.type.kind !== "array" || src.type.elem.kind !== "f64") { lowerer.unsupported( "SC1090", diff --git a/packages/compiler/src/frontend/lowering/lower-stmts.ts b/packages/compiler/src/frontend/lowering/lower-stmts.ts index 36abb9130..766583917 100644 --- a/packages/compiler/src/frontend/lowering/lower-stmts.ts +++ b/packages/compiler/src/frontend/lowering/lower-stmts.ts @@ -217,6 +217,7 @@ import { constituentTypes } from "../ts7/checker.js"; import { npmStaticPackageOfPath } from "../npm-static.js"; import { fenceNodeModuleMutation } from "./lower-node-module.js"; import { fencePerfEntryKeyWalk } from "./builtins/performance.js"; +import { classGeneratorIterable } from "./class-iterables.js"; interface UsingRegistration { acquire: IrStmt[]; @@ -8767,6 +8768,16 @@ export function lowerForOf(lowerer: Lowerer, stmt: ts.ForOfStatement): IrStmt { ); } if (iterable.type.kind === "object") { + // A [Symbol.iterator] generator method: the loop drives the + // generator it returns. + const generator = classGeneratorIterable(lowerer, iterable); + if (generator) + return lowerForOfGenerator( + lowerer, + stmt, + generator as IrExpr & { type: IrType & { kind: "generator" } }, + labels, + ); // Class iterables ([Symbol.iterator]() + next() — classIteratorOf) // drive the same protocol with ordinary method calls. const cit = lowerer.classIteratorOf(iterable.type); diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index d252af250..a4fa362c5 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -2584,6 +2584,10 @@ export class Lowerer { /** Named method slots observed before class collection need a callable * ABI and shared prototype descriptors instead of call-site specialization. */ readonly prototypeMethodAccesses = new Map(); + /** Well-known symbol methods (`sym:iterator`) the program may replace + * through a symbol-keyed write: an instance or prototype property, or a + * reflection call. */ + readonly replacedSymbolMethods = new Set(); /** Typed calls through class-backed protocols, by signature; class * dynamic dispatch adds their native arms (class-protocol-dispatch.ts). */ readonly protocolDispatches = new Map(); diff --git a/packages/compiler/src/frontend/lowering/symbol-methods.ts b/packages/compiler/src/frontend/lowering/symbol-methods.ts index f6a8e68e4..b02e95641 100644 --- a/packages/compiler/src/frontend/lowering/symbol-methods.ts +++ b/packages/compiler/src/frontend/lowering/symbol-methods.ts @@ -2,10 +2,11 @@ import * as ts from "../ts7/adapter.js"; import { locOf } from "../program.js"; import type { IrExpr } from "../../ir/ir.js"; import type { Lowerer } from "./lowerer.js"; -import { accessorCall } from "./lower-classes.js"; +import { accessorCall, type ClassInfo } from "./lower-classes.js"; import { classSymbolKeyOf } from "./symbol-fields.js"; -/** A stable symbol method uses the same completed arguments and virtual +/** A stable symbol method (a module-scope symbol, or a well-known symbol + * such as `Symbol.iterator`) uses the same completed arguments and virtual * dispatch as a string-named class method. Runtime-selected keys stay on * the checked-property path. */ export function lowerClassSymbolMethodCall( @@ -15,12 +16,13 @@ export function lowerClassSymbolMethodCall( const access = call.expression; if (!ts.isElementAccessExpression(access) || access.questionDotToken || call.questionDotToken) return null; - const key = classSymbolKeyOf(lowerer, access.argumentExpression); - if (!key) return null; const type = lowerer.mapTypeOf(lowerer.typeOf(access.expression)); if (type?.kind !== "object") return null; const info = lowerer.classes.get(type.className); - const name = info?.symbolMethods?.get(key.identity); + const key = classSymbolKeyOf(lowerer, access.argumentExpression); + const name = key + ? info?.symbolMethods?.get(key.identity) + : wellKnownSymbolMethod(lowerer, info, access.argumentExpression); if (!name) return null; const method = lowerer.findMethodOn(info ?? null, name); if (!method) return null; @@ -29,3 +31,21 @@ export function lowerClassSymbolMethodCall( const args = lowerer.completeArgs(call.arguments.nodes, method.sig.params, loc, call); return accessorCall(lowerer, type.className, name, receiver, args, method.sig.ret, loc); } + +/** The `sym:` method a well-known symbol key such as `Symbol.iterator` + * names, when the class declares it as a method and the program cannot + * replace it at run time (an instance property of that name, a prototype + * write). */ +function wellKnownSymbolMethod( + lowerer: Lowerer, + info: ClassInfo | undefined, + key: ts.Expression, +): string | null { + while (ts.isParenthesizedExpression(key)) key = key.expression; + if (!info || info.def.runtime || !ts.isPropertyAccessExpression(key)) return null; + const member = lowerer.stdlibGlobalMember(key, "Symbol"); + if (member === null) return null; + const name = `sym:${member}`; + if (info.fields.has(name) || lowerer.replacedSymbolMethods.has(name)) return null; + return name; +} diff --git a/packages/compiler/test/class-method-mutations.test.ts b/packages/compiler/test/class-method-mutations.test.ts index 2258bd54a..b508a3949 100644 --- a/packages/compiler/test/class-method-mutations.test.ts +++ b/packages/compiler/test/class-method-mutations.test.ts @@ -114,3 +114,24 @@ test("a computed key in an Object.assign source keeps checked dispatch", async ( .replace("LITERAL", "Object.assign(target, { [key]: value })"); expect(await callersOf(source, "%dyn.class.properties")).not.toEqual([]); }); + +const range = ` +class Range { + hi: number; + constructor(hi: number) { this.hi = hi; } + *[Symbol.iterator](): Generator { for (let i = 0; i < this.hi; i++) yield i; } +} +const r = new Range(3); +console.log(r[Symbol.iterator]().next().value); +`; + +test("a well-known symbol method call lowers to the class's method", async () => { + expect(await callersOf(range, "%Range.sym:iterator")).not.toEqual([]); +}); + +test("a reflection call with a well-known symbol key keeps checked dispatch", async () => { + const write = "Object.defineProperty(r, Symbol.iterator, { value: function* () {} });\n"; + // The checked path reads the declared method through its value thunk. + const callers = await callersOf(`${range}${write}`, "%Range.sym:iterator"); + expect(callers.filter((name) => !name.startsWith("%method.value:"))).toEqual([]); +}); diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6055-class-generator-iterables.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6055-class-generator-iterables.ts.json new file mode 100644 index 000000000..661995e3f --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6055-class-generator-iterables.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6055-class-generator-iterables.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6056-class-symbol-method-calls.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6056-class-symbol-method-calls.ts.json new file mode 100644 index 000000000..445389b5e --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6056-class-symbol-method-calls.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6056-class-symbol-method-calls.ts" + ], + "diags": [] +} diff --git a/tests/corpus/6055-class-generator-iterables.ts b/tests/corpus/6055-class-generator-iterables.ts new file mode 100644 index 000000000..50641c261 --- /dev/null +++ b/tests/corpus/6055-class-generator-iterables.ts @@ -0,0 +1,86 @@ +// Classes whose [Symbol.iterator] is a generator method iterate the +// generator it returns, called once: in for-of (an early exit closes it), +// array and Math.max spreads, Array.from, new Set, new Map, +// Object.fromEntries and yield*. +class Counter implements Iterable { + n: number; + constructor(n: number) { + this.n = n; + } + *[Symbol.iterator](): Generator { + for (let i = 0; i < this.n; i++) yield i; + } +} +const c = new Counter(3); +for (const v of c) console.log("of", v); +console.log([...c], Array.from(c), Math.max(...c)); +for (const v of c) { + if (v === 1) break; + console.log("early", v); +} + +class Pairs implements Iterable<[string, number]> { + *[Symbol.iterator](): Generator<[string, number]> { + yield ["a", 1]; + yield ["b", 2]; + } +} +class Nums implements Iterable { + *[Symbol.iterator](): Generator { + yield 1; + yield 2; + yield 2; + } +} +const p = new Pairs(); +const n = new Nums(); +console.log(new Set(n).size); +console.log(new Map(p).get("b")); + +console.log(Object.fromEntries(p)); +function* outer(): Generator { + yield 0; + yield* n; +} +console.log([...outer()]); +function* plain(): Generator { + yield 3; + yield 3; + yield 4; +} +console.log(new Set(plain()).size, [...new Set(n)]); + +class Tracked implements Iterable { + log: string[] = []; + *[Symbol.iterator](): Generator { + try { + for (const word of ["a", "b", "c"]) { + this.log.push("yield " + word); + yield word; + } + } finally { + this.log.push("closed"); + } + } +} +const t = new Tracked(); +for (const w of t) { + if (w === "b") break; + t.log.push("saw " + w); +} +console.log(t.log.join(", ")); +class Tree implements Iterable { + value: number; + children: Tree[]; + constructor(value: number, children: Tree[] = []) { + this.value = value; + this.children = children; + } + *[Symbol.iterator](): Generator { + yield this.value; + for (const child of this.children) yield* child; + } +} +class Leafy extends Tree {} +const tree = new Tree(1, [new Tree(2, [new Tree(3)]), new Leafy(4)]); +console.log([...tree], Array.from(new Leafy(9)), Math.min(...tree)); diff --git a/tests/corpus/6056-class-symbol-method-calls.ts b/tests/corpus/6056-class-symbol-method-calls.ts new file mode 100644 index 000000000..2829dc067 --- /dev/null +++ b/tests/corpus/6056-class-symbol-method-calls.ts @@ -0,0 +1,78 @@ +// Well-known symbol methods called through element access, and +// Iterator.from over a class whose [Symbol.iterator] is a generator method: +// both give the class's own generator, so iterator helpers apply to it. +class Range { + lo: number; + hi: number; + constructor(lo: number, hi: number) { + this.lo = lo; + this.hi = hi; + } + *[Symbol.iterator](): Generator { + for (let i = this.lo; i < this.hi; i++) yield i; + } +} +class Labeled extends Range { + label = "l"; +} +class Words { + text: string; + constructor(text: string) { + this.text = text; + } + *[Symbol.iterator](): Generator { + for (const w of this.text.split(" ")) yield w; + } +} +class Money { + cents: number; + constructor(cents: number) { + this.cents = cents; + } + [Symbol.toPrimitive](hint: string): string | number { + return hint === "number" ? this.cents / 100 : `$${(this.cents / 100).toFixed(2)}`; + } +} +class Handle { + name: string; + constructor(name: string) { + this.name = name; + } + [Symbol.dispose](): void { + console.log("dispose", this.name); + } +} + +const r = new Range(1, 6); +const direct = r[Symbol.iterator](); +console.log(direct.next(), direct.next()); +console.log(r[Symbol.iterator]().map((x) => x + 1).toArray()); +const ranges: Range[] = [new Range(0, 3), new Labeled(5, 7)]; +for (const x of ranges) console.log(x[Symbol.iterator]().toArray()); +console.log(r[Symbol.iterator]().filter((x) => x > 3).toArray()); + +const it = Iterator.from(r); +console.log(it.next().value, it.next().value); +console.log([...it]); +console.log(Iterator.from(r).map((x) => x * 10).toArray()); +console.log(Iterator.from(r).filter((x) => x % 2 === 1).toArray()); +console.log(Iterator.from(r).take(2).toArray(), Iterator.from(r).drop(3).toArray()); +console.log(Iterator.from(r).reduce((a, b) => a + b, 0)); +console.log(Iterator.from(r).some((x) => x > 4), Iterator.from(r).every((x) => x > 0)); +console.log(Iterator.from(r).find((x) => x > 2)); +console.log(Iterator.from(new Words("a bb ccc")).map((w) => w.length).toArray()); +Iterator.from(r).forEach((x) => console.log("each", x)); +for (const x of Iterator.from(r).flatMap((x) => [x, -x]).take(4)) console.log(x); +const g = Iterator.from(r); +for (const x of g) { + if (x === 2) break; +} +console.log(g.next()); + +console.log(Iterator.from(r), r[Symbol.iterator](), Iterator.from(r).map((x) => x)); +console.log(Iterator.from(r).flatMap((x) => new Set([x, x + 10])).toArray()); +console.log(r[Symbol.iterator]().flatMap((x) => new Map([[x, String(x)]])).toArray()); + +const m = new Money(1234); +console.log(m[Symbol.toPrimitive]("number"), m[Symbol.toPrimitive]("default")); +new Handle("h")[Symbol.dispose](); diff --git a/tests/harness/corpus-costs/6055-class-generator-iterables.ts.json b/tests/harness/corpus-costs/6055-class-generator-iterables.ts.json new file mode 100644 index 000000000..68b6f9db4 --- /dev/null +++ b/tests/harness/corpus-costs/6055-class-generator-iterables.ts.json @@ -0,0 +1 @@ +{ "plain": 4921, "san": 6451 } diff --git a/tests/harness/corpus-costs/6056-class-symbol-method-calls.ts.json b/tests/harness/corpus-costs/6056-class-symbol-method-calls.ts.json new file mode 100644 index 000000000..4207a0ed0 --- /dev/null +++ b/tests/harness/corpus-costs/6056-class-symbol-method-calls.ts.json @@ -0,0 +1 @@ +{ "plain": 3468, "san": 5120 } From f42fd146d9eaff8d192d9326ccbd97d7499f0997 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Mon, 19 Oct 2026 14:00:23 -0700 Subject: [PATCH 04/35] Lower esbuild's lazy ES modules; select @types-typed packages by default An __esm initializer becomes a run-once closure over its body in the bundle's own scope. Interop bindings it assigns once from require() become that require at the assignment, with their member reads respelled as reads of the required module; a namespace read only through members becomes an object literal of getters over the module's live bindings. Asynchronous initializers, initializer results, and namespaces kept as values (a memoized lazy loader) stay refused with a message naming them. The default selection now also compiles a package typed by a DefinitelyTyped twin that describes its installed release (same major, or major.minor below 1.0), and untyped dependencies of compiled packages. A twin for another release is refused with both versions. Co-Authored-By: Claude Opus 5.5 --- .../src/frontend/npm-static-bundled-chunks.ts | 89 +-- .../frontend/npm-static-bundled-esm.test.ts | 149 +++++ .../src/frontend/npm-static-bundled-esm.ts | 582 ++++++++++++++++++ .../src/frontend/npm-static-rewrite-host.ts | 12 + .../src/frontend/npm-static-rewrite-syntax.ts | 1 + packages/compiler/src/frontend/npm-static.ts | 53 +- packages/compiler/src/frontend/pipeline.ts | 21 +- packages/compiler/src/frontend/services.ts | 10 +- .../node_modules/@types/twinstale/index.d.ts | 1 + .../@types/twinstale/package.json | 5 + .../node_modules/twinstale/index.js | 6 + .../node_modules/twinstale/package.json | 5 + tests/fixtures/npm-static/twinstale-cli.ts | 5 + .../npm/cases/2563-bundled-esm-inits/main.ts | 16 + .../npm/node_modules/gtesm/dist/index.d.ts | 2 + .../npm/node_modules/gtesm/dist/index.js | 90 +++ .../npm/node_modules/gtesm/package.json | 10 + .../npm/node_modules/gtesmws/dist/index.d.ts | 2 + .../npm/node_modules/gtesmws/dist/index.js | 1 + .../npm/node_modules/gtesmws/package.json | 10 + tests/harness/npm-cases.ts | 4 +- tests/harness/npm-static.test.ts | 54 +- 22 files changed, 1064 insertions(+), 64 deletions(-) create mode 100644 packages/compiler/src/frontend/npm-static-bundled-esm.test.ts create mode 100644 packages/compiler/src/frontend/npm-static-bundled-esm.ts create mode 100644 tests/fixtures/npm-static/node_modules/@types/twinstale/index.d.ts create mode 100644 tests/fixtures/npm-static/node_modules/@types/twinstale/package.json create mode 100644 tests/fixtures/npm-static/node_modules/twinstale/index.js create mode 100644 tests/fixtures/npm-static/node_modules/twinstale/package.json create mode 100644 tests/fixtures/npm-static/twinstale-cli.ts create mode 100644 tests/fixtures/npm/cases/2563-bundled-esm-inits/main.ts create mode 100644 tests/fixtures/npm/node_modules/gtesm/dist/index.d.ts create mode 100644 tests/fixtures/npm/node_modules/gtesm/dist/index.js create mode 100644 tests/fixtures/npm/node_modules/gtesm/package.json create mode 100644 tests/fixtures/npm/node_modules/gtesmws/dist/index.d.ts create mode 100644 tests/fixtures/npm/node_modules/gtesmws/dist/index.js create mode 100644 tests/fixtures/npm/node_modules/gtesmws/package.json diff --git a/packages/compiler/src/frontend/npm-static-bundled-chunks.ts b/packages/compiler/src/frontend/npm-static-bundled-chunks.ts index cdb9dff64..28fe4a1b9 100644 --- a/packages/compiler/src/frontend/npm-static-bundled-chunks.ts +++ b/packages/compiler/src/frontend/npm-static-bundled-chunks.ts @@ -31,7 +31,7 @@ export interface BundledChunkSplit { chunks: { path: string; source: string }[]; } -interface Edit { +export interface Edit { start: number; end: number; text: string; @@ -42,11 +42,11 @@ const CHUNK_SHAPE_DEGRADE = "recognizer cannot verify"; /** The helper a counter-suffixed bundler name spells (`__commonJS2`). */ -function helperBase(name: string): string { +export function helperBase(name: string): string { return /^__[A-Za-z]+\d+$/.test(name) ? name.replace(/\d+$/, "") : name; } -function unparen(e: ts.Expression): ts.Expression { +export function unparen(e: ts.Expression): ts.Expression { while (ts.isParenthesizedExpression(e)) e = e.expression; return e; } @@ -54,7 +54,7 @@ function unparen(e: ts.Expression): ts.Expression { /** True for an identifier occurrence that reads or writes a binding (as * opposed to a property name, a label, or a declaration's own name). * Shorthand property values count: `{ x }` reads x. */ -function isReference(id: ts.Identifier): boolean { +export function isReference(id: ts.Identifier): boolean { const p = id.parent; if (p === undefined) return false; if (ts.isPropertyAccessExpression(p)) return p.expression === id; @@ -83,7 +83,7 @@ function isReference(id: ts.Identifier): boolean { /** Names a subtree declares anywhere inside it: variables (destructuring * included), parameters, function and class names, catch bindings. */ -function declaredNames(root: ts.Node): Set { +export function declaredNames(root: ts.Node): Set { const names = new Set(); // Binding patterns nest BindingElements, which the walk visits too. ts.walkPreorder(root, (n) => { @@ -106,7 +106,7 @@ function declaredNames(root: ts.Node): Set { } /** Every binding reference inside a subtree. */ -function references(root: ts.Node): ts.Identifier[] { +export function references(root: ts.Node): ts.Identifier[] { const out: ts.Identifier[] = []; ts.walkPreorder(root, (n) => { if (n.kind >= ts.SyntaxKind.FirstJSDocNode && n.kind <= ts.SyntaxKind.LastJSDocNode) @@ -119,7 +119,7 @@ function references(root: ts.Node): ts.Identifier[] { /** True when the reference is written (assignment target, update operand, * loop target, delete operand). */ -function isWritten(id: ts.Identifier): boolean { +export function isWritten(id: ts.Identifier): boolean { let child: ts.Node = id; for (let p = child.parent; p !== undefined; child = p, p = p.parent) { if (ts.isBinaryExpression(p)) { @@ -247,7 +247,7 @@ interface TopLevel { } /** True when the file opens with a "use strict" directive prologue. */ -function strictPrologue(stmts: readonly ts.Statement[]): boolean { +export function strictPrologue(stmts: readonly ts.Statement[]): boolean { for (const s of stmts) { if (!ts.isExpressionStatement(s) || !ts.isStringLiteral(s.expression)) return false; if (s.expression.text === "use strict") return true; @@ -430,7 +430,7 @@ export function splitBundledChunks( return { text: applyEdits(source, 0, source.length, bundleEdits), chunks: out }; } -function countDeclarations(sf: ts.SourceFile, name: string): number { +export function countDeclarations(sf: ts.SourceFile, name: string): number { let n = 0; ts.walkPreorder(sf, (node) => { if ( @@ -454,7 +454,7 @@ function countDeclarations(sf: ts.SourceFile, name: string): number { /** Applies non-overlapping edits inside [from, to): replaced spans keep * their newlines (blanked text stays line-aligned); edits nested in a * blanked span vanish with it. */ -function applyEdits(source: string, from: number, to: number, edits: Edit[]): string { +export function applyEdits(source: string, from: number, to: number, edits: Edit[]): string { const inside = edits .filter((e) => e.start >= from && e.end <= to) .sort((a, b) => a.start - b.start || b.end - a.end); @@ -472,37 +472,13 @@ function applyEdits(source: string, from: number, to: number, edits: Edit[]): st return text + source.slice(at, to); } -/** The virtual module text for one chunk, or a degrade reason. */ -function chunkSource( - sf: ts.SourceFile, - chunk: Chunk, - chunks: ReadonlyMap, - top: ReadonlyMap, - stable: (name: string) => boolean, - callEdits: readonly Edit[], - bundleStrict: boolean, -): string | { degrade: string } { - const why = (reason: string): { degrade: string } => ({ - degrade: `its bundled CommonJS module '${chunk.name}' ${reason}`, - }); - const [exportsParam, moduleParam] = chunk.params; - const body = chunk.body; - const local = declaredNames(body); - for (const p of chunk.params) { - if ( - local.has(p) || - (p !== "exports" && local.has("exports")) || - (p !== "module" && local.has("module")) - ) - return why( - "rebinds its module parameters inside its body (an identifier-minified bundle the recognizer cannot respell)", - ); - } - - /* The factory's own frame must stay unobservable: its receiver, its - * arguments object, and a top-level return have no module equivalent. */ +/** How code in a factory body (a chunk or a lazy ES-module initializer) + * observes the factory's own frame, or null: its receiver, its arguments + * object, `super`, a meta property, or (when `returns`) a top-level + * return have no module-scope equivalent. */ +export function factoryFrameUse(body: ts.Block, returns: boolean): string | null { let frameUse: string | null = null; - /** The innermost frame boundary between a node and the chunk body. */ + /** The innermost frame boundary between a node and the factory body. */ const frameOf = (n: ts.Node): "factory" | "arrow" | "function" => { let arrow = false; for (let p = n.parent; p !== undefined && p !== body; p = p.parent) { @@ -532,12 +508,43 @@ function chunkSource( use = "reads its factory's arguments object"; else if (ts.isMetaProperty(n)) use = "reads a meta property at its top level"; else if (ts.isReturnStatement(n)) { - if (frameOf(n) === "factory") frameUse = "returns from its top level"; + if (returns && frameOf(n) === "factory") frameUse = "returns from its top level"; return undefined; } if (use !== null && frameOf(n) !== "function") frameUse = use; return undefined; }); + return frameUse; +} + +/** The virtual module text for one chunk, or a degrade reason. */ +function chunkSource( + sf: ts.SourceFile, + chunk: Chunk, + chunks: ReadonlyMap, + top: ReadonlyMap, + stable: (name: string) => boolean, + callEdits: readonly Edit[], + bundleStrict: boolean, +): string | { degrade: string } { + const why = (reason: string): { degrade: string } => ({ + degrade: `its bundled CommonJS module '${chunk.name}' ${reason}`, + }); + const [exportsParam, moduleParam] = chunk.params; + const body = chunk.body; + const local = declaredNames(body); + for (const p of chunk.params) { + if ( + local.has(p) || + (p !== "exports" && local.has("exports")) || + (p !== "module" && local.has("module")) + ) + return why( + "rebinds its module parameters inside its body (an identifier-minified bundle the recognizer cannot respell)", + ); + } + + const frameUse = factoryFrameUse(body, true); if (frameUse !== null) return why(frameUse); const edits: Edit[] = callEdits.filter((e) => e.start >= body.pos && e.end <= body.end); diff --git a/packages/compiler/src/frontend/npm-static-bundled-esm.test.ts b/packages/compiler/src/frontend/npm-static-bundled-esm.test.ts new file mode 100644 index 000000000..9c064cb55 --- /dev/null +++ b/packages/compiler/src/frontend/npm-static-bundled-esm.test.ts @@ -0,0 +1,149 @@ +import { spawnSync } from "node:child_process"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, expect, test } from "vitest"; +import { nodeFrontendServices } from "./services-node.js"; + +const helpers = `var __getOwnPropNames = Object.getOwnPropertyNames; +var __esm = (fn, res) => function __init() { + return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res; +}; +var __defProp = Object.defineProperty; +var __getOwnPropDesc = Object.getOwnPropertyDescriptor; +var __hasOwnProp = Object.prototype.hasOwnProperty; +var __create = Object.create; +var __getProtoOf = Object.getPrototypeOf; +var __export = (target, all) => { + for (var name in all) + __defProp(target, name, { get: all[name], enumerable: true }); +}; +var __copyProps = (to, from, except, desc) => { + if (from && typeof from === "object" || typeof from === "function") { + for (let key of __getOwnPropNames(from)) + if (!__hasOwnProp.call(to, key) && key !== except) + __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable }); + } + return to; +}; +var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps( + isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target, + mod +)); +var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod); +`; + +const dirs: string[] = []; +afterAll(() => { + for (const dir of dirs) rmSync(dir, { recursive: true, force: true }); +}); + +function lower(source: string, file = "/pkg/dist/index.js") { + return nodeFrontendServices().lowerEsmInits(source, file); +} + +function run(file: string) { + const result = spawnSync(process.execPath, [file], { encoding: "utf8" }); + return { status: result.status, stdout: result.stdout, stderr: result.stderr }; +} + +function expectSameBehavior(source: string): string { + const dir = mkdtempSync(join(tmpdir(), "scriptc-esm-inits-")); + dirs.push(dir); + const original = join(dir, "original.js"); + const lowered = join(dir, "index.js"); + writeFileSync(original, source); + const result = lower(source, lowered); + expect(typeof result).toBe("string"); + writeFileSync(lowered, result as string); + const oracle = run(original); + expect(oracle.status).toBe(0); + expect(run(lowered)).toEqual(oracle); + return result as string; +} + +const lazyModule = ` +function helper(x) { COUNT++; return "[" + x + "]"; } +var COUNT; +var init_helper = __esm({ + "src/helper.ts"() { + console.log("helper init"); + COUNT = 0; + } +}); +var lazy_exports = {}; +__export(lazy_exports, { + VALUE: () => VALUE, + value: () => value +}); +function value() { + return VALUE + ":" + COUNT + ":" + import_os.default.EOL.length; +} +var path, import_os, VALUE; +var init_lazy = __esm({ + "src/lazy.ts"() { + path = __toESM(require("path")); + import_os = __toESM(require("os")); + init_helper(); + console.log("lazy init"); + VALUE = helper(path.basename("/a/b.txt")); + } +}); +`; + +test("initializers run once, at their first call, and namespaces read live bindings", () => { + const lowered = expectSameBehavior(`"use strict"; +${helpers}${lazyModule} +console.log("start"); +function load() { + const lazy = (init_lazy(), __toCommonJS(lazy_exports)); + return lazy.value() + " " + lazy.VALUE; +} +console.log(load()); +COUNT = 5; +console.log(load(), (init_lazy(), __toCommonJS(lazy_exports)).value()); +`); + expect(lowered).not.toContain("__esm({"); + expect(lowered).not.toContain("import_os"); + expect(lowered).toContain('(require("os")).EOL'); +}); + +test("minified initializers sharing a var statement lower the same way", () => { + expectSameBehavior( + helpers + + `var x_exports={};__export(x_exports,{v:()=>v});var v,import_path,init_x=__esm({"x.ts"(){"use strict";import_path=__toESM(require("path")),v=import_path.default.sep}});` + + `console.log((init_x(),__toCommonJS(x_exports)).v);`, + ); +}); + +test.each([ + [ + "is asynchronous", + `var init_a = __esm({ async "a.ts"() { await 0; } });\ninit_a();`, + ], + [ + "is used other than as a statement call", + `var init_a = __esm({ "a.ts"() {} });\nvar r = init_a();`, + ], + [ + "reads its factory's receiver", + `var init_a = __esm({ "a.ts"() { console.log(this); } });\ninit_a();`, + ], + [ + "stores the namespace of a bundled ES module ('a_exports') in a variable", + `var a_exports = {};\n__export(a_exports, { v: () => v });\nvar v, init_a = __esm({ "a.ts"() { v = 1; } });\nvar ns, load = () => ns ??= __toESM((init_a(), __toCommonJS(a_exports)));\nload();`, + ], + [ + "reads the namespace of a bundled ES module ('a_exports')", + `var a_exports = {};\n__export(a_exports, { v: () => v });\nvar v, init_a = __esm({ "a.ts"() { v = 1; } });\nconsole.log(Object.keys((init_a(), __toCommonJS(a_exports))));`, + ], +])("an initializer that %s keeps the package refused", (reason, code) => { + expect(lower(`${helpers}${code}\n`)).toEqual({ + degrade: expect.stringContaining(reason) as string, + }); +}); + +test("files without __esm initializers and ES-module bundles are left alone", () => { + expect(lower("exports.x = 1;\n")).toBeNull(); + expect(lower(`${helpers}var init_a = __esm({ "a.ts"() {} });\ninit_a();\nexport {};\n`)).toBeNull(); +}); diff --git a/packages/compiler/src/frontend/npm-static-bundled-esm.ts b/packages/compiler/src/frontend/npm-static-bundled-esm.ts new file mode 100644 index 000000000..ae5985e51 --- /dev/null +++ b/packages/compiler/src/frontend/npm-static-bundled-esm.ts @@ -0,0 +1,582 @@ +/* Lazily initialized ES modules in a CommonJS bundle. esbuild scope-hoists + * every ES module it bundles into the bundle's own scope; one that some + * code require()s instead of importing runs from a lazy initializer, + * + * var COUNT, import_fs; + * var init_x = __esm({ "src/x.ts"() { import_fs = __toESM(require("fs")); COUNT = 0; } }); + * + * and the requiring site reads its namespace as + * `(init_x(), __toCommonJS(x_exports))`. `init_x()` runs the body once (a + * call during the body answers immediately). The pass below keeps that + * shape in the bundle's scope and respells only what has no static + * representation: + * + * - the initializer becomes a run-once closure over the same body; + * - an interop binding the body assigns once from require() (`import_fs` + * above) becomes that require at the assignment's position, and its + * member reads become member reads of the required module, exactly + * like the top-level interop erasure (npm-static-rewrite-syntax.ts); + * - a namespace read only through member reads becomes an object + * literal of getters over the module's live bindings (what + * __toCommonJS copies, minus the non-enumerable __esModule mark no + * member read can reach). + * + * The respelled interop reads resolve the required module even if they + * ran before the initializer, where Node would read a member of + * undefined and throw; esbuild only reaches them through the module's + * own code, which runs after its initializer. Shapes outside the + * recognized forms answer a degrade reason. */ +import * as ts from "./ts7/syntax.js"; +import { bareRequireSpecOf } from "./cjs-syntax.js"; +import { helperTokens } from "./helper-tokens.js"; +import { + applyEdits, + countDeclarations, + declaredNames, + factoryFrameUse, + helperBase, + isWritten, + references, + unparen, + type Edit, +} from "./npm-static-bundled-chunks.js"; + +export interface EsmInitHost { + requireTargetEsModuleStamped(fromFile: string, specifier: string): boolean; +} + +const ESM_SHAPE_DEGRADE = + "its shipped JS initializes bundled ES modules through the __esm helper in a shape the " + + "recognizer cannot verify"; + +/* The recognized __esm forms over its parameter names (FN, RES) and the + * own-key reader (KEYS): keyed `(FN, RES) => function [name]() { return + * ; }` with a one-method object argument, and arrow + * `(FN, RES) => () => ()` with a function argument. */ +const KEYED_RETURN = "FN && (RES = (0, FN[KEYS(FN)[0]])(FN = 0)), RES"; +const ARROW_BODY = "(FN && (RES = FN(FN = 0)), RES)"; + +function template(text: string, fn: string, res: string, keys: string): string | null { + return helperTokens( + text + .replace(/\bFN\b/g, fn) + .replace(/\bRES\b/g, res) + .replace(/\bKEYS\b/g, keys), + ); +} + +function esmHelperForm( + init: ts.Expression, + sf: ts.SourceFile, + ownNames: (name: string) => boolean, +): "keyed" | "arrow" | null { + const fn = unparen(init); + if (!ts.isArrowFunction(fn) || fn.parameters.nodes.length !== 2 || ts.isBlock(fn.body)) + return null; + const [p0, p1] = fn.parameters.nodes; + if ( + !p0 || + !p1 || + !ts.isIdentifier(p0.name) || + !ts.isIdentifier(p1.name) || + p0.initializer || + p1.initializer || + p0.name.text === p1.name.text + ) + return null; + const [f, r] = [p0.name.text, p1.name.text]; + const body = unparen(fn.body); + if (ts.isFunctionExpression(body)) { + if (body.parameters.nodes.length !== 0 || body.asteriskToken || body.modifiers?.nodes.length) + return null; + const stmts = body.body.statements.nodes; + const ret = stmts[0]; + if (stmts.length !== 1 || !ret || !ts.isReturnStatement(ret) || !ret.expression) return null; + const text = ret.expression.getText(sf); + const keys = /\[\s*([A-Za-z_$][\w$]*(?:\.keys)?)\s*\(/.exec(text)?.[1]; + if (keys === undefined || (keys !== "Object.keys" && !ownNames(keys))) return null; + return helperTokens(text) === template(KEYED_RETURN, f, r, keys) ? "keyed" : null; + } + if (ts.isArrowFunction(body) && body.parameters.nodes.length === 0 && !ts.isBlock(body.body)) { + const actual = helperTokens(body.body.getText(sf)); + return actual !== null && actual === template(ARROW_BODY, f, r, "") ? "arrow" : null; + } + return null; +} + +/** Pads the given declarators out of their statements (separating commas + * included; a statement whose declarators all go pads whole). */ +function removeDeclarators(sf: ts.SourceFile, dead: ReadonlySet): Edit[] { + const edits: Edit[] = []; + const stmts = new Set(); + for (const d of dead) { + const stmt = d.parent?.parent; + if (stmt !== undefined && ts.isVariableStatement(stmt)) stmts.add(stmt); + } + for (const stmt of stmts) { + const decls = stmt.declarationList.declarations.nodes; + const deadAt = decls.map((d) => dead.has(d)); + if (deadAt.every((x) => x)) { + edits.push({ start: stmt.getStart(sf), end: stmt.getEnd(), text: "" }); + continue; + } + const lastLive = deadAt.lastIndexOf(false); + for (let i = 0; i < decls.length; i++) { + if (!deadAt[i]) continue; + edits.push( + i < lastLive + ? { start: decls[i]!.getStart(sf), end: decls[i + 1]!.getStart(sf), text: "" } + : { start: decls[lastLive]!.getEnd(), end: decls[i]!.getEnd(), text: "" }, + ); + } + } + return edits; +} + +/** The elements of an expression statement's comma chain, in order. */ +function commaElements(e: ts.Expression): ts.Expression[] { + const out: ts.Expression[] = []; + const visit = (x: ts.Expression): void => { + if (ts.isBinaryExpression(x) && x.operatorToken.kind === ts.SyntaxKind.CommaToken) { + visit(x.left); + visit(x.right); + } else out.push(x); + }; + visit(e); + return out; +} + +/** True when a call's result is discarded: an expression statement or a + * non-final comma operand (parentheses included). */ +function resultUnused(call: ts.Expression): boolean { + let child: ts.Node = call; + let p = call.parent; + while (p !== undefined && ts.isParenthesizedExpression(p)) { + child = p; + p = p.parent; + } + if (p === undefined) return false; + if (ts.isExpressionStatement(p)) return true; + if (ts.isBinaryExpression(p) && p.operatorToken.kind === ts.SyntaxKind.CommaToken) { + if (p.left === child) return true; + return resultUnused(p); + } + return false; +} + +/** True when a member access is a plain read (not written, deleted or + * updated). */ +function plainRead(access: ts.Expression): boolean { + const p = access.parent; + if (p === undefined) return true; + if (ts.isDeleteExpression(p)) return false; + if ( + ts.isBinaryExpression(p) && + p.left === access && + p.operatorToken.kind >= ts.SyntaxKind.FirstAssignment && + p.operatorToken.kind <= ts.SyntaxKind.LastAssignment + ) + return false; + if ( + (ts.isPrefixUnaryExpression(p) || ts.isPostfixUnaryExpression(p)) && + (p.operator === ts.SyntaxKind.PlusPlusToken || p.operator === ts.SyntaxKind.MinusMinusToken) + ) + return false; + return true; +} + +/** The `__export(T, { name: () => value, … })` getter table of T, or null. */ +function exportTableOf( + call: ts.CallExpression, +): { target: string; entries: [string, ts.Expression][] } | null { + if (!ts.isIdentifier(call.expression) || helperBase(call.expression.text) !== "__export") + return null; + const [target, table] = call.arguments.nodes; + if ( + call.arguments.nodes.length !== 2 || + !target || + !table || + !ts.isIdentifier(target) || + !ts.isObjectLiteralExpression(table) + ) + return null; + const entries: [string, ts.Expression][] = []; + for (const prop of table.properties.nodes) { + if (!ts.isPropertyAssignment(prop)) return null; + if (!ts.isIdentifier(prop.name) && !ts.isStringLiteral(prop.name)) return null; + const getter = unparen(prop.initializer); + if (!ts.isArrowFunction(getter) || getter.parameters.nodes.length !== 0) return null; + if (ts.isBlock(getter.body)) return null; + entries.push([prop.name.text, getter.body]); + } + return { target: target.text, entries }; +} + +/** Lowers a CommonJS bundle's __esm initializers (see the header). Null: + * nothing to lower. */ +export function lowerBundledEsmInits( + sf: ts.SourceFile, + filePath: string, + host: EsmInitHost, +): string | { degrade: string } | null { + const source = sf.text; + if (!source.includes("__esm")) return null; + if (!filePath.endsWith(".cjs") && ts.isExternalModule(sf)) return null; + + interface Top { + stmt: ts.Statement; + node: ts.VariableDeclaration | ts.FunctionDeclaration; + } + const top = new Map(); + const duplicate = new Set(); + for (const stmt of sf.statements.nodes) { + const add = (name: string, node: Top["node"]): void => { + if (top.has(name)) duplicate.add(name); + else top.set(name, { stmt, node }); + }; + if (ts.isFunctionDeclaration(stmt) && stmt.name !== undefined) add(stmt.name.text, stmt); + else if (ts.isVariableStatement(stmt)) { + for (const d of stmt.declarationList.declarations.nodes) { + if (ts.isIdentifier(d.name)) add(d.name.text, d); + } + } + } + const helpers = [...top.keys()].filter((n) => helperBase(n) === "__esm"); + if (helpers.length === 0) return null; + + const allRefs = references(sf); + const writes = new Map(); + for (const r of allRefs) if (isWritten(r)) writes.set(r.text, (writes.get(r.text) ?? 0) + 1); + const declaredAnywhere = declaredNames(sf); + if (declaredAnywhere.has("Object") || writes.has("Object")) return { degrade: ESM_SHAPE_DEGRADE }; + const stable = (name: string): boolean => !duplicate.has(name) && !writes.has(name); + const ownNames = (name: string): boolean => { + const t = top.get(name); + const init = t !== undefined && ts.isVariableDeclaration(t.node) ? t.node.initializer : undefined; + return ( + init !== undefined && + stable(name) && + helperTokens(init.getText(sf)) === helperTokens("Object.getOwnPropertyNames") + ); + }; + const forms = new Map(); + for (const name of helpers) { + const t = top.get(name)!; + const init = ts.isVariableDeclaration(t.node) ? t.node.initializer : undefined; + const form = init !== undefined && stable(name) ? esmHelperForm(init, sf, ownNames) : null; + if (form === null) return { degrade: ESM_SHAPE_DEGRADE }; + forms.set(name, form); + } + + // The initializers: `NAME = HELPER(factory)` declarators. + interface Init { + name: string; + call: ts.CallExpression; + body: ts.Block; + } + const inits = new Map(); + for (const [name, t] of top) { + if (!ts.isVariableDeclaration(t.node) || t.node.initializer === undefined) continue; + const call = unparen(t.node.initializer); + if (!ts.isCallExpression(call) || !ts.isIdentifier(call.expression)) continue; + const form = forms.get(call.expression.text); + if (form === undefined) continue; + if (call.arguments.nodes.length !== 1 || !stable(name)) return { degrade: ESM_SHAPE_DEGRADE }; + let factory: ts.Node = call.arguments.nodes[0]!; + if (form === "keyed") { + if (!ts.isObjectLiteralExpression(factory) || factory.properties.nodes.length !== 1) + return { degrade: ESM_SHAPE_DEGRADE }; + factory = factory.properties.nodes[0]!; + if (!ts.isMethodDeclaration(factory) || ts.isComputedPropertyName(factory.name)) + return { degrade: ESM_SHAPE_DEGRADE }; + } else { + factory = unparen(factory as ts.Expression); + if (!ts.isArrowFunction(factory) && !ts.isFunctionExpression(factory)) + return { degrade: ESM_SHAPE_DEGRADE }; + } + const fn = factory as ts.MethodDeclaration | ts.ArrowFunction | ts.FunctionExpression; + if (fn.modifiers?.nodes.some((m) => m.kind === ts.SyntaxKind.AsyncKeyword)) + return { + degrade: `its bundled ES module initializer '${name}' is asynchronous (top-level await), which the lazy initialization rewrite does not cover`, + }; + if ( + fn.body === undefined || + !ts.isBlock(fn.body) || + fn.asteriskToken !== undefined || + (fn.modifiers?.nodes.length ?? 0) > 0 || + fn.parameters.nodes.length > 0 + ) + return { degrade: ESM_SHAPE_DEGRADE }; + const frame = factoryFrameUse(fn.body, false); + if (frame !== null) + return { degrade: `its bundled ES module initializer '${name}' ${frame}` }; + inits.set(name, { name, call, body: fn.body }); + } + if (inits.size === 0) return null; + + const edits: Edit[] = []; + // Every initializer reference is a direct call whose result is unused. + for (const r of allRefs) { + if (!inits.has(r.text)) continue; + const call = r.parent; + if ( + call === undefined || + !ts.isCallExpression(call) || + call.expression !== r || + call.arguments.nodes.length !== 0 || + !resultUnused(call) + ) + return { + degrade: `its bundled ES module initializer '${r.text}' is used other than as a statement call`, + }; + } + for (const name of inits.keys()) { + if (countDeclarations(sf, name) !== 1) return { degrade: ESM_SHAPE_DEGRADE }; + } + + /* Interop bindings an initializer assigns once from require(). */ + const refsByName = new Map(); + for (const r of allRefs) { + const list = refsByName.get(r.text); + if (list === undefined) refsByName.set(r.text, [r]); + else list.push(r); + } + const deadDeclarators = new Set(); + for (const init of inits.values()) { + for (const stmt of init.body.statements.nodes) { + if (!ts.isExpressionStatement(stmt)) continue; + for (const el of commaElements(stmt.expression)) { + if ( + !ts.isBinaryExpression(el) || + el.operatorToken.kind !== ts.SyntaxKind.EqualsToken || + !ts.isIdentifier(el.left) + ) + continue; + const name = el.left.text; + const decl = top.get(name)?.node; + if ( + decl === undefined || + !ts.isVariableDeclaration(decl) || + decl.initializer !== undefined || + duplicate.has(name) || + writes.get(name) !== 1 || + countDeclarations(sf, name) !== 1 + ) + continue; + let rhs = unparen(el.right); + let isNodeMode = false; + let interop = false; + if ( + ts.isCallExpression(rhs) && + ts.isIdentifier(rhs.expression) && + helperBase(rhs.expression.text) === "__toESM" + ) { + const [arg, mode] = rhs.arguments.nodes; + if (!arg || rhs.arguments.nodes.length > 2) continue; + if (mode !== undefined) { + if (ts.isNumericLiteral(mode)) isNodeMode = Number(mode.text) !== 0; + else if (mode.kind === ts.SyntaxKind.TrueKeyword) isNodeMode = true; + else if (mode.kind !== ts.SyntaxKind.FalseKeyword) continue; + } + rhs = unparen(arg); + interop = true; + } + const spec = bareRequireSpecOf(rhs); + if (spec === null) continue; + const req = rhs.getText(sf); + const defaultIsModule = + interop && (isNodeMode || !host.requireTargetEsModuleStamped(filePath, spec)); + // Every other reference must be a plain member read. + const uses: Edit[] = []; + let ok = true; + for (const r of refsByName.get(name) ?? []) { + if (r === el.left) continue; + const access = r.parent; + if ( + access === undefined || + !ts.isPropertyAccessExpression(access) || + access.expression !== r || + !plainRead(access) + ) { + ok = false; + break; + } + if (defaultIsModule && access.name.text === "default") + uses.push({ start: access.getStart(sf), end: access.getEnd(), text: `(${req})` }); + else uses.push({ start: r.getStart(sf), end: r.getEnd(), text: req }); + } + if (!ok) continue; + edits.push({ start: el.getStart(sf), end: el.getEnd(), text: req }, ...uses); + deadDeclarators.add(decl); + } + } + } + edits.push(...removeDeclarators(sf, deadDeclarators)); + + /* Namespaces of lazily initialized modules read through members. */ + let mainTarget: string | null = null; + for (const stmt of sf.statements.nodes) { + if (!ts.isExpressionStatement(stmt) || !ts.isBinaryExpression(stmt.expression)) continue; + const e = stmt.expression; + const rhs = e.right; + if ( + e.operatorToken.kind === ts.SyntaxKind.EqualsToken && + ts.isPropertyAccessExpression(e.left) && + ts.isIdentifier(e.left.expression) && + e.left.expression.text === "module" && + e.left.name.text === "exports" && + ts.isCallExpression(rhs) && + ts.isIdentifier(rhs.expression) && + helperBase(rhs.expression.text) === "__toCommonJS" && + rhs.arguments.nodes.length === 1 && + ts.isIdentifier(rhs.arguments.nodes[0]!) + ) + mainTarget = (rhs.arguments.nodes[0] as ts.Identifier).text; + } + const tables = new Map(); + for (const stmt of sf.statements.nodes) { + if (!ts.isExpressionStatement(stmt) || !ts.isCallExpression(stmt.expression)) continue; + const table = exportTableOf(stmt.expression); + if (table !== null && table.target !== mainTarget) { + if (tables.has(table.target)) return { degrade: ESM_SHAPE_DEGRADE }; + tables.set(table.target, { stmt, entries: table.entries }); + } + } + const lowered = new Set(); + for (const r of allRefs) { + const table = tables.get(r.text); + if (table === undefined) continue; + const p = r.parent; + // The table's own declaration and __export call are handled below. + if (p !== undefined && ts.isVariableDeclaration(p) && p.name === r) continue; + if ( + p !== undefined && + ts.isCallExpression(p) && + p.arguments.nodes[0] === r && + ts.isExpressionStatement(p.parent) && + p.parent === table.stmt + ) + continue; + const why = `reads the namespace of a bundled ES module ('${r.text}') in a way the static rewrite cannot follow (only member reads of its exports compile)`; + if ( + p === undefined || + !ts.isCallExpression(p) || + !ts.isIdentifier(p.expression) || + helperBase(p.expression.text) !== "__toCommonJS" || + p.arguments.nodes.length !== 1 + ) + return { degrade: `its shipped JS ${why}` }; + // The namespace expression: `(init_x(), __toCommonJS(T))` or the call. + let ns: ts.Expression = p; + for (;;) { + const up = ns.parent; + if (up !== undefined && ts.isParenthesizedExpression(up)) ns = up; + else if ( + up !== undefined && + ts.isBinaryExpression(up) && + up.operatorToken.kind === ts.SyntaxKind.CommaToken && + up.right === ns + ) + ns = up; + else break; + } + const names = new Set(table.entries.map(([n]) => n)); + const memberRead = (access: ts.Node | undefined, of: ts.Node): boolean => + access !== undefined && + ts.isPropertyAccessExpression(access) && + access.expression === of && + names.has(access.name.text) && + plainRead(access); + const holder = ns.parent; + let readable = memberRead(holder, ns); + if ( + !readable && + holder !== undefined && + ts.isVariableDeclaration(holder) && + holder.initializer === ns && + ts.isIdentifier(holder.name) + ) { + const local = holder.name.text; + const localRefs = (refsByName.get(local) ?? []).filter((x) => x !== holder.name); + readable = + countDeclarations(sf, local) === 1 && + !writes.has(local) && + localRefs.every((x) => memberRead(x.parent, x)); + } + if (!readable) { + // A namespace kept for later (`x ??= __toESM(ns)`, a hand-rolled + // lazy loader) is a value with no static representation. + let outer: ts.Node | undefined = holder; + while ( + outer !== undefined && + (ts.isParenthesizedExpression(outer) || + (ts.isCallExpression(outer) && + ts.isIdentifier(outer.expression) && + helperBase(outer.expression.text) === "__toESM")) + ) + outer = outer.parent; + if ( + outer !== undefined && + ts.isBinaryExpression(outer) && + outer.operatorToken.kind >= ts.SyntaxKind.FirstAssignment && + outer.operatorToken.kind <= ts.SyntaxKind.LastAssignment + ) + return { + degrade: `its shipped JS stores the namespace of a bundled ES module ('${r.text}') in a variable (a memoized lazy loader), a module object the static rewrite cannot represent`, + }; + return { degrade: `its shipped JS ${why}` }; + } + // The getters read top-level bindings: nothing between the use site + // and the top level may rebind one of their names. + const read = new Set(); + for (const [, value] of table.entries) for (const x of references(value)) read.add(x.text); + for (let a = p.parent; a !== undefined && !ts.isSourceFile(a); a = a.parent) { + if (ts.isFunctionLike(a) || ts.isBlock(a) || ts.isClassDeclaration(a) || ts.isClassExpression(a)) { + for (const n of declaredNames(a)) { + if (read.has(n)) return { degrade: `its shipped JS ${why}` }; + } + } + } + const getters = table.entries + .map( + ([n, value]) => + `get ${/^[A-Za-z_$][\w$]*$/.test(n) ? n : JSON.stringify(n)}() { return ${applyEdits(source, value.getStart(sf), value.getEnd(), edits)}; }`, + ) + .join(", "); + edits.push({ start: p.getStart(sf), end: p.getEnd(), text: `{ ${getters} }` }); + lowered.add(r.text); + } + for (const target of lowered) { + const table = tables.get(target)!; + const decl = top.get(target)?.node; + if ( + decl === undefined || + !ts.isVariableDeclaration(decl) || + decl.initializer === undefined || + !ts.isObjectLiteralExpression(unparen(decl.initializer)) || + (unparen(decl.initializer) as ts.ObjectLiteralExpression).properties.nodes.length !== 0 || + writes.has(target) + ) + return { degrade: ESM_SHAPE_DEGRADE }; + edits.push({ start: table.stmt.getStart(sf), end: table.stmt.getEnd(), text: "" }); + edits.push(...removeDeclarators(sf, new Set([decl]))); + } + + /* The run-once closures. */ + for (const init of inits.values()) { + const { call, body } = init; + edits.push( + { + start: call.getStart(sf), + end: body.getStart(sf) + 1, + text: "(() => { let __scriptc_done = false; function __scriptc_init() {", + }, + { + start: body.getEnd() - 1, + end: call.getEnd(), + text: "} return function () { if (__scriptc_done) return; __scriptc_done = true; __scriptc_init(); }; })()", + }, + ); + } + return applyEdits(source, 0, source.length, edits); +} diff --git a/packages/compiler/src/frontend/npm-static-rewrite-host.ts b/packages/compiler/src/frontend/npm-static-rewrite-host.ts index 524239901..9ff14c455 100644 --- a/packages/compiler/src/frontend/npm-static-rewrite-host.ts +++ b/packages/compiler/src/frontend/npm-static-rewrite-host.ts @@ -11,6 +11,7 @@ import { import { resolveExports } from "./resolve.js"; import { packageNameOfSpecifier } from "./workspace-registry.js"; import { isBundlerCjsCandidate, rewriteBundlerCjsSyntax } from "./npm-static-rewrite-syntax.js"; +import { lowerBundledEsmInits } from "./npm-static-bundled-esm.js"; const NODE_REQUIRE_CONDITIONS = new Set(["require", "node", "default"]); @@ -164,6 +165,17 @@ function starTargetNames(parser: Ts7SourceParser, file: string): Set { } } +export function lowerEsmInitsWithParser( + parser: Ts7SourceParser, + source: string, + filePath: string, +): string | { degrade: string } | null { + if (!source.includes("__esm")) return null; + return lowerBundledEsmInits(parser.parse(filePath, source, "js"), filePath, { + requireTargetEsModuleStamped: (from, spec) => requireTargetEsModuleStamped(parser, from, spec), + }); +} + export function rewriteBundlerCjsWithParser( parser: Ts7SourceParser, source: string, diff --git a/packages/compiler/src/frontend/npm-static-rewrite-syntax.ts b/packages/compiler/src/frontend/npm-static-rewrite-syntax.ts index 1730e1153..08dbc0db5 100644 --- a/packages/compiler/src/frontend/npm-static-rewrite-syntax.ts +++ b/packages/compiler/src/frontend/npm-static-rewrite-syntax.ts @@ -893,6 +893,7 @@ export function rewriteBundlerCjsSyntax( "__toESM", "__toCommonJS", "__commonJS", + "__esm", "__exportStar", "__createBinding", "__setModuleDefault", diff --git a/packages/compiler/src/frontend/npm-static.ts b/packages/compiler/src/frontend/npm-static.ts index 7dffd05f7..7c9b7f6a1 100644 --- a/packages/compiler/src/frontend/npm-static.ts +++ b/packages/compiler/src/frontend/npm-static.ts @@ -407,6 +407,15 @@ export function npmStaticFsShadow(services: FrontendServices): NpmStaticFsShadow for (const chunk of split.chunks) setVirtualSource(chunk.path, chunk.source); original = split.text; } + // Then its lazily initialized ES modules become run-once closures. + const esm = original === null ? null : services.lowerEsmInits(original, path); + if (esm !== null && typeof esm === "object") { + reportFileDegrade(path, target.pkg, esm.degrade); + original = null; + } else if (esm !== null) { + original = esm; + } + const respelled = split !== null || esm !== null; if (original !== null) { const namepaths = services.jsDocNamepaths(path, original); const source = namepaths ?? original; @@ -441,7 +450,7 @@ export function npmStaticFsShadow(services: FrontendServices): NpmStaticFsShadow findWidened?.text ?? classFields?.text ?? namepaths ?? - (split !== null ? original : null); + (respelled ? original : null); } } } catch { @@ -492,6 +501,48 @@ function hasTransformMarkers(source: string): boolean { return TRANSFORM_MARKERS.some((m) => source.includes(m)); } +/** The installed version of the package owning `file` (the innermost + * node_modules segment), or null when no package.json names one. */ +function installedVersionOf(file: string): string | null { + const norm = file.split("\\").join("/"); + const idx = norm.lastIndexOf("/node_modules/"); + if (idx < 0) return null; + const pkg = npmPackageNameOf(norm); + if (pkg === null) return null; + const text = trackedReadFile(`${norm.slice(0, idx)}/node_modules/${pkg}/package.json`); + if (text === null) return null; + try { + const version = (JSON.parse(text) as { version?: unknown }).version; + return typeof version === "string" ? version : null; + } catch { + return null; + } +} + +/** Why a DefinitelyTyped twin cannot vouch for the runtime package `pkg`, + * or null when it can. DefinitelyTyped versions an @types package by the + * library version it describes (major.minor; 0.x minors break): the twin + * must describe the release that is installed, read from both package + * manifests next to the resolved declaration and JavaScript files. */ +export function definitelyTypedMismatch( + pkg: string, + typesFile: string, + jsFile: string, +): string | null { + const typesVersion = installedVersionOf(typesFile); + const version = installedVersionOf(jsFile); + const twin = npmPackageNameOf(typesFile.split("\\").join("/")) ?? "its @types package"; + if (typesVersion === null || version === null) + return `its declared types come from the separate ${twin} package, whose version cannot be matched against the installed ${pkg}`; + const line = (v: string): string => { + const [major = "", minor = ""] = v.split("."); + return major === "0" ? `0.${minor}` : major; + }; + return line(typesVersion) === line(version) + ? null + : `its declared types come from ${twin}@${typesVersion}, which describes a different release than the installed ${pkg}@${version}`; +} + /** Auto-mode eligibility for one candidate package: `typesFile` is what * the ordinary (types-first) resolution answered for the import, * `jsEntry` what the js-only resolution answers. Returns null when diff --git a/packages/compiler/src/frontend/pipeline.ts b/packages/compiler/src/frontend/pipeline.ts index 676b2bb3e..5c929960a 100644 --- a/packages/compiler/src/frontend/pipeline.ts +++ b/packages/compiler/src/frontend/pipeline.ts @@ -16,6 +16,7 @@ import { type LoadResult, } from "./program.js"; import { + definitelyTypedMismatch, npmStaticFileDegrades, npmStaticIneligibleReason, npmStaticOffenders, @@ -219,13 +220,25 @@ function detectAutoPackages( chosen.push(runtimePkg); continue; } + // Both answers compile the JavaScript with inferred types; the + // declarations only check the importer. A DefinitelyTyped twin + // qualifies when it describes the installed release. A package + // without declarations qualifies as a dependency of a compiled + // package, whose JavaScript importer has no declarations to check. + const refusal = + npm === null + ? owner === null + ? "it ships no own .d.ts declaration surface" + : null + : definitelyTypedMismatch(runtimePkg, npm.typesFile, js!.typesFile); + if (refusal === null) { + chosen.push(runtimePkg); + continue; + } statuses.push({ package: runtimePkg, status: "refused", - detail: - npm === null - ? "it ships no own .d.ts declaration surface" - : `its declared types come from the separate ${npm.packageName} package, not the package itself`, + detail: refusal, hint: npmStaticOptInHint(runtimePkg, owner), }); continue; diff --git a/packages/compiler/src/frontend/services.ts b/packages/compiler/src/frontend/services.ts index c6bb24ae5..aeed52161 100644 --- a/packages/compiler/src/frontend/services.ts +++ b/packages/compiler/src/frontend/services.ts @@ -1,6 +1,6 @@ import { Ts7SourceParser, type Ts7SourceKind } from "./ts7/source-parser.js"; import { Ts7Host, type Ts7ApiFactory, type Ts7HostOptions } from "./ts7/program-host.js"; -import { rewriteBundlerCjsWithParser } from "./npm-static-rewrite-host.js"; +import { lowerEsmInitsWithParser, rewriteBundlerCjsWithParser } from "./npm-static-rewrite-host.js"; import { isBundlerCjsCandidate } from "./npm-static-rewrite-syntax.js"; import { mayBundleChunks, @@ -76,6 +76,14 @@ export class FrontendServices { if (!mayBundleChunks(source)) return null; return splitBundledChunks(this.parse(path, source, "js"), path); } + /** A CommonJS bundle's lazily initialized ES modules (__esm) lowered to + * run-once closures (null: none; a degrade reason: shapes the rewrite + * cannot lower). */ + lowerEsmInits(source: string, path: string): string | { degrade: string } | null { + this.ensureOpen(); + if (!source.includes("__esm")) return null; + return lowerEsmInitsWithParser(this.sourceParser(), source, path); + } nullableClassFields(path: string, source: string): syntax.NpmStaticOverloadRewrite | null { return syntax.applyNpmStaticNullableClassFields(this.parse(path, source, "js"), source); } diff --git a/tests/fixtures/npm-static/node_modules/@types/twinstale/index.d.ts b/tests/fixtures/npm-static/node_modules/@types/twinstale/index.d.ts new file mode 100644 index 000000000..0017b82be --- /dev/null +++ b/tests/fixtures/npm-static/node_modules/@types/twinstale/index.d.ts @@ -0,0 +1 @@ +export default function stale(text: string): string; diff --git a/tests/fixtures/npm-static/node_modules/@types/twinstale/package.json b/tests/fixtures/npm-static/node_modules/@types/twinstale/package.json new file mode 100644 index 000000000..7f7850ce6 --- /dev/null +++ b/tests/fixtures/npm-static/node_modules/@types/twinstale/package.json @@ -0,0 +1,5 @@ +{ + "name": "@types/twinstale", + "version": "2.0.4", + "types": "index.d.ts" +} diff --git a/tests/fixtures/npm-static/node_modules/twinstale/index.js b/tests/fixtures/npm-static/node_modules/twinstale/index.js new file mode 100644 index 000000000..129abfcce --- /dev/null +++ b/tests/fixtures/npm-static/node_modules/twinstale/index.js @@ -0,0 +1,6 @@ +// The 3.x release changed the export: its @types twin still describes 2.x. +'use strict'; + +module.exports = function stale(text) { + return 'stale:' + text; +}; diff --git a/tests/fixtures/npm-static/node_modules/twinstale/package.json b/tests/fixtures/npm-static/node_modules/twinstale/package.json new file mode 100644 index 000000000..9c104d6bf --- /dev/null +++ b/tests/fixtures/npm-static/node_modules/twinstale/package.json @@ -0,0 +1,5 @@ +{ + "name": "twinstale", + "version": "3.1.0", + "main": "index.js" +} diff --git a/tests/fixtures/npm-static/twinstale-cli.ts b/tests/fixtures/npm-static/twinstale-cli.ts new file mode 100644 index 000000000..1938cbf8b --- /dev/null +++ b/tests/fixtures/npm-static/twinstale-cli.ts @@ -0,0 +1,5 @@ +// twinstale's @types twin describes an older major release than the +// installed package, so it cannot vouch for the JavaScript. +import stale from "twinstale"; + +console.log(stale("ready")); diff --git a/tests/fixtures/npm/cases/2563-bundled-esm-inits/main.ts b/tests/fixtures/npm/cases/2563-bundled-esm-inits/main.ts new file mode 100644 index 000000000..02ac60cba --- /dev/null +++ b/tests/fixtures/npm/cases/2563-bundled-esm-inits/main.ts @@ -0,0 +1,16 @@ +// esbuild's lazily initialized ES modules: each package is real esbuild +// output (0.25 plain, and whitespace/syntax minified) of a source whose +// load() require()s an ES module. That module and the module it imports +// run behind the __esm helper: once, at the first load(), assigning +// their interop bindings (a builtin and an external CommonJS package) +// and live exports that the namespace's getters read. +import { describe, load } from "gtesm"; +import { describe as dws, load as lws } from "gtesmws"; + +console.log("main start"); +console.log(describe()); +console.log(load()); +console.log(load()); +console.log(dws()); +console.log(lws()); +console.log(lws()); diff --git a/tests/fixtures/npm/node_modules/gtesm/dist/index.d.ts b/tests/fixtures/npm/node_modules/gtesm/dist/index.d.ts new file mode 100644 index 000000000..af4369397 --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtesm/dist/index.d.ts @@ -0,0 +1,2 @@ +export declare function describe(): string; +export declare function load(): string; diff --git a/tests/fixtures/npm/node_modules/gtesm/dist/index.js b/tests/fixtures/npm/node_modules/gtesm/dist/index.js new file mode 100644 index 000000000..241c5e048 --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtesm/dist/index.js @@ -0,0 +1,90 @@ +var __create = Object.create; +var __defProp = Object.defineProperty; +var __getOwnPropDesc = Object.getOwnPropertyDescriptor; +var __getOwnPropNames = Object.getOwnPropertyNames; +var __getProtoOf = Object.getPrototypeOf; +var __hasOwnProp = Object.prototype.hasOwnProperty; +var __esm = (fn, res) => function __init() { + return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res; +}; +var __export = (target, all) => { + for (var name in all) + __defProp(target, name, { get: all[name], enumerable: true }); +}; +var __copyProps = (to, from, except, desc) => { + if (from && typeof from === "object" || typeof from === "function") { + for (let key of __getOwnPropNames(from)) + if (!__hasOwnProp.call(to, key) && key !== except) + __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable }); + } + return to; +}; +var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps( + // If the importer is in node compatibility mode or this is not an ESM + // file that has been converted to a CommonJS file using a Babel- + // compatible transform (i.e. "__esModule" has not been set), then set + // "default" to the CommonJS "module.exports" for node compatibility. + isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target, + mod +)); +var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod); + +// src/helper.ts +function helper(x) { + COUNT++; + return "[" + x + "]"; +} +var COUNT; +var init_helper = __esm({ + "src/helper.ts"() { + console.log("helper init"); + COUNT = 0; + } +}); + +// src/lazy.ts +var lazy_exports = {}; +__export(lazy_exports, { + VALUE: () => VALUE, + value: () => value +}); +function value() { + return VALUE + ":" + import_gtcore.default.tag + ":" + COUNT; +} +var path, import_gtcore, VALUE; +var init_lazy = __esm({ + "src/lazy.ts"() { + path = __toESM(require("path")); + import_gtcore = __toESM(require("gtcore")); + init_helper(); + console.log("lazy init"); + VALUE = helper(path.basename("/a/b.txt")); + } +}); + +// src/index.ts +var index_exports = {}; +__export(index_exports, { + describe: () => describe, + load: () => load +}); +module.exports = __toCommonJS(index_exports); + +// src/greet.ts +function greet(name) { + return "hello " + name; +} + +// src/index.ts +function describe() { + return greet("main"); +} +function load() { + const lazy = (init_lazy(), __toCommonJS(lazy_exports)); + return lazy.value() + " " + lazy.VALUE; +} +// Annotate the CommonJS export names for ESM import in node: +0 && (module.exports = { + describe, + load +}); diff --git a/tests/fixtures/npm/node_modules/gtesm/package.json b/tests/fixtures/npm/node_modules/gtesm/package.json new file mode 100644 index 000000000..3913402f9 --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtesm/package.json @@ -0,0 +1,10 @@ +{ + "name": "gtesm", + "version": "1.0.0", + "description": "esbuild 0.25.10 output (--bundle --platform=node --format=cjs --external:gtcore) of a source that require()s an ES module: lazily initialized ES modules behind the __esm helper", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "dependencies": { + "gtcore": "1.0.0" + } +} diff --git a/tests/fixtures/npm/node_modules/gtesmws/dist/index.d.ts b/tests/fixtures/npm/node_modules/gtesmws/dist/index.d.ts new file mode 100644 index 000000000..af4369397 --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtesmws/dist/index.d.ts @@ -0,0 +1,2 @@ +export declare function describe(): string; +export declare function load(): string; diff --git a/tests/fixtures/npm/node_modules/gtesmws/dist/index.js b/tests/fixtures/npm/node_modules/gtesmws/dist/index.js new file mode 100644 index 000000000..cef009700 --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtesmws/dist/index.js @@ -0,0 +1 @@ +var __create=Object.create;var __defProp=Object.defineProperty;var __getOwnPropDesc=Object.getOwnPropertyDescriptor;var __getOwnPropNames=Object.getOwnPropertyNames;var __getProtoOf=Object.getPrototypeOf,__hasOwnProp=Object.prototype.hasOwnProperty;var __esm=(fn,res)=>function(){return fn&&(res=(0,fn[__getOwnPropNames(fn)[0]])(fn=0)),res};var __export=(target,all)=>{for(var name in all)__defProp(target,name,{get:all[name],enumerable:!0})},__copyProps=(to,from,except,desc)=>{if(from&&typeof from=="object"||typeof from=="function")for(let key of __getOwnPropNames(from))!__hasOwnProp.call(to,key)&&key!==except&&__defProp(to,key,{get:()=>from[key],enumerable:!(desc=__getOwnPropDesc(from,key))||desc.enumerable});return to};var __toESM=(mod,isNodeMode,target)=>(target=mod!=null?__create(__getProtoOf(mod)):{},__copyProps(isNodeMode||!mod||!mod.__esModule?__defProp(target,"default",{value:mod,enumerable:!0}):target,mod)),__toCommonJS=mod=>__copyProps(__defProp({},"__esModule",{value:!0}),mod);function helper(x){return COUNT++,"["+x+"]"}var COUNT,init_helper=__esm({"src/helper.ts"(){console.log("helper init");COUNT=0}});var lazy_exports={};__export(lazy_exports,{VALUE:()=>VALUE,value:()=>value});function value(){return VALUE+":"+import_gtcore.default.tag+":"+COUNT}var path,import_gtcore,VALUE,init_lazy=__esm({"src/lazy.ts"(){path=__toESM(require("path")),import_gtcore=__toESM(require("gtcore"));init_helper();console.log("lazy init");VALUE=helper(path.basename("/a/b.txt"))}});var index_exports={};__export(index_exports,{describe:()=>describe,load:()=>load});module.exports=__toCommonJS(index_exports);function greet(name){return"hello "+name}function describe(){return greet("main")}function load(){let lazy=(init_lazy(),__toCommonJS(lazy_exports));return lazy.value()+" "+lazy.VALUE}0&&(module.exports={describe,load}); diff --git a/tests/fixtures/npm/node_modules/gtesmws/package.json b/tests/fixtures/npm/node_modules/gtesmws/package.json new file mode 100644 index 000000000..3dfe90e90 --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtesmws/package.json @@ -0,0 +1,10 @@ +{ + "name": "gtesmws", + "version": "1.0.0", + "description": "the gtesm source bundled by esbuild 0.25.10 with --minify-whitespace --minify-syntax", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "dependencies": { + "gtcore": "1.0.0" + } +} diff --git a/tests/harness/npm-cases.ts b/tests/harness/npm-cases.ts index 4b3b9fdee..f8d346154 100644 --- a/tests/harness/npm-cases.ts +++ b/tests/harness/npm-cases.ts @@ -19,12 +19,12 @@ export function npmCases(fixturesRoot: string): NpmCase[] { return [ ...globSync(join(fixturesRoot, "npm/cases/*/main.ts")) .sort() - // 2465-2469 and 2556-2562 are the --npm-static bundler-emitted-CJS + // 2465-2469 and 2556-2563 are the --npm-static bundler-emitted-CJS // cases (npm-static.test.ts drives them with the opt-in): their // consumers exercise the static rewrite's surface (lexer-visible // names the shipped .d.ts never declares, the __toESM interop // family), so they stay out of the flagless island lane by design. - .filter((entry) => !/\/(246[5-9]|255[6-9]|256[0-2])-[^/]+\/main\.ts$/.test(entry)) + .filter((entry) => !/\/(246[5-9]|255[6-9]|256[0-3])-[^/]+\/main\.ts$/.test(entry)) .map((entry) => ({ name: entry.split("/").at(-2)!, entry, diff --git a/tests/harness/npm-static.test.ts b/tests/harness/npm-static.test.ts index a8df98c52..a1703adf7 100644 --- a/tests/harness/npm-static.test.ts +++ b/tests/harness/npm-static.test.ts @@ -76,7 +76,7 @@ async function buildStatic(entry: string, npmStatic: string[] | "auto"): Promise ...globSync( join(fixturesRoot, "npm/node_modules/cryptozoo/**/*.{js,mjs,cjs,json,d.ts}"), ).sort(), - // the bundler-emitted-CJS mini packages (cases 2465-2469, 2556-2562) + // the bundler-emitted-CJS mini packages (cases 2465-2469, 2556-2563) ...globSync(join(fixturesRoot, "npm/node_modules/gt*/**/*.{js,cjs,json}")).sort(), ]; for (const f of inputs) hash.update(f).update(readFileSync(f)); @@ -828,30 +828,44 @@ describe(`npm-static pilots${sanitize ? " (sanitized)" : ""}`, () => { // Types-first resolution answers @types/twinned, but the refusal names the // package that has the code and the flag that compiles it. - test("an @types-typed package is refused under its runtime name with the opt-in remedy", () => { - const { coverage } = analyze(join(pilotRoot, "twinned-cli.ts")); + // The default selection compiles a package typed by a DefinitelyTyped + // twin that describes its installed release, and the untyped + // dependencies of compiled packages. + test("an @types-typed package and its untyped dependency compile by default", async () => { + const entry = join(pilotRoot, "twinned-cli.ts"); + const { coverage } = analyze(entry); + expect(coverage.npmStatic).toEqual([ + { package: "twinned", status: "static" }, + { package: "ms", status: "static" }, + ]); + expect(coverage.preflightFailed).toBe(false); + const binary = await buildStatic(entry, "auto"); + const [nodeRes, nativeRes] = await Promise.all([ + runBinary("node", [entry]), + runBinary(binary, []), + ]); + expect(nativeRes.stdout.toString("utf8")).toBe(nodeRes.stdout.toString("utf8")); + expect(nativeRes.exitCode).toBe(nodeRes.exitCode); + }, 180_000); + + test("an @types twin describing another release is refused with the opt-in remedy", () => { + const { coverage } = analyze(join(pilotRoot, "twinstale-cli.ts")); expect(coverage.npmStatic).toEqual([ { - package: "twinned", + package: "twinstale", status: "refused", detail: - "its declared types come from the separate @types/twinned package, not the package itself", - hint: "to compile it anyway, name it with --npm-static=twinned (or --npm-static=twinned... to include its dependencies)", + "its declared types come from @types/twinstale@2.0.4, which describes a different release than the installed twinstale@3.1.0", + hint: "to compile it anyway, name it with --npm-static=twinstale (or --npm-static=twinstale... to include its dependencies)", }, ]); const diagnostic = coverage.diagnostics.find((d) => d.code === "SC2013"); - expect(diagnostic?.message).toContain("'twinned'"); - expect(diagnostic?.hint).toContain("--npm-static=twinned"); + expect(diagnostic?.message).toContain("'twinstale'"); }, 120_000); - test("a named package's untyped dependency points at the transitive opt-in", () => { + test("a named package's untyped dependency compiles with it", () => { const { coverage } = analyze(join(pilotRoot, "twinned-cli.ts"), { npmStatic: ["twinned"] }); - expect(coverage.npmStatic).toContainEqual({ - package: "ms", - status: "refused", - detail: "it ships no own .d.ts declaration surface", - hint: "to compile it anyway, name it with --npm-static=ms, or use --npm-static=twinned... to include all of twinned's dependencies", - }); + expect(coverage.npmStatic).toContainEqual({ package: "ms", status: "static" }); }, 120_000); test("--npm-static=pkg... compiles the package's dependency closure", async () => { @@ -1457,6 +1471,16 @@ describe(`npm-static pilots${sanitize ? " (sanitized)" : ""}`, () => { ]); }, 120_000); + // 2563: lazily initialized ES modules (__esm) in esbuild 0.25 plain and + // whitespace/syntax-minified bundles. + test("bundled ES module initializers 2563-bundled-esm-inits compile statically and byte-match Node", async () => { + await expectNodeParity(join(fixturesRoot, "npm/cases/2563-bundled-esm-inits/main.ts"), [ + "gtesm", + "gtesmws", + "gtcore", + ]); + }, 180_000); + // 2562: a file with unverifiable interop that no compiled code can load // (the program imports a leaf data module and only a type from the // main entry) leaves its package static. From 0d9a6da8c96556f042690d0d8a294bd19e964f27 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Mon, 19 Oct 2026 14:19:07 -0700 Subject: [PATCH 05/35] Select untyped dependencies and @types twins by identity An untyped package another package's JavaScript imports is selected whether resolution answers no declarations or its JavaScript. A DefinitelyTyped twin qualifies by identity (@types/): the declarations only check the program, and the JavaScript compiles with its inferred surface, so a twin that disagrees with the shipped JavaScript fails the build at the use rather than compiling against it. Co-Authored-By: Claude Opus 5.5 --- packages/compiler/src/frontend/npm-static.ts | 65 ++++++------------- packages/compiler/src/frontend/pipeline.ts | 14 ++-- .../node_modules/@types/twinstale/index.d.ts | 2 +- .../node_modules/twinstale/index.js | 5 +- tests/fixtures/npm-static/twinstale-cli.ts | 7 +- tests/harness/npm-static.test.ts | 21 +++--- 6 files changed, 46 insertions(+), 68 deletions(-) diff --git a/packages/compiler/src/frontend/npm-static.ts b/packages/compiler/src/frontend/npm-static.ts index 7c9b7f6a1..4c1bc9ae6 100644 --- a/packages/compiler/src/frontend/npm-static.ts +++ b/packages/compiler/src/frontend/npm-static.ts @@ -501,46 +501,18 @@ function hasTransformMarkers(source: string): boolean { return TRANSFORM_MARKERS.some((m) => source.includes(m)); } -/** The installed version of the package owning `file` (the innermost - * node_modules segment), or null when no package.json names one. */ -function installedVersionOf(file: string): string | null { - const norm = file.split("\\").join("/"); - const idx = norm.lastIndexOf("/node_modules/"); - if (idx < 0) return null; - const pkg = npmPackageNameOf(norm); - if (pkg === null) return null; - const text = trackedReadFile(`${norm.slice(0, idx)}/node_modules/${pkg}/package.json`); - if (text === null) return null; - try { - const version = (JSON.parse(text) as { version?: unknown }).version; - return typeof version === "string" ? version : null; - } catch { - return null; - } -} - -/** Why a DefinitelyTyped twin cannot vouch for the runtime package `pkg`, - * or null when it can. DefinitelyTyped versions an @types package by the - * library version it describes (major.minor; 0.x minors break): the twin - * must describe the release that is installed, read from both package - * manifests next to the resolved declaration and JavaScript files. */ -export function definitelyTypedMismatch( - pkg: string, - typesFile: string, - jsFile: string, -): string | null { - const typesVersion = installedVersionOf(typesFile); - const version = installedVersionOf(jsFile); - const twin = npmPackageNameOf(typesFile.split("\\").join("/")) ?? "its @types package"; - if (typesVersion === null || version === null) - return `its declared types come from the separate ${twin} package, whose version cannot be matched against the installed ${pkg}`; - const line = (v: string): string => { - const [major = "", minor = ""] = v.split("."); - return major === "0" ? `0.${minor}` : major; - }; - return line(typesVersion) === line(version) +/** Why the declarations at `typesFile` cannot stand for the runtime + * package `pkg`, or null when they are its DefinitelyTyped twin + * (`@types/`). The declarations only check the importing + * program; the JavaScript compiles with its inferred types, and an + * inferred surface the program does not typecheck against refuses the + * package afterwards, whatever release the twin describes. */ +export function definitelyTypedMismatch(pkg: string, typesFile: string): string | null { + const owner = npmPackageNameOf(typesFile.split("\\").join("/")); + const twin = `@types/${mangledTypesName(pkg)}`; + return owner === twin ? null - : `its declared types come from ${twin}@${typesVersion}, which describes a different release than the installed ${pkg}@${version}`; + : `its declared types come from ${owner ?? typesFile}, which is not ${pkg}'s DefinitelyTyped package ${twin}`; } /** Auto-mode eligibility for one candidate package: `typesFile` is what @@ -551,12 +523,17 @@ export function npmStaticIneligibleReason( pkgName: string, typesFile: string, jsEntry: string | null, + /** A dependency another package's JavaScript imports: no declarations + * check that importer, so the package need not ship its own. */ + untypedDependency = false, ): string | null { - if (npmPackageNameOf(typesFile) !== pkgName) { - return "its declared types come from a third-party @types package, not the package itself"; - } - if (!/\.d\.(ts|mts|cts)$/.test(typesFile)) { - return "it ships no own .d.ts declaration surface"; + if (!untypedDependency) { + if (npmPackageNameOf(typesFile) !== pkgName) { + return "its declared types come from a third-party @types package, not the package itself"; + } + if (!/\.d\.(ts|mts|cts)$/.test(typesFile)) { + return "it ships no own .d.ts declaration surface"; + } } if (jsEntry === null) return "no runtime JS entry resolves"; const source = trackedReadFile(jsEntry); diff --git a/packages/compiler/src/frontend/pipeline.ts b/packages/compiler/src/frontend/pipeline.ts index 5c929960a..5081fc7be 100644 --- a/packages/compiler/src/frontend/pipeline.ts +++ b/packages/compiler/src/frontend/pipeline.ts @@ -221,16 +221,17 @@ function detectAutoPackages( continue; } // Both answers compile the JavaScript with inferred types; the - // declarations only check the importer. A DefinitelyTyped twin - // qualifies when it describes the installed release. A package - // without declarations qualifies as a dependency of a compiled - // package, whose JavaScript importer has no declarations to check. + // declarations only check the importer, and an inferred surface + // the program does not typecheck against still refuses the package + // below. A DefinitelyTyped twin qualifies; a package without + // declarations qualifies as a dependency of another package, whose + // JavaScript importer has no declarations to check. const refusal = npm === null - ? owner === null + ? owner === null && !sf.fileName.includes("/node_modules/") ? "it ships no own .d.ts declaration surface" : null - : definitelyTypedMismatch(runtimePkg, npm.typesFile, js!.typesFile); + : definitelyTypedMismatch(runtimePkg, npm.typesFile); if (refusal === null) { chosen.push(runtimePkg); continue; @@ -269,6 +270,7 @@ function detectAutoPackages( pkg, typesFile, jsEntry !== null && isJsSourceFileName(jsEntry.typesFile) ? jsEntry.typesFile : null, + mode === "lib" && isJsSourceFileName(typesFile) && fromFile.includes("/node_modules/"), ); if (reason === null) chosen.push(pkg); else diff --git a/tests/fixtures/npm-static/node_modules/@types/twinstale/index.d.ts b/tests/fixtures/npm-static/node_modules/@types/twinstale/index.d.ts index 0017b82be..47f003c12 100644 --- a/tests/fixtures/npm-static/node_modules/@types/twinstale/index.d.ts +++ b/tests/fixtures/npm-static/node_modules/@types/twinstale/index.d.ts @@ -1 +1 @@ -export default function stale(text: string): string; +export function stale(text: string): string; diff --git a/tests/fixtures/npm-static/node_modules/twinstale/index.js b/tests/fixtures/npm-static/node_modules/twinstale/index.js index 129abfcce..6efa90723 100644 --- a/tests/fixtures/npm-static/node_modules/twinstale/index.js +++ b/tests/fixtures/npm-static/node_modules/twinstale/index.js @@ -1,6 +1,7 @@ -// The 3.x release changed the export: its @types twin still describes 2.x. +// The 3.x release renamed its export: its @types twin still describes the +// 2.x name. 'use strict'; -module.exports = function stale(text) { +exports.render = function render(text) { return 'stale:' + text; }; diff --git a/tests/fixtures/npm-static/twinstale-cli.ts b/tests/fixtures/npm-static/twinstale-cli.ts index 1938cbf8b..f6c8a679c 100644 --- a/tests/fixtures/npm-static/twinstale-cli.ts +++ b/tests/fixtures/npm-static/twinstale-cli.ts @@ -1,5 +1,6 @@ -// twinstale's @types twin describes an older major release than the -// installed package, so it cannot vouch for the JavaScript. -import stale from "twinstale"; +// twinstale's @types twin describes an older release than the installed +// package: the program typechecks against the twin, but not against the +// JavaScript's inferred surface (under Node the import is undefined). +import { stale } from "twinstale"; console.log(stale("ready")); diff --git a/tests/harness/npm-static.test.ts b/tests/harness/npm-static.test.ts index a1703adf7..8560973cd 100644 --- a/tests/harness/npm-static.test.ts +++ b/tests/harness/npm-static.test.ts @@ -848,19 +848,16 @@ describe(`npm-static pilots${sanitize ? " (sanitized)" : ""}`, () => { expect(nativeRes.exitCode).toBe(nodeRes.exitCode); }, 180_000); - test("an @types twin describing another release is refused with the opt-in remedy", () => { + // The twin only checks the program; the JavaScript compiles with its + // inferred surface, so a twin naming an export the JavaScript lacks + // (Node's link error) fails the build at the import's use instead of + // compiling against the declaration. + test("an @types twin that disagrees with the shipped JavaScript does not build", () => { const { coverage } = analyze(join(pilotRoot, "twinstale-cli.ts")); - expect(coverage.npmStatic).toEqual([ - { - package: "twinstale", - status: "refused", - detail: - "its declared types come from @types/twinstale@2.0.4, which describes a different release than the installed twinstale@3.1.0", - hint: "to compile it anyway, name it with --npm-static=twinstale (or --npm-static=twinstale... to include its dependencies)", - }, - ]); - const diagnostic = coverage.diagnostics.find((d) => d.code === "SC2013"); - expect(diagnostic?.message).toContain("'twinstale'"); + expect(coverage.npmStatic).toEqual([{ package: "twinstale", status: "static" }]); + expect(coverage.diagnostics).toContainEqual( + expect.objectContaining({ message: expect.stringContaining("'stale'") as string }), + ); }, 120_000); test("a named package's untyped dependency compiles with it", () => { From 4e0f71a7e5dae2f2b266049d6ce8d8473cbd5760 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Mon, 19 Oct 2026 14:21:57 -0700 Subject: [PATCH 06/35] Document the new package selection; record the 2563 order baseline Co-Authored-By: Claude Opus 5.5 --- docs/content/docs/dependencies.mdx | 14 ++++++++------ .../src/frontend/npm-static-bundled-esm.test.ts | 9 ++++----- .../src/frontend/npm-static-bundled-esm.ts | 13 +++++++++---- .../npm/cases/2563-bundled-esm-inits/main.ts.json | 6 ++++++ 4 files changed, 27 insertions(+), 15 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2563-bundled-esm-inits/main.ts.json diff --git a/docs/content/docs/dependencies.mdx b/docs/content/docs/dependencies.mdx index 3c6958841..e6020fe54 100644 --- a/docs/content/docs/dependencies.mdx +++ b/docs/content/docs/dependencies.mdx @@ -52,10 +52,12 @@ Type-only imports follow Node.js's type stripping. `import type` and `export typ scriptc compiles a package automatically when it: -- ships its own type declarations, and -- ships readable, unminified JavaScript. +- ships its own type declarations, or is typed by its DefinitelyTyped `@types/` package, or is a dependency of another package that has no declarations at all, and +- ships readable JavaScript: unminified, or minified by a bundler whose helpers keep their names. -A package that doesn't meet these requirements, or that uses something scriptc can't compile, gets a package-specific diagnostic that explains why. The usual fixes are to switch to a version or alternative package that compiles, or to replace the call with code you write yourself. A package whose declarations come from a separate `@types/` package doesn't ship its own; its diagnostic names the package and the `--npm-static` option that compiles it anyway. +The declarations check your program; the package's JavaScript compiles with the types scriptc infers from it. A program that doesn't typecheck against the inferred types, for example because an `@types/` package describes another release, fails the build at that use. + +A package that doesn't meet these requirements, or that uses something scriptc can't compile, gets a package-specific diagnostic that explains why. The usual fixes are to switch to a version or alternative package that compiles, or to replace the call with code you write yourself. @@ -63,17 +65,17 @@ A package that doesn't meet these requirements, or that uses something scriptc c ## Explicit package selection -Some packages rely on separately installed declarations, such as an `@types/` package, or on types inferred from their JavaScript. To try compiling one of them, name it with `--npm-static`: +Some packages your program imports directly ship no declarations, so their types can only be inferred from their JavaScript. To try compiling one of them, name it with `--npm-static`: ```console $ scriptc build app.ts --npm-static=three -o app >/dev/null ``` -Naming a package skips the declaration requirement above, but the package still has to compile. The option selects only the packages it names: a dependency of a named package that doesn't ship its own declarations is still refused. Add a trailing `...` to select a package together with every package it depends on, for example `--npm-static=fs-extra...`. `--npm-static=auto` uses the default automatic selection. The option accepts a comma-separated list and can be repeated. +Naming a package skips the declaration requirement above, but the package still has to compile. Add a trailing `...` to select a package together with every package it depends on, for example `--npm-static=fs-extra...`. `--npm-static=auto` uses the default automatic selection. The option accepts a comma-separated list and can be repeated. ## CommonJS packages -CommonJS packages compile with Node.js's module semantics. Each `require()` evaluates its module at the call, once, and later calls return the cached exports. A package can assign `module.exports` and run code before it requires its dependencies. If that code reads a `const` or `let` require binding before its `require` runs, the read throws the same `ReferenceError` as Node.js. A `var` binding that earlier code can reach is reported at build time. A `require()` of a JSON file, such as a package's own `package.json`, returns one object parsed at build time, shared by every module that requires it. A require cycle compiles when each of its modules assigns its exports before requiring anything, as semver's classes do, and uses the other modules only inside functions that its top-level code can't call. Other require cycles are reported at build time. The [limitations](/docs/limitations) page lists the supported `require` forms. +CommonJS packages compile with Node.js's module semantics. Each `require()` evaluates its module at the call, once, and later calls return the cached exports. A package can assign `module.exports` and run code before it requires its dependencies. If that code reads a `const` or `let` require binding before its `require` runs, the read throws the same `ReferenceError` as Node.js. A `var` binding that earlier code can reach is reported at build time. A `require()` of a JSON file, such as a package's own `package.json`, returns one object parsed at build time, shared by every module that requires it. A require cycle compiles when each of its modules assigns its exports before requiring anything, as semver's classes and esbuild's per-file output do, and uses the other modules only inside functions that its top-level code can't call. Bundles built by esbuild compile with their bundled CommonJS modules and lazily initialized ES modules, each evaluated once, at its first use, as under Node.js. Other require cycles are reported at build time. The [limitations](/docs/limitations) page lists the supported `require` forms. ## Unsupported code inside a package diff --git a/packages/compiler/src/frontend/npm-static-bundled-esm.test.ts b/packages/compiler/src/frontend/npm-static-bundled-esm.test.ts index 9c064cb55..0b9ecd042 100644 --- a/packages/compiler/src/frontend/npm-static-bundled-esm.test.ts +++ b/packages/compiler/src/frontend/npm-static-bundled-esm.test.ts @@ -117,10 +117,7 @@ test("minified initializers sharing a var statement lower the same way", () => { }); test.each([ - [ - "is asynchronous", - `var init_a = __esm({ async "a.ts"() { await 0; } });\ninit_a();`, - ], + ["is asynchronous", `var init_a = __esm({ async "a.ts"() { await 0; } });\ninit_a();`], [ "is used other than as a statement call", `var init_a = __esm({ "a.ts"() {} });\nvar r = init_a();`, @@ -145,5 +142,7 @@ test.each([ test("files without __esm initializers and ES-module bundles are left alone", () => { expect(lower("exports.x = 1;\n")).toBeNull(); - expect(lower(`${helpers}var init_a = __esm({ "a.ts"() {} });\ninit_a();\nexport {};\n`)).toBeNull(); + expect( + lower(`${helpers}var init_a = __esm({ "a.ts"() {} });\ninit_a();\nexport {};\n`), + ).toBeNull(); }); diff --git a/packages/compiler/src/frontend/npm-static-bundled-esm.ts b/packages/compiler/src/frontend/npm-static-bundled-esm.ts index ae5985e51..cf4f6280b 100644 --- a/packages/compiler/src/frontend/npm-static-bundled-esm.ts +++ b/packages/compiler/src/frontend/npm-static-bundled-esm.ts @@ -252,7 +252,8 @@ export function lowerBundledEsmInits( const stable = (name: string): boolean => !duplicate.has(name) && !writes.has(name); const ownNames = (name: string): boolean => { const t = top.get(name); - const init = t !== undefined && ts.isVariableDeclaration(t.node) ? t.node.initializer : undefined; + const init = + t !== undefined && ts.isVariableDeclaration(t.node) ? t.node.initializer : undefined; return ( init !== undefined && stable(name) && @@ -308,8 +309,7 @@ export function lowerBundledEsmInits( ) return { degrade: ESM_SHAPE_DEGRADE }; const frame = factoryFrameUse(fn.body, false); - if (frame !== null) - return { degrade: `its bundled ES module initializer '${name}' ${frame}` }; + if (frame !== null) return { degrade: `its bundled ES module initializer '${name}' ${frame}` }; inits.set(name, { name, call, body: fn.body }); } if (inits.size === 0) return null; @@ -531,7 +531,12 @@ export function lowerBundledEsmInits( const read = new Set(); for (const [, value] of table.entries) for (const x of references(value)) read.add(x.text); for (let a = p.parent; a !== undefined && !ts.isSourceFile(a); a = a.parent) { - if (ts.isFunctionLike(a) || ts.isBlock(a) || ts.isClassDeclaration(a) || ts.isClassExpression(a)) { + if ( + ts.isFunctionLike(a) || + ts.isBlock(a) || + ts.isClassDeclaration(a) || + ts.isClassExpression(a) + ) { for (const n of declaredNames(a)) { if (read.has(n)) return { degrade: `its shipped JS ${why}` }; } diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2563-bundled-esm-inits/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2563-bundled-esm-inits/main.ts.json new file mode 100644 index 000000000..953a75d7d --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2563-bundled-esm-inits/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/2563-bundled-esm-inits/main.ts" + ], + "diags": [] +} From 769e9e58cf768ced8e26510d7e42f35deee02355 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Mon, 19 Oct 2026 14:29:23 -0700 Subject: [PATCH 07/35] Run the two npm cases the default selection now compiles cjs-lexer-invisible and hoisted-workspace compiled only once their untyped dependencies were selected; both now match Node in both lanes. Co-Authored-By: Claude Opus 5.5 --- tests/harness/npm.test.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/harness/npm.test.ts b/tests/harness/npm.test.ts index b6af0dbb6..1f231b3bc 100644 --- a/tests/harness/npm.test.ts +++ b/tests/harness/npm.test.ts @@ -40,7 +40,6 @@ const KNOWN: Record = { "bytes-exit": { build: "SC2004" }, "cjs-esm-named": { build: "SC2013" }, "cjs-lexer-aligned": { build: "SC2013" }, - "cjs-lexer-invisible": { build: "SC2013" }, "commander-calc": { run: "SC1090", argvs: [["--help"]] }, "commander-calc-async": { run: "SC1090", argvs: [["--help"]] }, "create-require": { run: "SC1090" }, @@ -53,7 +52,6 @@ const KNOWN: Record = { "esm-missing-export": { build: "SC1090" }, "fetch-destructure": { run: "SC1031" }, "fs-shims": { run: "SC2020" }, - "hoisted-workspace": { build: "SC2013" }, "import-refusal": { build: "SC2020" }, "island-registry": { build: "SC2011" }, "island-timers": { From f7f239ad531fbc32b84b99187d421b8152bca71b Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Mon, 19 Oct 2026 15:55:37 -0700 Subject: [PATCH 08/35] Lower memoized lazy loaders of bundled ES-module namespaces A loader `L = () => M ??= NS` over a lazily initialized module's namespace (optionally through __toESM) is respelled at every `L().name` as the module's initializer call followed by the export's live read; loaders whose result escapes as a value stay refused. The canonical export table keeps a getter it cannot chase as a live accessor, so a lazy re-export still initializes its module at the first read. An ES-module importer copies those accessors once when it links the CommonJS module, in the lexer's order, exactly like Node's loader; require() keeps reading them live. Co-Authored-By: Claude Opus 5.5 --- .../src/frontend/lowering/lower-exprs.ts | 102 ++++++++++- .../src/frontend/lowering/lower-modules.ts | 10 +- .../compiler/src/frontend/lowering/lowerer.ts | 4 + .../src/frontend/npm-static-bundled-esm.ts | 173 ++++++++++++++++-- .../src/frontend/npm-static-rewrite-host.ts | 2 + .../src/frontend/npm-static-rewrite-syntax.ts | 28 ++- packages/compiler/src/frontend/npm-static.ts | 10 + packages/compiler/src/frontend/program.ts | 9 +- packages/compiler/src/frontend/services.ts | 9 +- .../npm/cases/2564-lazy-barrel/main.ts | 19 ++ .../cases/2565-lazy-barrel-require/main.ts | 9 + .../npm/node_modules/gtlazy/dist/index.d.ts | 7 + .../npm/node_modules/gtlazy/dist/index.js | 116 ++++++++++++ .../npm/node_modules/gtlazy/package.json | 7 + .../npm/node_modules/gtlazyuser/index.d.ts | 2 + .../npm/node_modules/gtlazyuser/index.js | 10 + .../npm/node_modules/gtlazyuser/package.json | 10 + .../npm/node_modules/gtlazyws/dist/index.d.ts | 7 + .../npm/node_modules/gtlazyws/dist/index.js | 3 + .../npm/node_modules/gtlazyws/package.json | 7 + tests/harness/npm-cases.ts | 4 +- tests/harness/npm-static.test.ts | 13 +- 22 files changed, 537 insertions(+), 24 deletions(-) create mode 100644 tests/fixtures/npm/cases/2564-lazy-barrel/main.ts create mode 100644 tests/fixtures/npm/cases/2565-lazy-barrel-require/main.ts create mode 100644 tests/fixtures/npm/node_modules/gtlazy/dist/index.d.ts create mode 100644 tests/fixtures/npm/node_modules/gtlazy/dist/index.js create mode 100644 tests/fixtures/npm/node_modules/gtlazy/package.json create mode 100644 tests/fixtures/npm/node_modules/gtlazyuser/index.d.ts create mode 100644 tests/fixtures/npm/node_modules/gtlazyuser/index.js create mode 100644 tests/fixtures/npm/node_modules/gtlazyuser/package.json create mode 100644 tests/fixtures/npm/node_modules/gtlazyws/dist/index.d.ts create mode 100644 tests/fixtures/npm/node_modules/gtlazyws/dist/index.js create mode 100644 tests/fixtures/npm/node_modules/gtlazyws/package.json diff --git a/packages/compiler/src/frontend/lowering/lower-exprs.ts b/packages/compiler/src/frontend/lowering/lower-exprs.ts index 09407f71f..cabbc31f5 100644 --- a/packages/compiler/src/frontend/lowering/lower-exprs.ts +++ b/packages/compiler/src/frontend/lowering/lower-exprs.ts @@ -114,6 +114,7 @@ import { cjsClassExprWholeExportOf, cjsExportAssignmentOf, cjsExportDiscardReason, + cjsExportTargetLiteral, isCjsExportTableLiteral, isCjsJsFile, isJsSourceFile, @@ -164,7 +165,7 @@ import { import { strCharsCall } from "./containers/array-construction.js"; import { lowerOptionalStringIndex, lowerUnionStringIndex } from "./string-index.js"; import { tryLowerIndexedComparison } from "./indexed-comparison.js"; -import { npmStaticPackageOfPath } from "../npm-static.js"; +import { npmStaticPackageOfPath, npmStaticLexerVisibleNames } from "../npm-static.js"; import { unsupportedModuleFeatureOf } from "../builtin-modules.js"; import { fenceEnumObjectValue, lowerEnumAccess } from "./lower-enums.js"; import { @@ -12541,9 +12542,108 @@ function cjsExportAccessorRead(lowerer: Lowerer, ident: ts.Identifier): IrExpr | if (!ts.isObjectLiteralExpression(getter.parent) || !isCjsExportTableLiteral(getter.parent)) { return null; } + // An ES-module import binding reads the value Node copied when the + // importer linked the CommonJS module (cjsAccessorSnapshotStores). + const raw = lowerer.checker.getSymbolAtLocation(ident); + const binding = raw ? lowerer.checker.declarationsOf(raw)[0] : undefined; + if ( + binding !== undefined && + (ts.isImportSpecifier(binding) || ts.isExportSpecifier(binding)) && + !isCjsJsFile(binding.getSourceFile(), lowerer.program) + ) { + const snapshot = cjsAccessorSnapshotOf(lowerer, getter); + if (snapshot !== null) + return { kind: "varRef", localId: snapshot.id, type: snapshot.type, loc: locOf(ident) }; + } return cjsAccessorCall(lowerer, getter, locOf(ident)); } +/** The names Node's ES-module loader copies off a CommonJS module that a + * bundler rewrite spells as accessors (npmStaticLexerVisibleNames), with + * their getters in that order; empty for every other module. */ +function lexerVisibleAccessors(dep: ts.SourceFile): ts.GetAccessorDeclaration[] { + const names = npmStaticLexerVisibleNames(dep.fileName); + if (names.length === 0) return []; + let table: ts.ObjectLiteralExpression | null = null; + for (const stmt of dep.statements.nodes) { + const cjs = cjsExportAssignmentOf(stmt); + if (cjs?.kind === "table" && cjsExportDiscardReason(stmt) === null) { + table = cjsExportTargetLiteral(cjs.expr.right, dep); + } + } + if (table === null) return []; + const getters = new Map(); + for (const p of table.properties.nodes) { + if (ts.isGetAccessorDeclaration(p) && (ts.isIdentifier(p.name) || ts.isStringLiteral(p.name))) + getters.set(p.name.text, p); + } + return names.flatMap((n) => { + const g = getters.get(n); + return g === undefined ? [] : [g]; + }); +} + +/** The module global holding the value an accessor export had when an + * ES-module importer first linked its CommonJS module. */ +function cjsAccessorSnapshotOf( + lowerer: Lowerer, + getter: ts.GetAccessorDeclaration, +): { id: string; type: IrType } | null { + const hit = lowerer.cjsAccessorSnapshots.get(getter); + if (hit !== undefined) return hit; + if (!lexerVisibleAccessors(getter.getSourceFile()).includes(getter)) return null; + const call = cjsAccessorCall(lowerer, getter, locOf(getter)); + if (call === null) return null; + const snapshot = { id: `%g.cjsSnapshot.${lowerer.cjsAccessorSnapshots.size}`, type: call.type }; + lowerer.globalsList.push({ id: snapshot.id, name: "%cjsSnapshot", type: call.type, mutable: true }); + lowerer.cjsAccessorSnapshots.set(getter, snapshot); + return snapshot; +} + +/** Node's ES-module loader reads every lexer-visible export of a CommonJS + * module once, when an importer first links it, and the import bindings + * keep those values. For the accessors a bundler rewrite introduced, the + * importer's link point runs each getter once, in the lexer's order, and + * stores the values its bindings read. Empty when the module has none. */ +export function cjsAccessorSnapshotStores( + lowerer: Lowerer, + dep: ts.SourceFile, + loc: SrcLoc, +): IrStmt[] { + const getters = lexerVisibleAccessors(dep); + if (getters.length === 0) return []; + let readyId = lowerer.cjsSnapshotReady.get(dep); + if (readyId === undefined) { + readyId = `%g.cjsSnapshotReady.${lowerer.cjsSnapshotReady.size}`; + lowerer.globalsList.push({ id: readyId, name: "%cjsSnapshotReady", type: BOOL, mutable: true }); + lowerer.cjsSnapshotReady.set(dep, readyId); + } + const stores: IrStmt[] = [ + { kind: "assign", localId: readyId, value: { kind: "boolLit", value: true, type: BOOL, loc }, loc }, + ]; + for (const getter of getters) { + const snapshot = cjsAccessorSnapshotOf(lowerer, getter); + const call = cjsAccessorCall(lowerer, getter, loc); + if (snapshot === null || call === null) return []; + stores.push({ kind: "assign", localId: snapshot.id, value: call, loc }); + } + return [ + { + kind: "if", + cond: { + kind: "unary", + op: "!", + operand: { kind: "varRef", localId: readyId, type: BOOL, loc }, + type: BOOL, + loc, + }, + then: stores, + else_: null, + loc, + }, + ]; +} + /** The interned lifted-getter call both accessor-read paths share. */ function cjsAccessorCall( lowerer: Lowerer, diff --git a/packages/compiler/src/frontend/lowering/lower-modules.ts b/packages/compiler/src/frontend/lowering/lower-modules.ts index 96e22f208..86fab82d4 100644 --- a/packages/compiler/src/frontend/lowering/lower-modules.ts +++ b/packages/compiler/src/frontend/lowering/lower-modules.ts @@ -143,7 +143,7 @@ import { forkTargetPaths } from "../fork-target.js"; import { workerTargetPaths } from "../worker-target.js"; import { isNativeProxyInitializer } from "./expressions/native-proxy.js"; import { handleEventReaderPrelude } from "./lower-event-target.js"; -import { jsHoldsDynObject } from "./lower-exprs.js"; +import { cjsAccessorSnapshotStores, jsHoldsDynObject } from "./lower-exprs.js"; /** One file's declarations, split for collection and init-body lowering. */ export interface FileParts { @@ -2823,6 +2823,14 @@ export function lowerFileInit( }); } else { header.push({ kind: "exprStmt", expr: call, loc }); + // An ES-module importer links a CommonJS dependency: Node + // copies its lexer-visible exports here. + if ( + (ts.isImportDeclaration(stmt) || ts.isExportDeclaration(stmt)) && + !isCjsJsFile(sf, lowerer.program) && + isCjsJsFile(dep, lowerer.program) + ) + header.push(...cjsAccessorSnapshotStores(lowerer, dep, loc)); } } } diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index d252af250..ce2390d95 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -2507,6 +2507,10 @@ export class Lowerer { * lifted lazily as module-level functions and interned per accessor * declaration: member reads call the getter (lower-exprs). */ readonly cjsAccessorFns = new Map(); + /** Accessor exports' link-time values for ES-module importers, and the + * per-module flag that the copy already ran (cjsAccessorSnapshotStores). */ + readonly cjsAccessorSnapshots = new Map(); + readonly cjsSnapshotReady = new Map(); /** Interned `%iter.drain.` helpers (classIteratorDrainCall): one per * receiver class — the eager drain of a class iterable's protocol into * a fresh element array, behind array/call spreads. */ diff --git a/packages/compiler/src/frontend/npm-static-bundled-esm.ts b/packages/compiler/src/frontend/npm-static-bundled-esm.ts index cf4f6280b..cf3664264 100644 --- a/packages/compiler/src/frontend/npm-static-bundled-esm.ts +++ b/packages/compiler/src/frontend/npm-static-bundled-esm.ts @@ -443,9 +443,154 @@ export function lowerBundledEsmInits( } } const lowered = new Set(); + /** Some binding between the use site and the top level rebinds a name + * the table's getters read. */ + const shadowed = (site: ts.Node, entries: readonly [string, ts.Expression][]): boolean => { + const read = new Set(); + for (const [, value] of entries) for (const x of references(value)) read.add(x.text); + for (let a = site.parent; a !== undefined && !ts.isSourceFile(a); a = a.parent) { + if ( + ts.isFunctionLike(a) || + ts.isBlock(a) || + ts.isClassDeclaration(a) || + ts.isClassExpression(a) + ) { + for (const n of declaredNames(a)) if (read.has(n)) return true; + } + } + return false; + }; + const getterText = (value: ts.Expression): string => + applyEdits(source, value.getStart(sf), value.getEnd(), edits); + + /* Memoized lazy loaders of those namespaces, `L = () => M ??= NS` (a + * bundle plugin's lazy re-exports): every `L().name` becomes the + * initializer call followed by the export's live read, so the first + * access still initializes the module, once, and later accesses read + * the binding. */ + const loaderSpans: { start: number; end: number }[] = []; + for (const [name, t] of top) { + const d = t.node; + if (!ts.isVariableDeclaration(d) || d.initializer === undefined) continue; + const fn = unparen(d.initializer); + if (!ts.isArrowFunction(fn) || fn.parameters.nodes.length !== 0 || ts.isBlock(fn.body)) + continue; + const memoize = unparen(fn.body); + if ( + !ts.isBinaryExpression(memoize) || + (memoize.operatorToken.kind !== ts.SyntaxKind.QuestionQuestionEqualsToken && + memoize.operatorToken.kind !== ts.SyntaxKind.BarBarEqualsToken) || + !ts.isIdentifier(memoize.left) + ) + continue; + let w = unparen(memoize.right); + let interop = false; + let nodeMode = false; + if ( + ts.isCallExpression(w) && + ts.isIdentifier(w.expression) && + helperBase(w.expression.text) === "__toESM" + ) { + const [arg, mode] = w.arguments.nodes; + if (!arg || w.arguments.nodes.length > 2) continue; + if (mode !== undefined) { + if (ts.isNumericLiteral(mode)) nodeMode = Number(mode.text) !== 0; + else if (mode.kind === ts.SyntaxKind.TrueKeyword) nodeMode = true; + else if (mode.kind !== ts.SyntaxKind.FalseKeyword) continue; + } + interop = true; + w = unparen(arg); + } + let initCall: ts.CallExpression | null = null; + if (ts.isBinaryExpression(w) && w.operatorToken.kind === ts.SyntaxKind.CommaToken) { + const left = unparen(w.left); + if ( + !ts.isCallExpression(left) || + !ts.isIdentifier(left.expression) || + !inits.has(left.expression.text) || + left.arguments.nodes.length !== 0 + ) + continue; + initCall = left; + w = unparen(w.right); + } + if ( + !ts.isCallExpression(w) || + !ts.isIdentifier(w.expression) || + helperBase(w.expression.text) !== "__toCommonJS" || + w.arguments.nodes.length !== 1 || + !ts.isIdentifier(w.arguments.nodes[0]!) + ) + continue; + const target = (w.arguments.nodes[0] as ts.Identifier).text; + const table = tables.get(target); + if (table === undefined) continue; + const loaderWhy = { + degrade: `its memoized lazy loader '${name}' returns the namespace of a bundled ES module as a value, a module object the static rewrite cannot represent (only member reads of its result compile)`, + }; + const memo = memoize.left.text; + const memoDecl = top.get(memo)?.node; + if ( + memoDecl === undefined || + !ts.isVariableDeclaration(memoDecl) || + memoDecl.initializer !== undefined || + writes.get(memo) !== 1 || + countDeclarations(sf, memo) !== 1 || + (refsByName.get(memo) ?? []).some((x) => x !== memoize.left) || + writes.has(name) || + countDeclarations(sf, name) !== 1 + ) + return loaderWhy; + const entries = new Map(table.entries); + for (const ref of refsByName.get(name) ?? []) { + const call = ref.parent; + if ( + call === undefined || + !ts.isCallExpression(call) || + call.expression !== ref || + call.arguments.nodes.length !== 0 + ) + return loaderWhy; + let access: ts.Node | undefined = call.parent; + if ( + access === undefined || + !ts.isPropertyAccessExpression(access) || + access.expression !== call || + !plainRead(access) + ) + return loaderWhy; + // Node mode binds `default` to the namespace object itself. + if (interop && nodeMode && access.name.text === "default") { + const outer: ts.Node | undefined = access.parent; + if ( + outer === undefined || + !ts.isPropertyAccessExpression(outer) || + outer.expression !== access || + !plainRead(outer) + ) + return loaderWhy; + access = outer; + } + const member = (access as ts.PropertyAccessExpression).name.text; + const value = entries.get(member); + if (value === undefined || shadowed(access, table.entries)) return loaderWhy; + const init = initCall === null ? "" : `${initCall.getText(sf)}, `; + edits.push({ + start: access.getStart(sf), + end: access.getEnd(), + text: `(${init}${getterText(value)})`, + }); + } + edits.push(...removeDeclarators(sf, new Set([d, memoDecl]))); + loaderSpans.push({ start: d.getStart(sf), end: d.getEnd() }); + lowered.add(target); + } + for (const r of allRefs) { const table = tables.get(r.text); if (table === undefined) continue; + if (loaderSpans.some((span) => r.getStart(sf) >= span.start && r.getEnd() <= span.end)) + continue; const p = r.parent; // The table's own declaration and __export call are handled below. if (p !== undefined && ts.isVariableDeclaration(p) && p.name === r) continue; @@ -528,29 +673,27 @@ export function lowerBundledEsmInits( } // The getters read top-level bindings: nothing between the use site // and the top level may rebind one of their names. - const read = new Set(); - for (const [, value] of table.entries) for (const x of references(value)) read.add(x.text); - for (let a = p.parent; a !== undefined && !ts.isSourceFile(a); a = a.parent) { - if ( - ts.isFunctionLike(a) || - ts.isBlock(a) || - ts.isClassDeclaration(a) || - ts.isClassExpression(a) - ) { - for (const n of declaredNames(a)) { - if (read.has(n)) return { degrade: `its shipped JS ${why}` }; - } - } - } + if (shadowed(p, table.entries)) return { degrade: `its shipped JS ${why}` }; const getters = table.entries .map( ([n, value]) => - `get ${/^[A-Za-z_$][\w$]*$/.test(n) ? n : JSON.stringify(n)}() { return ${applyEdits(source, value.getStart(sf), value.getEnd(), edits)}; }`, + `get ${/^[A-Za-z_$][\w$]*$/.test(n) ? n : JSON.stringify(n)}() { return ${getterText(value)}; }`, ) .join(", "); edits.push({ start: p.getStart(sf), end: p.getEnd(), text: `{ ${getters} }` }); lowered.add(r.text); } + // A table nothing reads (its module is never required) goes too: its + // names are no exports of the bundle. + for (const [target, table] of tables) { + if (lowered.has(target)) continue; + const live = (refsByName.get(target) ?? []).some((x) => { + const p = x.parent; + if (p !== undefined && ts.isVariableDeclaration(p) && p.name === x) return false; + return !(x.getStart(sf) >= table.stmt.getStart(sf) && x.getEnd() <= table.stmt.getEnd()); + }); + if (!live) lowered.add(target); + } for (const target of lowered) { const table = tables.get(target)!; const decl = top.get(target)?.node; diff --git a/packages/compiler/src/frontend/npm-static-rewrite-host.ts b/packages/compiler/src/frontend/npm-static-rewrite-host.ts index 9ff14c455..4e0c92e31 100644 --- a/packages/compiler/src/frontend/npm-static-rewrite-host.ts +++ b/packages/compiler/src/frontend/npm-static-rewrite-host.ts @@ -181,10 +181,12 @@ export function rewriteBundlerCjsWithParser( source: string, filePath: string, liveExportTable?: (filePath: string) => void, + lexerVisibleNames?: (filePath: string, names: readonly string[]) => void, ): string | { degrade: string } | null { if (!isBundlerCjsCandidate(source)) return null; return rewriteBundlerCjsSyntax(parser.parse(filePath, source, "js"), filePath, { liveExportTable, + lexerVisibleNames, requireOnly: isVirtualSource, requireTargetEsModuleStamped: (from, spec) => requireTargetEsModuleStamped(parser, from, spec), starTargetNames: (from, spec) => { diff --git a/packages/compiler/src/frontend/npm-static-rewrite-syntax.ts b/packages/compiler/src/frontend/npm-static-rewrite-syntax.ts index 08dbc0db5..3b553c3f2 100644 --- a/packages/compiler/src/frontend/npm-static-rewrite-syntax.ts +++ b/packages/compiler/src/frontend/npm-static-rewrite-syntax.ts @@ -22,6 +22,9 @@ export interface CjsRewriteHost { * exists, so Node's lexer-visible name set bounds nothing and every * name the export plumbing defines is part of its surface. */ requireOnly?: ((filePath: string) => boolean) | undefined; + /** Told the names Node's lexer detects in the original file that the + * rewritten text spells only as accessors (which the lexer skips). */ + lexerVisibleNames?: ((filePath: string, names: readonly string[]) => void) | undefined; } /** True when esbuild's export object (`module.exports = __toCommonJS(T)` @@ -119,8 +122,12 @@ interface EntrySpec { name: string; /** "ident": VALUE is a plain identifier spelled directly in the table. * "hoist": VALUE is an expression text hoisted to a tail const first. + * "getter": VALUE is a getter body's returned expression the table + * keeps as a live accessor (read on every access, like the original + * getter, so a lazily initialized module still initializes at the + * first read). * "any": the checked-dynamic fallback (`__scriptc_any`). */ - kind: "ident" | "hoist" | "any"; + kind: "ident" | "hoist" | "getter" | "any"; value?: string; } @@ -760,7 +767,14 @@ export function rewriteBundlerCjsSyntax( ...exportTables.filter((t) => t.target !== esbuildMainTarget), ]; for (const t of tablesOrdered) { - for (const [name, value] of t.entries) claim(name, classify(name, value)); + for (const [name, value] of t.entries) { + const spec = classify(name, value); + // The table feeding module.exports keeps a getter it cannot chase + // as a live accessor rather than a snapshot. + if (spec.kind === "any" && value !== null && t.target === esbuildMainTarget) + claim(name, { name, kind: "getter", value: value.getText() }); + else claim(name, spec); + } } for (const d of definePropEntries) claim(d.name, classify(d.name, d.value)); @@ -1039,6 +1053,8 @@ export function rewriteBundlerCjsSyntax( const v = `__scriptc_e${hoistN++}`; hoists.push(`const ${v} = ${dropInteropDefault(spec.value!)};`); plainParts.push(`${keyOf(spec.name)}: ${v},`); + } else if (spec.kind === "getter") { + plainParts.push(`get ${keyOf(spec.name)}() { return ${dropInteropDefault(spec.value!)}; },`); } else { plainParts.push(`${keyOf(spec.name)}: __scriptc_any,`); } @@ -1064,5 +1080,13 @@ export function rewriteBundlerCjsSyntax( lines.push( `module.exports = {${spreadParts.join("")}${plainParts.join("")}${esModulePart}${memberParts.join("")}};`, ); + /* Node's lexer sees the original's names for these entries (esbuild's + * dead annotation), but no accessor of the canonical table: the host + * keeps those names lexer-visible, so ES-module importers link exactly + * as they do against the original. */ + const accessorNames = [...entries.values()] + .filter((spec) => spec.kind === "getter" && lexed.exports.has(spec.name)) + .map((spec) => spec.name); + if (accessorNames.length > 0) host.lexerVisibleNames?.(filePath, accessorNames); return text + lines.join("\n") + "\n"; } diff --git a/packages/compiler/src/frontend/npm-static.ts b/packages/compiler/src/frontend/npm-static.ts index 4c1bc9ae6..77eab1d3a 100644 --- a/packages/compiler/src/frontend/npm-static.ts +++ b/packages/compiler/src/frontend/npm-static.ts @@ -43,6 +43,14 @@ export function npmStaticLiveExportTable(path: string): boolean { return liveExportFiles.has(path.split("\\").join("/")); } +/** Names Node's lexer detects in a rewritten file's original text that + * its rewrite spells only as accessors (the lexer skips those). */ +const lexerNames = new Map(); + +export function npmStaticLexerVisibleNames(path: string): readonly string[] { + return lexerNames.get(path.split("\\").join("/")) ?? []; +} + export function setNpmStaticPackages>(packages: T): void { activePackages = new Set(packages); declarationOverloads = new Map(); @@ -51,6 +59,7 @@ export function setNpmStaticPackages>(packages: T): v fileDegrades.clear(); rewriteCache.clear(); liveExportFiles.clear(); + lexerNames.clear(); clearVirtualSources(); untypedPkgCache.clear(); realpathProbed.clear(); @@ -439,6 +448,7 @@ export function npmStaticFsShadow(services: FrontendServices): NpmStaticFsShadow source, path, (live) => liveExportFiles.add(live.split("\\").join("/")), + (file, names) => lexerNames.set(file.split("\\").join("/"), names), ); if (answer !== null && typeof answer === "object") { reportFileDegrade(path, target.pkg, answer.degrade); diff --git a/packages/compiler/src/frontend/program.ts b/packages/compiler/src/frontend/program.ts index 908a587ea..5c824c8ee 100644 --- a/packages/compiler/src/frontend/program.ts +++ b/packages/compiler/src/frontend/program.ts @@ -68,6 +68,7 @@ import { isNpmStaticPackage, npmStaticActive, npmStaticFsShadow, + npmStaticLexerVisibleNames, npmStaticLiveExportTable, npmStaticPackageOfPath, reportNpmStaticOffender, @@ -5440,12 +5441,18 @@ function cjsNamedImportLinkCheck( npmStaticDepSf7(program, from, spec, "require"); return dep !== null && isCjsJsFile7(dep, program) ? dep : null; }; + // A bundler rewrite spells some lexer-visible names as accessors. + const lexWithAccessorNames = (sf: ts.SourceFile): ReturnType => { + const lexed = cjsLexedExportsOfFile(sf); + for (const name of npmStaticLexerVisibleNames(sf.fileName)) lexed.exports.add(name); + return lexed; + }; const visible = (dep: ts.SourceFile, name: string): boolean => // `default` is the module.exports binding itself — always provided. // The program already owns native source ASTs; reuse them without // starting another syntax session or reparsing their source text. name === "default" || - cjsVisibleNames(dep, cjsLexedExportsOfFile, resolveCjsDep, lexMemo).has(name); + cjsVisibleNames(dep, lexWithAccessorNames, resolveCjsDep, lexMemo).has(name); /** The statement's resolved LOCAL CommonJS dependency, when it is an * import/re-export from one. */ diff --git a/packages/compiler/src/frontend/services.ts b/packages/compiler/src/frontend/services.ts index aeed52161..c5dcbf513 100644 --- a/packages/compiler/src/frontend/services.ts +++ b/packages/compiler/src/frontend/services.ts @@ -64,10 +64,17 @@ export class FrontendServices { source: string, path: string, liveExportTable?: (path: string) => void, + lexerVisibleNames?: (path: string, names: readonly string[]) => void, ): string | { degrade: string } | null { this.ensureOpen(); if (!isBundlerCjsCandidate(source)) return null; - return rewriteBundlerCjsWithParser(this.sourceParser(), source, path, liveExportTable); + return rewriteBundlerCjsWithParser( + this.sourceParser(), + source, + path, + liveExportTable, + lexerVisibleNames, + ); } /** A CommonJS bundle's __commonJS chunks as their own modules (null: no * chunks; a degrade reason: chunks the recognizer cannot move). */ diff --git a/tests/fixtures/npm/cases/2564-lazy-barrel/main.ts b/tests/fixtures/npm/cases/2564-lazy-barrel/main.ts new file mode 100644 index 000000000..31c88db9b --- /dev/null +++ b/tests/fixtures/npm/cases/2564-lazy-barrel/main.ts @@ -0,0 +1,19 @@ +// A barrel whose re-exports load their modules lazily: a build plugin +// (real esbuild output, plain and whitespace/syntax minified) turns each +// re-export into a getter over a memoized loader of a lazily initialized +// ES module. Node's ES-module import of a CommonJS module reads every +// lexer-visible export when it links, so each module initializes once, +// before main runs, and the imports are snapshots (greet reassigns +// GREETING afterwards). +import { greet, GREETING, Counter, describe } from "gtlazy"; +import { describe as wsDescribe, GREETING as wsGreeting } from "gtlazyws"; + +console.log("main start"); +console.log(GREETING); +console.log(greet("a")); +console.log(GREETING); +const c = new Counter(); +console.log(c.next(), c.next()); +console.log(describe()); +console.log(wsDescribe()); +console.log(wsGreeting); diff --git a/tests/fixtures/npm/cases/2565-lazy-barrel-require/main.ts b/tests/fixtures/npm/cases/2565-lazy-barrel-require/main.ts new file mode 100644 index 000000000..42bed4781 --- /dev/null +++ b/tests/fixtures/npm/cases/2565-lazy-barrel-require/main.ts @@ -0,0 +1,9 @@ +// The lazy barrel (gtlazy) required from CommonJS: require() hands out the +// export object itself, so each re-export's getter runs at its read. A +// module initializes at the first read of one of its exports, once, and +// later reads see its live bindings (greet reassigns GREETING). +import { first, again } from "gtlazyuser"; + +console.log("main start"); +console.log(first()); +console.log(again()); diff --git a/tests/fixtures/npm/node_modules/gtlazy/dist/index.d.ts b/tests/fixtures/npm/node_modules/gtlazy/dist/index.d.ts new file mode 100644 index 000000000..d3b4bac73 --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtlazy/dist/index.d.ts @@ -0,0 +1,7 @@ +export declare let GREETING: string; +export declare function greet(name: string): string; +export declare class Counter { + n: number; + next(): number; +} +export declare function describe(): string; diff --git a/tests/fixtures/npm/node_modules/gtlazy/dist/index.js b/tests/fixtures/npm/node_modules/gtlazy/dist/index.js new file mode 100644 index 000000000..9a94faa11 --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtlazy/dist/index.js @@ -0,0 +1,116 @@ +var __defProp = Object.defineProperty; +var __getOwnPropDesc = Object.getOwnPropertyDescriptor; +var __getOwnPropNames = Object.getOwnPropertyNames; +var __hasOwnProp = Object.prototype.hasOwnProperty; +var __esm = (fn, res) => function __init() { + return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res; +}; +var __export = (target, all) => { + for (var name in all) + __defProp(target, name, { get: all[name], enumerable: true }); +}; +var __copyProps = (to, from, except, desc) => { + if (from && typeof from === "object" || typeof from === "function") { + for (let key of __getOwnPropNames(from)) + if (!__hasOwnProp.call(to, key) && key !== except) + __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable }); + } + return to; +}; +var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod); + +// src/greet.ts +var greet_exports = {}; +__export(greet_exports, { + GREETING: () => GREETING, + greet: () => greet +}); +function greet(name) { + GREETING = "hi"; + return GREETING + " " + name; +} +var GREETING; +var init_greet = __esm({ + "src/greet.ts"() { + console.log("greet init"); + GREETING = "hello"; + } +}); + +// src/counter.ts +var counter_exports = {}; +__export(counter_exports, { + default: () => Counter +}); +var Counter; +var init_counter = __esm({ + "src/counter.ts"() { + console.log("counter init"); + Counter = class { + n = 0; + next() { + return ++this.n; + } + }; + } +}); + +// src/describe.ts +var describe_exports = {}; +__export(describe_exports, { + describe: () => describe +}); +function describe() { + return "described: " + greet("d"); +} +var init_describe = __esm({ + "src/describe.ts"() { + init_greet(); + console.log("describe init"); + } +}); + +// src/index.ts +var __create = Object.create; +var __defProp2 = Object.defineProperty; +var __getOwnPropDesc2 = Object.getOwnPropertyDescriptor; +var __getOwnPropNames2 = Object.getOwnPropertyNames; +var __getProtoOf = Object.getPrototypeOf; +var __hasOwnProp2 = Object.prototype.hasOwnProperty; +var __export2 = (target, all) => { + for (var name in all) + __defProp2(target, name, { get: all[name], enumerable: true }); +}; +var __copyProps2 = (to, from, except, desc) => { + if (from && typeof from === "object" || typeof from === "function") { + for (let key of __getOwnPropNames2(from)) + if (!__hasOwnProp2.call(to, key) && key !== except) + __defProp2(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc2(from, key)) || desc.enumerable }); + } + return to; +}; +var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps2( + // If the importer is in node compatibility mode or this is not an ESM + // file that has been converted to a CommonJS file using a Babel- + // compatible transform (i.e. "__esModule" has not been set), then set + // "default" to the CommonJS "module.exports" for node compatibility. + isNodeMode || !mod || !mod.__esModule ? __defProp2(target, "default", { value: mod, enumerable: true }) : target, + mod +)); +var __toCommonJS2 = (mod) => __copyProps2(__defProp2({}, "__esModule", { value: true }), mod); +var stdin_exports = {}; +__export2(stdin_exports, { + Counter: () => load_import_counter().default, + GREETING: () => load_import_greet().GREETING, + describe: () => load_import_describe().describe, + greet: () => load_import_greet().greet +}); +module.exports = __toCommonJS2(stdin_exports); +var import_greet2; +var load_import_greet = () => import_greet2 ??= (init_greet(), __toCommonJS(greet_exports)); +var import_counter; +var load_import_counter = () => import_counter ??= __toESM((init_counter(), __toCommonJS(counter_exports))); +var import_describe; +var load_import_describe = () => import_describe ??= (init_describe(), __toCommonJS(describe_exports)); + +0 && (module.exports = {"Counter": null,"GREETING": null,"describe": null,"greet": null}); diff --git a/tests/fixtures/npm/node_modules/gtlazy/package.json b/tests/fixtures/npm/node_modules/gtlazy/package.json new file mode 100644 index 000000000..e63f696a6 --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtlazy/package.json @@ -0,0 +1,7 @@ +{ + "name": "gtlazy", + "version": "1.0.0", + "description": "esbuild 0.25.10 output (--bundle --platform=node --format=cjs) of a barrel whose re-exports a build plugin turns into memoized lazy loaders: `load_x = () => x ??= (init_x(), __toCommonJS(x_exports))`", + "main": "./dist/index.js", + "types": "./dist/index.d.ts" +} diff --git a/tests/fixtures/npm/node_modules/gtlazyuser/index.d.ts b/tests/fixtures/npm/node_modules/gtlazyuser/index.d.ts new file mode 100644 index 000000000..948fec847 --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtlazyuser/index.d.ts @@ -0,0 +1,2 @@ +export declare function first(): string; +export declare function again(): string; diff --git a/tests/fixtures/npm/node_modules/gtlazyuser/index.js b/tests/fixtures/npm/node_modules/gtlazyuser/index.js new file mode 100644 index 000000000..d5666d6e1 --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtlazyuser/index.js @@ -0,0 +1,10 @@ +"use strict"; +const lazy = require("gtlazy"); + +console.log("user loaded"); +exports.first = function first() { + return lazy.GREETING + "/" + lazy.greet("u"); +}; +exports.again = function again() { + return lazy.GREETING + "/" + lazy.describe(); +}; diff --git a/tests/fixtures/npm/node_modules/gtlazyuser/package.json b/tests/fixtures/npm/node_modules/gtlazyuser/package.json new file mode 100644 index 000000000..841bdc7b2 --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtlazyuser/package.json @@ -0,0 +1,10 @@ +{ + "name": "gtlazyuser", + "version": "1.0.0", + "description": "a CommonJS package that require()s the gtlazy barrel: its lazy re-exports initialize at the first read and read live bindings", + "main": "index.js", + "types": "index.d.ts", + "dependencies": { + "gtlazy": "1.0.0" + } +} diff --git a/tests/fixtures/npm/node_modules/gtlazyws/dist/index.d.ts b/tests/fixtures/npm/node_modules/gtlazyws/dist/index.d.ts new file mode 100644 index 000000000..d3b4bac73 --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtlazyws/dist/index.d.ts @@ -0,0 +1,7 @@ +export declare let GREETING: string; +export declare function greet(name: string): string; +export declare class Counter { + n: number; + next(): number; +} +export declare function describe(): string; diff --git a/tests/fixtures/npm/node_modules/gtlazyws/dist/index.js b/tests/fixtures/npm/node_modules/gtlazyws/dist/index.js new file mode 100644 index 000000000..a472466b0 --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtlazyws/dist/index.js @@ -0,0 +1,3 @@ +var __defProp=Object.defineProperty;var __getOwnPropDesc=Object.getOwnPropertyDescriptor;var __getOwnPropNames=Object.getOwnPropertyNames;var __hasOwnProp=Object.prototype.hasOwnProperty;var __esm=(fn,res)=>function(){return fn&&(res=(0,fn[__getOwnPropNames(fn)[0]])(fn=0)),res};var __export=(target,all)=>{for(var name in all)__defProp(target,name,{get:all[name],enumerable:!0})},__copyProps=(to,from,except,desc)=>{if(from&&typeof from=="object"||typeof from=="function")for(let key of __getOwnPropNames(from))!__hasOwnProp.call(to,key)&&key!==except&&__defProp(to,key,{get:()=>from[key],enumerable:!(desc=__getOwnPropDesc(from,key))||desc.enumerable});return to};var __toCommonJS=mod=>__copyProps(__defProp({},"__esModule",{value:!0}),mod);var greet_exports={};__export(greet_exports,{GREETING:()=>GREETING,greet:()=>greet});function greet(name){return GREETING="hi",GREETING+" "+name}var GREETING,init_greet=__esm({"src/greet.ts"(){console.log("greet init");GREETING="hello"}});var counter_exports={};__export(counter_exports,{default:()=>Counter});var Counter,init_counter=__esm({"src/counter.ts"(){console.log("counter init");Counter=class{n=0;next(){return++this.n}}}});var describe_exports={};__export(describe_exports,{describe:()=>describe});function describe(){return"described: "+greet("d")}var init_describe=__esm({"src/describe.ts"(){init_greet();console.log("describe init")}});var __create=Object.create,__defProp2=Object.defineProperty,__getOwnPropDesc2=Object.getOwnPropertyDescriptor,__getOwnPropNames2=Object.getOwnPropertyNames,__getProtoOf=Object.getPrototypeOf,__hasOwnProp2=Object.prototype.hasOwnProperty,__export2=(target,all)=>{for(var name in all)__defProp2(target,name,{get:all[name],enumerable:!0})},__copyProps2=(to,from,except,desc)=>{if(from&&typeof from=="object"||typeof from=="function")for(let key of __getOwnPropNames2(from))!__hasOwnProp2.call(to,key)&&key!==except&&__defProp2(to,key,{get:()=>from[key],enumerable:!(desc=__getOwnPropDesc2(from,key))||desc.enumerable});return to},__toESM=(mod,isNodeMode,target)=>(target=mod!=null?__create(__getProtoOf(mod)):{},__copyProps2(isNodeMode||!mod||!mod.__esModule?__defProp2(target,"default",{value:mod,enumerable:!0}):target,mod)),__toCommonJS2=mod=>__copyProps2(__defProp2({},"__esModule",{value:!0}),mod),stdin_exports={};__export2(stdin_exports,{Counter:()=>load_import_counter().default,GREETING:()=>load_import_greet().GREETING,describe:()=>load_import_describe().describe,greet:()=>load_import_greet().greet});module.exports=__toCommonJS2(stdin_exports);var import_greet2,load_import_greet=()=>import_greet2??=(init_greet(),__toCommonJS(greet_exports)),import_counter,load_import_counter=()=>import_counter??=__toESM((init_counter(),__toCommonJS(counter_exports))),import_describe,load_import_describe=()=>import_describe??=(init_describe(),__toCommonJS(describe_exports)); + +0 && (module.exports = {"Counter": null,"GREETING": null,"describe": null,"greet": null}); diff --git a/tests/fixtures/npm/node_modules/gtlazyws/package.json b/tests/fixtures/npm/node_modules/gtlazyws/package.json new file mode 100644 index 000000000..bf5ec5b93 --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtlazyws/package.json @@ -0,0 +1,7 @@ +{ + "name": "gtlazyws", + "version": "1.0.0", + "description": "the gtlazy source and plugin bundled by esbuild 0.25.10 with --minify-whitespace --minify-syntax", + "main": "./dist/index.js", + "types": "./dist/index.d.ts" +} diff --git a/tests/harness/npm-cases.ts b/tests/harness/npm-cases.ts index f8d346154..87a6de2a8 100644 --- a/tests/harness/npm-cases.ts +++ b/tests/harness/npm-cases.ts @@ -19,12 +19,12 @@ export function npmCases(fixturesRoot: string): NpmCase[] { return [ ...globSync(join(fixturesRoot, "npm/cases/*/main.ts")) .sort() - // 2465-2469 and 2556-2563 are the --npm-static bundler-emitted-CJS + // 2465-2469 and 2556-2565 are the --npm-static bundler-emitted-CJS // cases (npm-static.test.ts drives them with the opt-in): their // consumers exercise the static rewrite's surface (lexer-visible // names the shipped .d.ts never declares, the __toESM interop // family), so they stay out of the flagless island lane by design. - .filter((entry) => !/\/(246[5-9]|255[6-9]|256[0-3])-[^/]+\/main\.ts$/.test(entry)) + .filter((entry) => !/\/(246[5-9]|255[6-9]|256[0-5])-[^/]+\/main\.ts$/.test(entry)) .map((entry) => ({ name: entry.split("/").at(-2)!, entry, diff --git a/tests/harness/npm-static.test.ts b/tests/harness/npm-static.test.ts index 8560973cd..943b81695 100644 --- a/tests/harness/npm-static.test.ts +++ b/tests/harness/npm-static.test.ts @@ -76,7 +76,7 @@ async function buildStatic(entry: string, npmStatic: string[] | "auto"): Promise ...globSync( join(fixturesRoot, "npm/node_modules/cryptozoo/**/*.{js,mjs,cjs,json,d.ts}"), ).sort(), - // the bundler-emitted-CJS mini packages (cases 2465-2469, 2556-2563) + // the bundler-emitted-CJS mini packages (cases 2465-2469, 2556-2565) ...globSync(join(fixturesRoot, "npm/node_modules/gt*/**/*.{js,cjs,json}")).sort(), ]; for (const f of inputs) hash.update(f).update(readFileSync(f)); @@ -1478,6 +1478,17 @@ describe(`npm-static pilots${sanitize ? " (sanitized)" : ""}`, () => { ]); }, 180_000); + // 2564-2565: a barrel whose re-exports are memoized lazy loaders of + // lazily initialized ES modules (a bundle plugin's shape): ES-module + // importers copy every export when they link (Node's CommonJS import), + // require() reads each export's getter at the read. + test.each([ + ["2564-lazy-barrel", ["gtlazy", "gtlazyws"]], + ["2565-lazy-barrel-require", ["gtlazyuser", "gtlazy"]], + ])("the lazy barrel %s compiles statically and byte-matches Node", async (name, packages) => { + await expectNodeParity(join(fixturesRoot, `npm/cases/${name}/main.ts`), packages); + }, 180_000); + // 2562: a file with unverifiable interop that no compiled code can load // (the program imports a leaf data module and only a type from the // main entry) leaves its package static. From bc32d97339b1b0f82618e4d38feb4b78babef6f0 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Mon, 19 Oct 2026 16:52:01 -0700 Subject: [PATCH 09/35] Compile build-utils' bundled code: star re-exports, aliases, three lowering fixes - A star re-export of a lazily initialized module into the bundle's exports (__reExport(main, (init_x(), __toCommonJS(x_exports)))) initializes the module at the statement and adds its names to the export table after the explicit ones, without default. - The rewrite keeps the original's lexer answer whenever its export table holds accessors (Node's lexer stops at the first one). - An npm alias directory of a selected package ("x-updated": "npm:x@6.3.0") resolves to that package's JavaScript in both resolvers. - "use strict" code may read arguments in functions with parameters. - A comma expression ending in null/undefined coerces its result. - A void conditional return lowers as control flow. - A detached method of a local class drops its empty capture list. Co-Authored-By: Claude Opus 5.5 --- .../src/frontend/lowering/call-signatures.ts | 27 +++++++++++++- .../frontend/lowering/class-method-values.ts | 5 ++- .../compiler/src/frontend/lowering/lowerer.ts | 20 +++++++++++ .../src/frontend/npm-static-bundled-esm.ts | 35 +++++++++++++++++++ .../src/frontend/npm-static-rewrite-syntax.ts | 15 ++++---- packages/compiler/src/frontend/npm-static.ts | 24 +++++++++++++ packages/compiler/src/frontend/resolve.ts | 24 ++++++++++++- .../6260-local-class-detached-method.ts | 16 +++++++++ tests/corpus/6262-comma-null-union-return.cjs | 15 ++++++++ .../6263-strict-arguments-parameters.cjs | 15 ++++++++ .../cases/2566-npm-alias-two-releases/main.ts | 7 ++++ .../node_modules/gtroute | 1 + .../node_modules/gtpattern/lib/index.d.ts | 2 ++ .../node_modules/gtpattern/lib/index.js | 5 +++ .../node_modules/gtpattern/package.json | 6 ++++ .../node_modules/gtpattern/lib/index.d.ts | 2 ++ .../node_modules/gtpattern/lib/index.js | 5 +++ .../node_modules/gtpattern/package.json | 6 ++++ .../gtroute@1.0.0/node_modules/gtpattern | 1 + .../gtroute@1.0.0/node_modules/gtpattern-next | 1 + .../node_modules/gtroute/index.d.ts | 1 + .../node_modules/gtroute/index.js | 7 ++++ .../node_modules/gtroute/package.json | 11 ++++++ tests/harness/npm-cases.ts | 4 +-- tests/harness/npm-static.test.ts | 23 +++++++++++- 25 files changed, 264 insertions(+), 14 deletions(-) create mode 100644 tests/corpus/6260-local-class-detached-method.ts create mode 100644 tests/corpus/6262-comma-null-union-return.cjs create mode 100644 tests/corpus/6263-strict-arguments-parameters.cjs create mode 100644 tests/fixtures/npm/cases/2566-npm-alias-two-releases/main.ts create mode 120000 tests/fixtures/npm/cases/2566-npm-alias-two-releases/node_modules/gtroute create mode 100644 tests/fixtures/npm/store/gtpattern@1.0.0/node_modules/gtpattern/lib/index.d.ts create mode 100644 tests/fixtures/npm/store/gtpattern@1.0.0/node_modules/gtpattern/lib/index.js create mode 100644 tests/fixtures/npm/store/gtpattern@1.0.0/node_modules/gtpattern/package.json create mode 100644 tests/fixtures/npm/store/gtpattern@2.0.0/node_modules/gtpattern/lib/index.d.ts create mode 100644 tests/fixtures/npm/store/gtpattern@2.0.0/node_modules/gtpattern/lib/index.js create mode 100644 tests/fixtures/npm/store/gtpattern@2.0.0/node_modules/gtpattern/package.json create mode 120000 tests/fixtures/npm/store/gtroute@1.0.0/node_modules/gtpattern create mode 120000 tests/fixtures/npm/store/gtroute@1.0.0/node_modules/gtpattern-next create mode 100644 tests/fixtures/npm/store/gtroute@1.0.0/node_modules/gtroute/index.d.ts create mode 100644 tests/fixtures/npm/store/gtroute@1.0.0/node_modules/gtroute/index.js create mode 100644 tests/fixtures/npm/store/gtroute@1.0.0/node_modules/gtroute/package.json diff --git a/packages/compiler/src/frontend/lowering/call-signatures.ts b/packages/compiler/src/frontend/lowering/call-signatures.ts index 9d9c9668c..9de7cd026 100644 --- a/packages/compiler/src/frontend/lowering/call-signatures.ts +++ b/packages/compiler/src/frontend/lowering/call-signatures.ts @@ -707,6 +707,27 @@ export function collectSignatureInner(lowerer: Lowerer, decl: ts.FunctionDeclara }); } +/** True when a "use strict" directive prologue covers the node: its + * file's or an enclosing function's (strict code's arguments object never + * aliases the named parameters). */ +function inStrictCode(node: ts.Node): boolean { + const strict = (stmts: readonly ts.Statement[]): boolean => { + for (const st of stmts) { + if (!ts.isExpressionStatement(st) || !ts.isStringLiteral(st.expression)) return false; + if (st.expression.text === "use strict") return true; + } + return false; + }; + for (let n: ts.Node | undefined = node; n !== undefined; n = n.parent) { + if (ts.isSourceFile(n)) return strict(n.statements.nodes); + if (ts.isFunctionLike(n)) { + const body = (n as { body?: ts.Node }).body; + if (body !== undefined && ts.isBlock(body) && strict(body.statements.nodes)) return true; + } + } + return false; +} + /** Arrows inherit arguments. Ordinary JS functions receive a hidden * array unless a declared rest parameter already owns that contract. * Class methods are strict; other parameterized forms require ES modules @@ -732,7 +753,11 @@ function appendArgumentsParameter( const classMethod = ts.isMethodDeclaration(node) && (ts.isClassDeclaration(node.parent) || ts.isClassExpression(node.parent)); - if (!classMethod && !isNodeEsmFile(node.getSourceFile(), lowerer.program)) { + if ( + !classMethod && + !isNodeEsmFile(node.getSourceFile(), lowerer.program) && + !inStrictCode(node) + ) { lowerer.unsupported( "SC1090", node, diff --git a/packages/compiler/src/frontend/lowering/class-method-values.ts b/packages/compiler/src/frontend/lowering/class-method-values.ts index 10572377e..ff732efff 100644 --- a/packages/compiler/src/frontend/lowering/class-method-values.ts +++ b/packages/compiler/src/frontend/lowering/class-method-values.ts @@ -539,8 +539,11 @@ export function finalizeClassMethodValues(lowerer: Lowerer, functions: IrFunctio if (usesThis) continue; // A method that never uses this remains callable when detached. const name = `%method.receiverless:${target.name}`; + // A local class's method keeps an (empty) class-capture list; the + // detached copy has no receiver to read captures through. + const { classCaptures: _none, ...detached } = target; functions.push({ - ...target, + ...detached, name, params: target.params.slice(1), locals: target.locals.filter((local) => local.id !== thisParam.localId), diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index ce2390d95..1f905cae4 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -8478,6 +8478,17 @@ export class Lowerer { * else (including a DIFFERENT union) is left for requireExactShape, which * rejects union mismatches with SC2003. */ coerceToExpected(expr: IrExpr, expected: IrType): IrExpr { + // A comma expression ending in a unit literal (`return log(), null`): + // its effects run first, unchanged, and the unit value takes the + // destination's representation (a bare unit literal has none). + if ( + expr.kind === "seqExpr" && + expr.result.kind === "unitLit" && + !typeEquals(expr.type, expected) + ) { + const result = this.coerceToExpected(expr.result, expected); + return { ...expr, result, type: result.type }; + } // A destination that holds undefined takes an unassigned deferred-init // field's slot as is (Node passes the undefined along). if (expected.kind === "union" && this.armTag(expected.unionId, UNDEFINED_T) >= 0) { @@ -9905,6 +9916,15 @@ export class Lowerer { e = this.returnedPromiseAwait(e); } if (e.kind === "unitLit") return { kind: "return", value: null, loc }; + // `return c ? f() : g()` over void calls: a void conditional has no + // value representation, so its branch selection becomes control flow. + if (e.kind === "ternary" && e.type.kind === "void") { + const branch = (x: IrExpr): IrStmt => + x.kind === "ternary" && x.type.kind === "void" + ? { kind: "if", cond: x.cond, then: [branch(x.then)], else_: [branch(x.else_)], loc } + : { kind: "exprStmt", expr: x, loc }; + return { kind: "block", body: [branch(e), { kind: "return", value: null, loc }], loc }; + } if (e.type.kind === "void") return { kind: "return", value: e, loc }; return { kind: "block", diff --git a/packages/compiler/src/frontend/npm-static-bundled-esm.ts b/packages/compiler/src/frontend/npm-static-bundled-esm.ts index cf3664264..10177ccb9 100644 --- a/packages/compiler/src/frontend/npm-static-bundled-esm.ts +++ b/packages/compiler/src/frontend/npm-static-bundled-esm.ts @@ -633,6 +633,41 @@ export function lowerBundledEsmInits( names.has(access.name.text) && plainRead(access); const holder = ns.parent; + // `__reExport(main, ns, module.exports)`: a star re-export of the + // module into the bundle's own exports (`export * from` in its entry). + // The initializer runs at the statement; the names become live getter + // entries of the export table, after its explicit ones (first wins) + // and without `default`, exactly the names __reExport copies. + if ( + holder !== undefined && + ts.isCallExpression(holder) && + ts.isIdentifier(holder.expression) && + helperBase(holder.expression.text) === "__reExport" && + holder.arguments.nodes[1] === ns && + holder.arguments.nodes.length <= 3 && + ts.isExpressionStatement(holder.parent) && + mainTarget !== null && + ts.isIdentifier(holder.arguments.nodes[0]!) && + (holder.arguments.nodes[0] as ts.Identifier).text === mainTarget + ) { + const inner = unparen(ns); + const init = + ts.isBinaryExpression(inner) && inner.operatorToken.kind === ts.SyntaxKind.CommaToken + ? `${unparen(inner.left).getText(sf)}; ` + : ""; + if (shadowed(holder, table.entries)) return { degrade: `its shipped JS ${why}` }; + const entries = table.entries + .filter(([n]) => n !== "default") + .map(([n, value]) => `${JSON.stringify(n)}: () => ${getterText(value)}`) + .join(", "); + edits.push({ + start: holder.parent.getStart(sf), + end: holder.parent.getEnd(), + text: `${init}__export(${mainTarget}, { ${entries} });`, + }); + lowered.add(r.text); + continue; + } let readable = memberRead(holder, ns); if ( !readable && diff --git a/packages/compiler/src/frontend/npm-static-rewrite-syntax.ts b/packages/compiler/src/frontend/npm-static-rewrite-syntax.ts index 3b553c3f2..5963cf7b5 100644 --- a/packages/compiler/src/frontend/npm-static-rewrite-syntax.ts +++ b/packages/compiler/src/frontend/npm-static-rewrite-syntax.ts @@ -1080,13 +1080,12 @@ export function rewriteBundlerCjsSyntax( lines.push( `module.exports = {${spreadParts.join("")}${plainParts.join("")}${esModulePart}${memberParts.join("")}};`, ); - /* Node's lexer sees the original's names for these entries (esbuild's - * dead annotation), but no accessor of the canonical table: the host - * keeps those names lexer-visible, so ES-module importers link exactly - * as they do against the original. */ - const accessorNames = [...entries.values()] - .filter((spec) => spec.kind === "getter" && lexed.exports.has(spec.name)) - .map((spec) => spec.name); - if (accessorNames.length > 0) host.lexerVisibleNames?.(filePath, accessorNames); + /* Node's lexer skips accessors (and stops reading an object literal at + * the first one), so a canonical table with accessor entries hides + * names the original showed it (esbuild's dead annotation): the host + * keeps the original's lexer answer, so ES-module importers link + * exactly as they do against the original. */ + if ([...entries.values()].some((spec) => spec.kind === "getter")) + host.lexerVisibleNames?.(filePath, [...lexed.exports]); return text + lines.join("\n") + "\n"; } diff --git a/packages/compiler/src/frontend/npm-static.ts b/packages/compiler/src/frontend/npm-static.ts index 77eab1d3a..e00122238 100644 --- a/packages/compiler/src/frontend/npm-static.ts +++ b/packages/compiler/src/frontend/npm-static.ts @@ -63,6 +63,7 @@ export function setNpmStaticPackages>(packages: T): v clearVirtualSources(); untypedPkgCache.clear(); realpathProbed.clear(); + aliasNames.clear(); } export function setNpmStaticDeclarationOverloads( @@ -177,9 +178,32 @@ function shadowTargetOf(path: string): { pkg: string; viaTypes: boolean } | null if (mangledTypesName(pkg) === mangled) return { pkg, viaTypes: true }; } } + // An npm alias (`"x-updated": "npm:x@6.3.0"`) installs the package under + // another directory name: its own manifest names the package. + const aliased = aliasedPackageName(`${norm.slice(0, idx)}/node_modules/${dirName}`); + if (aliased !== null && aliased !== dirName && activePackages.has(aliased)) + return { pkg: aliased, viaTypes: false }; return null; } +/** The "name" a node_modules directory's own package.json declares, or + * null (cached per load). */ +const aliasNames = new Map(); +function aliasedPackageName(pkgDir: string): string | null { + const hit = aliasNames.get(pkgDir); + if (hit !== undefined) return hit; + let name: string | null = null; + try { + const text = trackedReadFile(`${pkgDir}/package.json`); + const declared = text === null ? undefined : (JSON.parse(text) as { name?: unknown }).name; + if (typeof declared === "string") name = declared; + } catch { + name = null; + } + aliasNames.set(pkgDir, name); + return name; +} + /** The opted-in package.json transform, applied identically in BOTH * resolution worlds (the tsgo host's fs shadow and resolve.ts): * 1. every types claim goes — the top-level "types"/"typings" fields diff --git a/packages/compiler/src/frontend/resolve.ts b/packages/compiler/src/frontend/resolve.ts index 463400475..f061ca065 100644 --- a/packages/compiler/src/frontend/resolve.ts +++ b/packages/compiler/src/frontend/resolve.ts @@ -18,6 +18,7 @@ import { dirname, isAbsolute, join, resolve } from "node:path"; import { isNpmStaticPackage, + npmStaticActive, npmStaticPackageOfPath, npmStaticTransformPkgJson, } from "./npm-static.js"; @@ -868,7 +869,11 @@ export function resolveBareModule( // never consulted — mirroring the shadowed world the tsgo host serves. const runtimeImport = mode === "runtime-js" || mode === "runtime-source"; const npmStatic = - mode === "js-only" || (!runtimeImport && mode !== "types-only" && isNpmStaticPackage(pkgName)); + mode === "js-only" || + (!runtimeImport && + mode !== "types-only" && + (isNpmStaticPackage(pkgName) || + (npmStaticActive() && isNpmStaticPackage(aliasedPackageOf(fromFile, pkgName))))); const conditions = runtimeImport ? RUNTIME_IMPORT_CONDITIONS : npmStatic @@ -970,6 +975,23 @@ export function resolveBareModule( return npmStatic ? passOnce("js") : (passOnce("types") ?? passOnce("js")); } +/** The package an npm alias directory installs (`"x-updated": + * "npm:x@6.3.0"` puts package x under node_modules/x-updated): the name + * the nearest node_modules//package.json declares when it + * differs from the directory name, else null. */ +function aliasedPackageOf(fromFile: string, dirName: string): string | null { + for (let dir = dirname(resolve(fromFile)); ; ) { + const pkgDir = join(dir, "node_modules", dirName); + if (isDirectory(pkgDir)) { + const name = pkgJsonOf(pkgDir)?.name; + return typeof name === "string" && name !== dirName ? name : null; + } + const parent = dirname(dir); + if (parent === dir) return null; + dir = parent; + } +} + export function resolveWorkspaceSourceModule( fromFile: string, specifier: string, diff --git a/tests/corpus/6260-local-class-detached-method.ts b/tests/corpus/6260-local-class-detached-method.ts new file mode 100644 index 000000000..528539c48 --- /dev/null +++ b/tests/corpus/6260-local-class-detached-method.ts @@ -0,0 +1,16 @@ +// A method that never reads `this`, taken off an instance of a class +// declared inside a function: the detached method runs without a receiver. +function helper(): string { + return "helper"; +} +function make() { + return class Local { + tag = "t"; + plain(): string { + return helper(); + } + }; +} +const C = make(); +const detached = new C().plain; +console.log(detached()); diff --git a/tests/corpus/6262-comma-null-union-return.cjs b/tests/corpus/6262-comma-null-union-return.cjs new file mode 100644 index 000000000..52cef027e --- /dev/null +++ b/tests/corpus/6262-comma-null-union-return.cjs @@ -0,0 +1,15 @@ +// `return log(), null` from a function that otherwise returns a list: the +// comma expression's effects run, then null takes the result's +// representation. +"use strict"; +async function promote(list) { + if (list.length === 0) return console.log("no static mounts"), null; + return list.map((x) => x * 2); +} +function first(list) { + if (list.length === 0) return console.log("empty"), undefined; + return list[0]; +} +promote([]).then((v) => console.log("promoted", v)); +promote([1, 2]).then((v) => console.log("promoted", v)); +console.log(first([]), first([7])); diff --git a/tests/corpus/6263-strict-arguments-parameters.cjs b/tests/corpus/6263-strict-arguments-parameters.cjs new file mode 100644 index 000000000..463cb316a --- /dev/null +++ b/tests/corpus/6263-strict-arguments-parameters.cjs @@ -0,0 +1,15 @@ +// A CommonJS file under "use strict": a function with named parameters +// reads `arguments`, which in strict code never aliases those parameters. +"use strict"; +function wrap(fn, label) { + label = label + "!"; + return [label, arguments.length, arguments[1], typeof arguments[0]].join(" "); +} +function sum(a, b) { + a = 100; + let total = 0; + for (let i = 0; i < arguments.length; i++) total += arguments[i]; + return total; +} +console.log(wrap(() => 1, "tag")); +console.log(sum(1, 2, 3)); diff --git a/tests/fixtures/npm/cases/2566-npm-alias-two-releases/main.ts b/tests/fixtures/npm/cases/2566-npm-alias-two-releases/main.ts new file mode 100644 index 000000000..6ccf96cdb --- /dev/null +++ b/tests/fixtures/npm/cases/2566-npm-alias-two-releases/main.ts @@ -0,0 +1,7 @@ +// gtroute requires two releases of one package: gtpattern 1.0.0 under its +// own name and 2.0.0 through an npm alias directory (gtpattern-next), +// whose manifest still names gtpattern. Both compile from their own +// JavaScript, as Node loads them. +import { compile } from "gtroute"; + +console.log(compile("/users/:id")); diff --git a/tests/fixtures/npm/cases/2566-npm-alias-two-releases/node_modules/gtroute b/tests/fixtures/npm/cases/2566-npm-alias-two-releases/node_modules/gtroute new file mode 120000 index 000000000..e0ddd1565 --- /dev/null +++ b/tests/fixtures/npm/cases/2566-npm-alias-two-releases/node_modules/gtroute @@ -0,0 +1 @@ +../../../store/gtroute@1.0.0/node_modules/gtroute \ No newline at end of file diff --git a/tests/fixtures/npm/store/gtpattern@1.0.0/node_modules/gtpattern/lib/index.d.ts b/tests/fixtures/npm/store/gtpattern@1.0.0/node_modules/gtpattern/lib/index.d.ts new file mode 100644 index 000000000..1ba7851fa --- /dev/null +++ b/tests/fixtures/npm/store/gtpattern@1.0.0/node_modules/gtpattern/lib/index.d.ts @@ -0,0 +1,2 @@ +export declare const version: string; +export declare function compile(path: string): string; diff --git a/tests/fixtures/npm/store/gtpattern@1.0.0/node_modules/gtpattern/lib/index.js b/tests/fixtures/npm/store/gtpattern@1.0.0/node_modules/gtpattern/lib/index.js new file mode 100644 index 000000000..c76f7222f --- /dev/null +++ b/tests/fixtures/npm/store/gtpattern@1.0.0/node_modules/gtpattern/lib/index.js @@ -0,0 +1,5 @@ +"use strict"; +exports.version = "v1"; +exports.compile = function compile(path) { + return path.split("/").filter((part) => part.length > 0).join("."); +}; diff --git a/tests/fixtures/npm/store/gtpattern@1.0.0/node_modules/gtpattern/package.json b/tests/fixtures/npm/store/gtpattern@1.0.0/node_modules/gtpattern/package.json new file mode 100644 index 000000000..fd67b847e --- /dev/null +++ b/tests/fixtures/npm/store/gtpattern@1.0.0/node_modules/gtpattern/package.json @@ -0,0 +1,6 @@ +{ + "name": "gtpattern", + "version": "1.0.0", + "main": "lib/index.js", + "typings": "lib/index.d.ts" +} diff --git a/tests/fixtures/npm/store/gtpattern@2.0.0/node_modules/gtpattern/lib/index.d.ts b/tests/fixtures/npm/store/gtpattern@2.0.0/node_modules/gtpattern/lib/index.d.ts new file mode 100644 index 000000000..1ba7851fa --- /dev/null +++ b/tests/fixtures/npm/store/gtpattern@2.0.0/node_modules/gtpattern/lib/index.d.ts @@ -0,0 +1,2 @@ +export declare const version: string; +export declare function compile(path: string): string; diff --git a/tests/fixtures/npm/store/gtpattern@2.0.0/node_modules/gtpattern/lib/index.js b/tests/fixtures/npm/store/gtpattern@2.0.0/node_modules/gtpattern/lib/index.js new file mode 100644 index 000000000..a192ea3ab --- /dev/null +++ b/tests/fixtures/npm/store/gtpattern@2.0.0/node_modules/gtpattern/lib/index.js @@ -0,0 +1,5 @@ +"use strict"; +exports.version = "v2"; +exports.compile = function compile(path) { + return path.split("/").filter((part) => part.length > 0).map((part) => (part.startsWith(":") ? "<" + part.slice(1) + ">" : part)).join("/"); +}; diff --git a/tests/fixtures/npm/store/gtpattern@2.0.0/node_modules/gtpattern/package.json b/tests/fixtures/npm/store/gtpattern@2.0.0/node_modules/gtpattern/package.json new file mode 100644 index 000000000..6eb6c6d78 --- /dev/null +++ b/tests/fixtures/npm/store/gtpattern@2.0.0/node_modules/gtpattern/package.json @@ -0,0 +1,6 @@ +{ + "name": "gtpattern", + "version": "2.0.0", + "main": "lib/index.js", + "typings": "lib/index.d.ts" +} diff --git a/tests/fixtures/npm/store/gtroute@1.0.0/node_modules/gtpattern b/tests/fixtures/npm/store/gtroute@1.0.0/node_modules/gtpattern new file mode 120000 index 000000000..25c5c39d5 --- /dev/null +++ b/tests/fixtures/npm/store/gtroute@1.0.0/node_modules/gtpattern @@ -0,0 +1 @@ +../../gtpattern@1.0.0/node_modules/gtpattern \ No newline at end of file diff --git a/tests/fixtures/npm/store/gtroute@1.0.0/node_modules/gtpattern-next b/tests/fixtures/npm/store/gtroute@1.0.0/node_modules/gtpattern-next new file mode 120000 index 000000000..916b8c35f --- /dev/null +++ b/tests/fixtures/npm/store/gtroute@1.0.0/node_modules/gtpattern-next @@ -0,0 +1 @@ +../../gtpattern@2.0.0/node_modules/gtpattern \ No newline at end of file diff --git a/tests/fixtures/npm/store/gtroute@1.0.0/node_modules/gtroute/index.d.ts b/tests/fixtures/npm/store/gtroute@1.0.0/node_modules/gtroute/index.d.ts new file mode 100644 index 000000000..2d203bf4e --- /dev/null +++ b/tests/fixtures/npm/store/gtroute@1.0.0/node_modules/gtroute/index.d.ts @@ -0,0 +1 @@ +export declare function compile(path: string): string; diff --git a/tests/fixtures/npm/store/gtroute@1.0.0/node_modules/gtroute/index.js b/tests/fixtures/npm/store/gtroute@1.0.0/node_modules/gtroute/index.js new file mode 100644 index 000000000..da52d5e12 --- /dev/null +++ b/tests/fixtures/npm/store/gtroute@1.0.0/node_modules/gtroute/index.js @@ -0,0 +1,7 @@ +"use strict"; +const legacy = require("gtpattern"); +const next = require("gtpattern-next"); + +exports.compile = function compile(path) { + return legacy.version + ":" + legacy.compile(path) + " " + next.version + ":" + next.compile(path); +}; diff --git a/tests/fixtures/npm/store/gtroute@1.0.0/node_modules/gtroute/package.json b/tests/fixtures/npm/store/gtroute@1.0.0/node_modules/gtroute/package.json new file mode 100644 index 000000000..10bee06e7 --- /dev/null +++ b/tests/fixtures/npm/store/gtroute@1.0.0/node_modules/gtroute/package.json @@ -0,0 +1,11 @@ +{ + "name": "gtroute", + "version": "1.0.0", + "description": "depends on two releases of gtpattern, the newer one through an npm alias (\"gtpattern-next\": \"npm:gtpattern@2.0.0\"), in a pnpm store layout", + "main": "index.js", + "types": "index.d.ts", + "dependencies": { + "gtpattern": "1.0.0", + "gtpattern-next": "npm:gtpattern@2.0.0" + } +} diff --git a/tests/harness/npm-cases.ts b/tests/harness/npm-cases.ts index 87a6de2a8..69ce3917f 100644 --- a/tests/harness/npm-cases.ts +++ b/tests/harness/npm-cases.ts @@ -19,12 +19,12 @@ export function npmCases(fixturesRoot: string): NpmCase[] { return [ ...globSync(join(fixturesRoot, "npm/cases/*/main.ts")) .sort() - // 2465-2469 and 2556-2565 are the --npm-static bundler-emitted-CJS + // 2465-2469 and 2556-2566 are the --npm-static bundler-emitted-CJS // cases (npm-static.test.ts drives them with the opt-in): their // consumers exercise the static rewrite's surface (lexer-visible // names the shipped .d.ts never declares, the __toESM interop // family), so they stay out of the flagless island lane by design. - .filter((entry) => !/\/(246[5-9]|255[6-9]|256[0-5])-[^/]+\/main\.ts$/.test(entry)) + .filter((entry) => !/\/(246[5-9]|255[6-9]|256[0-6])-[^/]+\/main\.ts$/.test(entry)) .map((entry) => ({ name: entry.split("/").at(-2)!, entry, diff --git a/tests/harness/npm-static.test.ts b/tests/harness/npm-static.test.ts index 943b81695..1741c7470 100644 --- a/tests/harness/npm-static.test.ts +++ b/tests/harness/npm-static.test.ts @@ -76,8 +76,9 @@ async function buildStatic(entry: string, npmStatic: string[] | "auto"): Promise ...globSync( join(fixturesRoot, "npm/node_modules/cryptozoo/**/*.{js,mjs,cjs,json,d.ts}"), ).sort(), - // the bundler-emitted-CJS mini packages (cases 2465-2469, 2556-2565) + // the bundler-emitted-CJS mini packages (cases 2465-2469, 2556-2566) ...globSync(join(fixturesRoot, "npm/node_modules/gt*/**/*.{js,cjs,json}")).sort(), + ...globSync(join(fixturesRoot, "npm/store/gt*/**/*.{js,cjs,json}")).sort(), ]; for (const f of inputs) hash.update(f).update(readFileSync(f)); const key = hash @@ -1489,6 +1490,26 @@ describe(`npm-static pilots${sanitize ? " (sanitized)" : ""}`, () => { await expectNodeParity(join(fixturesRoot, `npm/cases/${name}/main.ts`), packages); }, 180_000); + // 2566: two releases of one package in a pnpm store, the newer through + // an npm alias directory; the default selection compiles both copies. + test("an npm alias of a selected package compiles from its JavaScript", async () => { + const entry = join(fixturesRoot, "npm/cases/2566-npm-alias-two-releases/main.ts"); + const { coverage } = analyze(entry); + expect(coverage.npmStatic).toEqual( + expect.arrayContaining([ + { package: "gtroute", status: "static" }, + { package: "gtpattern", status: "static" }, + ]), + ); + const binary = await buildStatic(entry, "auto"); + const [nodeRes, nativeRes] = await Promise.all([ + runBinary("node", [entry]), + runBinary(binary, []), + ]); + expect(nativeRes.stdout.toString("utf8")).toBe(nodeRes.stdout.toString("utf8")); + expect(nativeRes.exitCode).toBe(nodeRes.exitCode); + }, 180_000); + // 2562: a file with unverifiable interop that no compiled code can load // (the program imports a leaf data module and only a type from the // main entry) leaves its package static. From 02dba55958f66d22f02e621a1d572e7fe09db367 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Mon, 19 Oct 2026 16:55:55 -0700 Subject: [PATCH 10/35] Record order baselines for the new cases and corpus programs; format Co-Authored-By: Claude Opus 5.5 --- .../compiler/src/frontend/lowering/lower-exprs.ts | 14 ++++++++++++-- packages/compiler/src/frontend/resolve.ts | 2 +- .../6260-local-class-detached-method.ts.json | 6 ++++++ .../corpus/6262-comma-null-union-return.cjs.json | 6 ++++++ .../6263-strict-arguments-parameters.cjs.json | 6 ++++++ .../npm/cases/2564-lazy-barrel/main.ts.json | 6 ++++++ .../cases/2565-lazy-barrel-require/main.ts.json | 6 ++++++ .../cases/2566-npm-alias-two-releases/main.ts.json | 6 ++++++ tests/harness/npm-static.test.ts | 10 +++++++--- 9 files changed, 56 insertions(+), 6 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6260-local-class-detached-method.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6262-comma-null-union-return.cjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6263-strict-arguments-parameters.cjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2564-lazy-barrel/main.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2565-lazy-barrel-require/main.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2566-npm-alias-two-releases/main.ts.json diff --git a/packages/compiler/src/frontend/lowering/lower-exprs.ts b/packages/compiler/src/frontend/lowering/lower-exprs.ts index cabbc31f5..afcecb2c5 100644 --- a/packages/compiler/src/frontend/lowering/lower-exprs.ts +++ b/packages/compiler/src/frontend/lowering/lower-exprs.ts @@ -12595,7 +12595,12 @@ function cjsAccessorSnapshotOf( const call = cjsAccessorCall(lowerer, getter, locOf(getter)); if (call === null) return null; const snapshot = { id: `%g.cjsSnapshot.${lowerer.cjsAccessorSnapshots.size}`, type: call.type }; - lowerer.globalsList.push({ id: snapshot.id, name: "%cjsSnapshot", type: call.type, mutable: true }); + lowerer.globalsList.push({ + id: snapshot.id, + name: "%cjsSnapshot", + type: call.type, + mutable: true, + }); lowerer.cjsAccessorSnapshots.set(getter, snapshot); return snapshot; } @@ -12619,7 +12624,12 @@ export function cjsAccessorSnapshotStores( lowerer.cjsSnapshotReady.set(dep, readyId); } const stores: IrStmt[] = [ - { kind: "assign", localId: readyId, value: { kind: "boolLit", value: true, type: BOOL, loc }, loc }, + { + kind: "assign", + localId: readyId, + value: { kind: "boolLit", value: true, type: BOOL, loc }, + loc, + }, ]; for (const getter of getters) { const snapshot = cjsAccessorSnapshotOf(lowerer, getter); diff --git a/packages/compiler/src/frontend/resolve.ts b/packages/compiler/src/frontend/resolve.ts index f061ca065..36470a643 100644 --- a/packages/compiler/src/frontend/resolve.ts +++ b/packages/compiler/src/frontend/resolve.ts @@ -980,7 +980,7 @@ export function resolveBareModule( * the nearest node_modules//package.json declares when it * differs from the directory name, else null. */ function aliasedPackageOf(fromFile: string, dirName: string): string | null { - for (let dir = dirname(resolve(fromFile)); ; ) { + for (let dir = dirname(resolve(fromFile)); ;) { const pkgDir = join(dir, "node_modules", dirName); if (isDirectory(pkgDir)) { const name = pkgJsonOf(pkgDir)?.name; diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6260-local-class-detached-method.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6260-local-class-detached-method.ts.json new file mode 100644 index 000000000..d73c701d7 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6260-local-class-detached-method.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6260-local-class-detached-method.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6262-comma-null-union-return.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6262-comma-null-union-return.cjs.json new file mode 100644 index 000000000..e167cc977 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6262-comma-null-union-return.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6262-comma-null-union-return.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6263-strict-arguments-parameters.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6263-strict-arguments-parameters.cjs.json new file mode 100644 index 000000000..b7ff6a960 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6263-strict-arguments-parameters.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6263-strict-arguments-parameters.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2564-lazy-barrel/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2564-lazy-barrel/main.ts.json new file mode 100644 index 000000000..b14560cea --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2564-lazy-barrel/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/2564-lazy-barrel/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2565-lazy-barrel-require/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2565-lazy-barrel-require/main.ts.json new file mode 100644 index 000000000..77cf9e52c --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2565-lazy-barrel-require/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/2565-lazy-barrel-require/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2566-npm-alias-two-releases/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2566-npm-alias-two-releases/main.ts.json new file mode 100644 index 000000000..a412bc875 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2566-npm-alias-two-releases/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/2566-npm-alias-two-releases/main.ts" + ], + "diags": [] +} diff --git a/tests/harness/npm-static.test.ts b/tests/harness/npm-static.test.ts index 1741c7470..3543aeb0a 100644 --- a/tests/harness/npm-static.test.ts +++ b/tests/harness/npm-static.test.ts @@ -1486,9 +1486,13 @@ describe(`npm-static pilots${sanitize ? " (sanitized)" : ""}`, () => { test.each([ ["2564-lazy-barrel", ["gtlazy", "gtlazyws"]], ["2565-lazy-barrel-require", ["gtlazyuser", "gtlazy"]], - ])("the lazy barrel %s compiles statically and byte-matches Node", async (name, packages) => { - await expectNodeParity(join(fixturesRoot, `npm/cases/${name}/main.ts`), packages); - }, 180_000); + ])( + "the lazy barrel %s compiles statically and byte-matches Node", + async (name, packages) => { + await expectNodeParity(join(fixturesRoot, `npm/cases/${name}/main.ts`), packages); + }, + 180_000, + ); // 2566: two releases of one package in a pnpm store, the newer through // an npm alias directory; the default selection compiles both copies. From 4f7539103c553f46a0cdfdf8bdafc7e5f611dd6d Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Mon, 19 Oct 2026 19:10:11 -0500 Subject: [PATCH 11/35] Keep a lowered ES module namespace unobservable through calls that pass it as `this` A call through a bundled ES module's namespace (`ns.fn()`, `(ns.fn)()`, a tagged template) passes the namespace as `this`. Node's namespace is the __toCommonJS object (`__esModule` first and non-enumerable, non-configurable getters); the lowered object of getters differs. The namespace lowers only when every such call targets a top-level function, never reassigned, that reads no `this` (an arrow inside it included); otherwise the package stays refused. --- .../frontend/npm-static-bundled-esm.test.ts | 36 +++++++ .../src/frontend/npm-static-bundled-esm.ts | 58 +++++++++++- .../2567-esm-namespace-this/main.ts.json | 6 ++ .../npm/cases/2567-esm-namespace-this/main.ts | 7 ++ .../node_modules/gtesmthis/dist/index.d.ts | 2 + .../npm/node_modules/gtesmthis/dist/index.js | 94 +++++++++++++++++++ .../npm/node_modules/gtesmthis/package.json | 10 ++ tests/harness/npm-cases.ts | 4 +- tests/harness/npm-static.test.ts | 17 +++- 9 files changed, 227 insertions(+), 7 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2567-esm-namespace-this/main.ts.json create mode 100644 tests/fixtures/npm/cases/2567-esm-namespace-this/main.ts create mode 100644 tests/fixtures/npm/node_modules/gtesmthis/dist/index.d.ts create mode 100644 tests/fixtures/npm/node_modules/gtesmthis/dist/index.js create mode 100644 tests/fixtures/npm/node_modules/gtesmthis/package.json diff --git a/packages/compiler/src/frontend/npm-static-bundled-esm.test.ts b/packages/compiler/src/frontend/npm-static-bundled-esm.test.ts index 0b9ecd042..46ebfad4f 100644 --- a/packages/compiler/src/frontend/npm-static-bundled-esm.test.ts +++ b/packages/compiler/src/frontend/npm-static-bundled-esm.test.ts @@ -108,6 +108,42 @@ console.log(load(), (init_lazy(), __toCommonJS(lazy_exports)).value()); expect(lowered).toContain('(require("os")).EOL'); }); +test("calls of exported functions that never read their own `this` still lower", () => { + const lowered = expectSameBehavior(`${helpers}var t_exports = {}; +__export(t_exports, { plain: () => plain, inner: () => inner }); +function plain(x) { return "plain " + x; } +function inner() { + function own() { return typeof this; } + return "inner " + own.call(1); +} +var init_t = __esm({ "t.ts"() {} }); +const t = (init_t(), __toCommonJS(t_exports)); +console.log(t.plain(1), (t.plain)(2), t.inner()); +`); + expect(lowered).not.toContain("__toCommonJS(t_exports)"); +}); + +test.each([ + ["a direct call", `NS.who()`], + ["a parenthesized callee", `(NS.who)()`], + ["a tagged template", "NS.who`x`"], + ["a call through a local", `(() => { const m = NS; return m.who(); })()`], + ["an arrow inside the function", `NS.arrow()()`], +])("%s that passes the namespace as `this` keeps the package refused", (_, use) => { + const result = lower(`${helpers}var w_exports = {}; +__export(w_exports, { who: () => who, arrow: () => arrow }); +function who() { return this.__esModule; } +function arrow() { return () => this; } +var init_w = __esm({ "w.ts"() {} }); +console.log(${use.replaceAll("NS", "(init_w(), __toCommonJS(w_exports))")}); +`); + expect(result).toEqual({ + degrade: expect.stringContaining( + "calls only of exported functions that do not read 'this'", + ) as string, + }); +}); + test("minified initializers sharing a var statement lower the same way", () => { expectSameBehavior( helpers + diff --git a/packages/compiler/src/frontend/npm-static-bundled-esm.ts b/packages/compiler/src/frontend/npm-static-bundled-esm.ts index cf4f6280b..37d2d7b1d 100644 --- a/packages/compiler/src/frontend/npm-static-bundled-esm.ts +++ b/packages/compiler/src/frontend/npm-static-bundled-esm.ts @@ -185,6 +185,44 @@ function plainRead(access: ts.Expression): boolean { return true; } +/** True when a member read is the callee of a call or a tag, which receives + * the object it was read from as `this`. */ +function calledWithReceiver(access: ts.Expression): boolean { + let callee: ts.Node = access; + while (callee.parent !== undefined && ts.isParenthesizedExpression(callee.parent)) + callee = callee.parent; + const p = callee.parent; + return ( + p !== undefined && + ((ts.isCallExpression(p) && p.expression === callee) || + (ts.isTaggedTemplateExpression(p) && p.tag === callee)) + ); +} + +/** True when a function reads its own `this`: in its body, or in an arrow + * function inside it. */ +function readsThis(fn: ts.FunctionDeclaration): boolean { + let found = false; + if (fn.body === undefined) return false; + ts.walkPreorder(fn.body, (n) => { + if (found) return "stop"; + if ( + ts.isFunctionDeclaration(n) || + ts.isFunctionExpression(n) || + ts.isMethodDeclaration(n) || + ts.isGetAccessorDeclaration(n) || + ts.isSetAccessorDeclaration(n) || + ts.isConstructorDeclaration(n) || + ts.isClassDeclaration(n) || + ts.isClassExpression(n) + ) + return "skip"; + if (n.kind === ts.SyntaxKind.ThisKeyword) found = true; + return undefined; + }); + return found; +} + /** The `__export(T, { name: () => value, … })` getter table of T, or null. */ function exportTableOf( call: ts.CallExpression, @@ -457,7 +495,7 @@ export function lowerBundledEsmInits( p.parent === table.stmt ) continue; - const why = `reads the namespace of a bundled ES module ('${r.text}') in a way the static rewrite cannot follow (only member reads of its exports compile)`; + const why = `reads the namespace of a bundled ES module ('${r.text}') in a way the static rewrite cannot follow (only member reads of its exports compile, and calls only of exported functions that do not read 'this')`; if ( p === undefined || !ts.isCallExpression(p) || @@ -480,13 +518,25 @@ export function lowerBundledEsmInits( ns = up; else break; } - const names = new Set(table.entries.map(([n]) => n)); + const values = new Map(table.entries); + // A call through the namespace (`ns.fn()`) passes the object the read + // came from as `this`; the lowered object differs from Node's module + // object (`__esModule`, property attributes), so it must stay + // unobservable: the callee is a top-level function that never reads + // `this`. + const receiverHidden = (name: string): boolean => { + const value = values.get(name); + if (value === undefined || !ts.isIdentifier(value) || !stable(value.text)) return false; + const decl = top.get(value.text)?.node; + return decl !== undefined && ts.isFunctionDeclaration(decl) && !readsThis(decl); + }; const memberRead = (access: ts.Node | undefined, of: ts.Node): boolean => access !== undefined && ts.isPropertyAccessExpression(access) && access.expression === of && - names.has(access.name.text) && - plainRead(access); + values.has(access.name.text) && + plainRead(access) && + (!calledWithReceiver(access) || receiverHidden(access.name.text)); const holder = ns.parent; let readable = memberRead(holder, ns); if ( diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2567-esm-namespace-this/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2567-esm-namespace-this/main.ts.json new file mode 100644 index 000000000..0739a6f0c --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2567-esm-namespace-this/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/2567-esm-namespace-this/main.ts" + ], + "diags": [] +} diff --git a/tests/fixtures/npm/cases/2567-esm-namespace-this/main.ts b/tests/fixtures/npm/cases/2567-esm-namespace-this/main.ts new file mode 100644 index 000000000..d6f084bc0 --- /dev/null +++ b/tests/fixtures/npm/cases/2567-esm-namespace-this/main.ts @@ -0,0 +1,7 @@ +// A lazily initialized ES module export that reads `this`, called through +// the module's namespace (`lazy.who()`): Node passes the namespace object, +// whose `__esModule` is true. A lowered namespace object would differ, so +// the package stays refused. +import { load } from "gtesmthis"; + +console.log(load()); diff --git a/tests/fixtures/npm/node_modules/gtesmthis/dist/index.d.ts b/tests/fixtures/npm/node_modules/gtesmthis/dist/index.d.ts new file mode 100644 index 000000000..af4369397 --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtesmthis/dist/index.d.ts @@ -0,0 +1,2 @@ +export declare function describe(): string; +export declare function load(): string; diff --git a/tests/fixtures/npm/node_modules/gtesmthis/dist/index.js b/tests/fixtures/npm/node_modules/gtesmthis/dist/index.js new file mode 100644 index 000000000..46b81aa7d --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtesmthis/dist/index.js @@ -0,0 +1,94 @@ +var __create = Object.create; +var __defProp = Object.defineProperty; +var __getOwnPropDesc = Object.getOwnPropertyDescriptor; +var __getOwnPropNames = Object.getOwnPropertyNames; +var __getProtoOf = Object.getPrototypeOf; +var __hasOwnProp = Object.prototype.hasOwnProperty; +var __esm = (fn, res) => function __init() { + return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res; +}; +var __export = (target, all) => { + for (var name in all) + __defProp(target, name, { get: all[name], enumerable: true }); +}; +var __copyProps = (to, from, except, desc) => { + if (from && typeof from === "object" || typeof from === "function") { + for (let key of __getOwnPropNames(from)) + if (!__hasOwnProp.call(to, key) && key !== except) + __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable }); + } + return to; +}; +var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps( + // If the importer is in node compatibility mode or this is not an ESM + // file that has been converted to a CommonJS file using a Babel- + // compatible transform (i.e. "__esModule" has not been set), then set + // "default" to the CommonJS "module.exports" for node compatibility. + isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target, + mod +)); +var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod); + +// src/helper.ts +function helper(x) { + COUNT++; + return "[" + x + "]"; +} +var COUNT; +var init_helper = __esm({ + "src/helper.ts"() { + console.log("helper init"); + COUNT = 0; + } +}); + +// src/lazy.ts +var lazy_exports = {}; +__export(lazy_exports, { + VALUE: () => VALUE, + value: () => value, + who: () => who +}); +function who() { + return [typeof this, this.__esModule, Object.getOwnPropertyNames(this).join(","), Object.keys(this).join(",")].join(" "); +} +function value() { + return VALUE + ":" + import_gtcore.default.tag + ":" + COUNT; +} +var path, import_gtcore, VALUE; +var init_lazy = __esm({ + "src/lazy.ts"() { + path = __toESM(require("path")); + import_gtcore = __toESM(require("gtcore")); + init_helper(); + console.log("lazy init"); + VALUE = helper(path.basename("/a/b.txt")); + } +}); + +// src/index.ts +var index_exports = {}; +__export(index_exports, { + describe: () => describe, + load: () => load +}); +module.exports = __toCommonJS(index_exports); + +// src/greet.ts +function greet(name) { + return "hello " + name; +} + +// src/index.ts +function describe() { + return greet("main"); +} +function load() { + const lazy = (init_lazy(), __toCommonJS(lazy_exports)); + return lazy.value() + " " + lazy.VALUE + " | " + lazy.who() + " | " + (init_lazy(), __toCommonJS(lazy_exports)).who(); +} +// Annotate the CommonJS export names for ESM import in node: +0 && (module.exports = { + describe, + load +}); diff --git a/tests/fixtures/npm/node_modules/gtesmthis/package.json b/tests/fixtures/npm/node_modules/gtesmthis/package.json new file mode 100644 index 000000000..c6344fa8c --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtesmthis/package.json @@ -0,0 +1,10 @@ +{ + "name": "gtesmthis", + "version": "1.0.0", + "description": "gtesm's esbuild 0.25.10 output shape with a lazily initialized ES module export that reads `this`, called through the module's namespace", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "dependencies": { + "gtcore": "1.0.0" + } +} diff --git a/tests/harness/npm-cases.ts b/tests/harness/npm-cases.ts index f8d346154..35a9df4a6 100644 --- a/tests/harness/npm-cases.ts +++ b/tests/harness/npm-cases.ts @@ -19,12 +19,12 @@ export function npmCases(fixturesRoot: string): NpmCase[] { return [ ...globSync(join(fixturesRoot, "npm/cases/*/main.ts")) .sort() - // 2465-2469 and 2556-2563 are the --npm-static bundler-emitted-CJS + // 2465-2469, 2556-2563 and 2567 are the --npm-static bundler-emitted-CJS // cases (npm-static.test.ts drives them with the opt-in): their // consumers exercise the static rewrite's surface (lexer-visible // names the shipped .d.ts never declares, the __toESM interop // family), so they stay out of the flagless island lane by design. - .filter((entry) => !/\/(246[5-9]|255[6-9]|256[0-3])-[^/]+\/main\.ts$/.test(entry)) + .filter((entry) => !/\/(246[5-9]|255[6-9]|256[0-37])-[^/]+\/main\.ts$/.test(entry)) .map((entry) => ({ name: entry.split("/").at(-2)!, entry, diff --git a/tests/harness/npm-static.test.ts b/tests/harness/npm-static.test.ts index 8560973cd..443c0c72d 100644 --- a/tests/harness/npm-static.test.ts +++ b/tests/harness/npm-static.test.ts @@ -76,7 +76,7 @@ async function buildStatic(entry: string, npmStatic: string[] | "auto"): Promise ...globSync( join(fixturesRoot, "npm/node_modules/cryptozoo/**/*.{js,mjs,cjs,json,d.ts}"), ).sort(), - // the bundler-emitted-CJS mini packages (cases 2465-2469, 2556-2563) + // the bundler-emitted-CJS mini packages (cases 2465-2469, 2556-2563, 2567) ...globSync(join(fixturesRoot, "npm/node_modules/gt*/**/*.{js,cjs,json}")).sort(), ]; for (const f of inputs) hash.update(f).update(readFileSync(f)); @@ -1478,6 +1478,21 @@ describe(`npm-static pilots${sanitize ? " (sanitized)" : ""}`, () => { ]); }, 180_000); + // 2567: an export that reads `this`, called through its namespace, sees + // Node's module object; the lowered namespace would differ, so the + // package stays refused. + test("a namespace call to an export that reads `this` stays refused", () => { + const entry = join(fixturesRoot, "npm/cases/2567-esm-namespace-this/main.ts"); + const { coverage } = analyze(entry, { npmStatic: ["gtesmthis", "gtcore"] }); + expect(coverage.npmStatic).toContainEqual({ + package: "gtesmthis", + status: "refused", + detail: expect.stringMatching( + /calls only of exported functions that do not read 'this'/, + ) as string, + }); + }, 120_000); + // 2562: a file with unverifiable interop that no compiled code can load // (the program imports a leaf data module and only a type from the // main entry) leaves its package static. From 1c3a405277793908948e99fe1c70720ac18ab1ec Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Mon, 19 Oct 2026 17:31:49 -0700 Subject: [PATCH 12/35] Expect the lazy-loader refusal for a namespace kept by a loader Co-Authored-By: Claude Opus 5.5 --- packages/compiler/src/frontend/npm-static-bundled-esm.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/compiler/src/frontend/npm-static-bundled-esm.test.ts b/packages/compiler/src/frontend/npm-static-bundled-esm.test.ts index 0b9ecd042..0a144036c 100644 --- a/packages/compiler/src/frontend/npm-static-bundled-esm.test.ts +++ b/packages/compiler/src/frontend/npm-static-bundled-esm.test.ts @@ -127,7 +127,7 @@ test.each([ `var init_a = __esm({ "a.ts"() { console.log(this); } });\ninit_a();`, ], [ - "stores the namespace of a bundled ES module ('a_exports') in a variable", + "memoized lazy loader 'load' returns the namespace of a bundled ES module as a value", `var a_exports = {};\n__export(a_exports, { v: () => v });\nvar v, init_a = __esm({ "a.ts"() { v = 1; } });\nvar ns, load = () => ns ??= __toESM((init_a(), __toCommonJS(a_exports)));\nload();`, ], [ From 5670bc182e53e2c0544c1038b12b0e5db01d5d9d Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Mon, 19 Oct 2026 18:21:52 -0700 Subject: [PATCH 13/35] Respell members of a function export object; name conditional exports A CommonJS module whose export object is one of its own top-level functions (semver 5/6's `exports = module.exports = SemVer`) reaches its members through that function after the statement: member reads and writes through module.exports (and, after the rebinding, every `exports` reference) become members of the function, the expando form both resolvers model, and the rebinding itself drops. Constructor functions exported whole with members attached afterwards no longer lose those members at runtime. Replacing module.exports or rebinding exports inside a branch (an environment-dependent export object, readable-stream 2.x) now names the construct instead of the generic assignment fence, and a required builtin module object is truthy in the LLVM backend. Co-Authored-By: Claude Opus 5.5 --- packages/compiler/src/backend/llvm/emitter.ts | 3 + .../src/frontend/lowering/lower-stmts.ts | 20 ++ .../frontend/npm-static-cjs-exports-alias.ts | 179 ++++++++++++++++++ packages/compiler/src/frontend/npm-static.ts | 6 +- packages/compiler/src/frontend/services.ts | 8 + .../6264-required-module-truthiness.cjs | 10 + .../cases/2567-cjs-function-exports/main.ts | 12 ++ .../2568-cjs-conditional-exports/main.ts | 7 + .../npm/node_modules/gtcondexports/index.d.ts | 2 + .../npm/node_modules/gtcondexports/index.js | 14 ++ .../node_modules/gtcondexports/package.json | 7 + .../npm/node_modules/gtfnexports/index.d.ts | 13 ++ .../npm/node_modules/gtfnexports/index.js | 25 +++ .../npm/node_modules/gtfnexports/package.json | 7 + .../npm/node_modules/gtfnmembers/index.d.ts | 11 ++ .../npm/node_modules/gtfnmembers/index.js | 13 ++ .../npm/node_modules/gtfnmembers/package.json | 7 + tests/harness/npm-cases.ts | 4 +- tests/harness/npm-static.test.ts | 28 ++- 19 files changed, 372 insertions(+), 4 deletions(-) create mode 100644 packages/compiler/src/frontend/npm-static-cjs-exports-alias.ts create mode 100644 tests/corpus/6264-required-module-truthiness.cjs create mode 100644 tests/fixtures/npm/cases/2567-cjs-function-exports/main.ts create mode 100644 tests/fixtures/npm/cases/2568-cjs-conditional-exports/main.ts create mode 100644 tests/fixtures/npm/node_modules/gtcondexports/index.d.ts create mode 100644 tests/fixtures/npm/node_modules/gtcondexports/index.js create mode 100644 tests/fixtures/npm/node_modules/gtcondexports/package.json create mode 100644 tests/fixtures/npm/node_modules/gtfnexports/index.d.ts create mode 100644 tests/fixtures/npm/node_modules/gtfnexports/index.js create mode 100644 tests/fixtures/npm/node_modules/gtfnexports/package.json create mode 100644 tests/fixtures/npm/node_modules/gtfnmembers/index.d.ts create mode 100644 tests/fixtures/npm/node_modules/gtfnmembers/index.js create mode 100644 tests/fixtures/npm/node_modules/gtfnmembers/package.json diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 9e2d3524e..f20c63b1d 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -5882,6 +5882,9 @@ export class LlEmitter { // for stdin. They remain JavaScript objects in boolean contexts. case "procStream": return "true"; + // A module namespace (a required builtin module object) is an object. + case "moduleNs": + return "true"; case "array": case "record": case "object": diff --git a/packages/compiler/src/frontend/lowering/lower-stmts.ts b/packages/compiler/src/frontend/lowering/lower-stmts.ts index 36abb9130..4a6fb15a3 100644 --- a/packages/compiler/src/frontend/lowering/lower-stmts.ts +++ b/packages/compiler/src/frontend/lowering/lower-stmts.ts @@ -30,6 +30,7 @@ import { import { cjsExportMemberWriteGlobalOf, isCjsExportsIdentifier, + isCjsModuleExports, isFreshExportsReset, } from "./cjs-exports-object.js"; import { isDeadTscHelperDecl, tscInteropBindingOfDecl } from "./cjs-tsc-interop.js"; @@ -6627,6 +6628,25 @@ export function lowerExprStatement(lowerer: Lowerer, expr: ts.Expression): IrStm "require() outside the module's top level (move it to the top of the file)", ); } + // Replacing the export object (or rebinding `exports`) anywhere but a + // top-level statement makes the module's exports depend on which path + // ran (`cond ? module.exports = a : module.exports = b`, an + // environment-dependent branch); the static export table has one shape. + if ( + !topLevelJs && + isJsSourceFile(expr.getSourceFile()) && + ts.isBinaryExpression(expr) && + expr.operatorToken.kind === ts.SyntaxKind.EqualsToken && + (isCjsModuleExports(lowerer, expr.left) || isCjsExportsIdentifier(lowerer, expr.left)) + ) { + lowerer.unsupported( + "SC1090", + expr.left, + isCjsModuleExports(lowerer, expr.left) + ? "replacing module.exports outside a top-level statement (a conditional or environment-dependent export object)" + : "rebinding 'exports' outside a top-level statement (a conditional or environment-dependent export object)", + ); + } if (topLevelJs && ts.isExpressionStatement(stmtNode)) { if (isFreshExportsReset(lowerer, stmtNode)) return { kind: "block", body: [], loc: locOf(expr) }; diff --git a/packages/compiler/src/frontend/npm-static-cjs-exports-alias.ts b/packages/compiler/src/frontend/npm-static-cjs-exports-alias.ts new file mode 100644 index 000000000..c88c9949e --- /dev/null +++ b/packages/compiler/src/frontend/npm-static-cjs-exports-alias.ts @@ -0,0 +1,179 @@ +/* CommonJS modules whose export object is one of their own functions: + * + * exports = module.exports = SemVer; // or module.exports = SemVer; + * exports.parse = parse; // or module.exports.parse = … + * function SemVer(version) { … } + * + * After that statement `module.exports` (and, after the rebinding form, + * `exports`) IS the function, so every member read or write through them + * is a member of the function itself. The pass respells those accesses + * through the function's name, the expando form both resolvers model + * (members on a function declaration), and drops the `exports =` + * rebinding the static export protocol has no storage for. + * + * Exact only while nothing can observe the module before the statement + * runs and nothing replaces the export object again: the statement is a + * top-level statement preceded only by inert declarations, it is the + * file's only write of `module.exports` and `exports`, the function is a + * top-level declaration nothing reassigns, and `exports`/`module` are the + * CommonJS globals (never declared in the file). Other shapes answer + * null and keep their existing treatment. */ +import * as ts from "./ts7/syntax.js"; +import { + applyEdits, + declaredNames, + isWritten, + references, + unparen, + type Edit, +} from "./npm-static-bundled-chunks.js"; + +function isModuleExports(e: ts.Expression): e is ts.PropertyAccessExpression { + const x = unparen(e); + return ( + ts.isPropertyAccessExpression(x) && + !x.questionDotToken && + ts.isIdentifier(x.expression) && + x.expression.text === "module" && + x.name.text === "exports" + ); +} + +/** True for a statement whose evaluation runs no code that could observe + * the export object: declarations and member writes (`F.prototype.m = + * function …`) without calls, and directives. */ +function inert(stmt: ts.Statement): boolean { + if (ts.isFunctionDeclaration(stmt) || ts.isEmptyStatement(stmt)) return true; + if (ts.isExpressionStatement(stmt) && ts.isStringLiteral(stmt.expression)) return true; + if (ts.isExpressionStatement(stmt)) { + const e = unparen(stmt.expression); + if ( + !ts.isBinaryExpression(e) || + e.operatorToken.kind !== ts.SyntaxKind.EqualsToken || + !(ts.isPropertyAccessExpression(e.left) || ts.isElementAccessExpression(e.left)) + ) + return false; + let root: ts.Expression = e.left; + while (ts.isPropertyAccessExpression(root) || ts.isElementAccessExpression(root)) + root = root.expression; + if (!ts.isIdentifier(root) || root.text === "module" || root.text === "exports") return false; + } else if (!ts.isVariableStatement(stmt)) return false; + let calls = false; + ts.walkPreorder(stmt, (n) => { + if (calls) return "stop"; + if (ts.isArrowFunction(n) || ts.isFunctionExpression(n)) return "skip"; + if (ts.isCallExpression(n) || ts.isNewExpression(n) || ts.isTaggedTemplateExpression(n)) + calls = true; + return undefined; + }); + return !calls; +} + +export function normalizeCjsFunctionExports(sf: ts.SourceFile): string | null { + const source = sf.text; + if (!source.includes("module.exports")) return null; + if (!sf.fileName.endsWith(".cjs") && ts.isExternalModule(sf)) return null; + const declared = declaredNames(sf); + if (declared.has("exports") || declared.has("module")) return null; + + // The whole-export statement: `[exports =] module.exports = W;`. + let whole: { stmt: ts.ExpressionStatement; fn: string; rebinds: boolean } | null = null; + const stmts = sf.statements.nodes; + for (let i = 0; i < stmts.length; i++) { + const stmt = stmts[i]!; + if (!ts.isExpressionStatement(stmt)) continue; + const top = unparen(stmt.expression); + if (!ts.isBinaryExpression(top) || top.operatorToken.kind !== ts.SyntaxKind.EqualsToken) + continue; + let e: ts.BinaryExpression = top; + let rebinds = false; + if (ts.isIdentifier(top.left) && top.left.text === "exports") { + const inner = unparen(top.right); + if (!ts.isBinaryExpression(inner) || inner.operatorToken.kind !== ts.SyntaxKind.EqualsToken) + continue; + rebinds = true; + e = inner; + } + if (!isModuleExports(e.left)) continue; + const value = unparen(e.right); + if (!ts.isIdentifier(value)) return null; + if (whole !== null || !stmts.slice(0, i).every(inert)) return null; + whole = { stmt, fn: value.text, rebinds }; + } + if (whole === null) return null; + const fnName = whole.fn; + const fnDecls = stmts.filter( + (st) => ts.isFunctionDeclaration(st) && st.name?.text === fnName && st.body !== undefined, + ); + if (fnDecls.length !== 1) return null; + + const edits: Edit[] = []; + let unsafe = false; + const refs = references(sf); + for (const r of refs) { + if (r.text === fnName && isWritten(r)) unsafe = true; + if (r.text === "exports") { + if (isWritten(r)) { + // Only the whole statement's own rebinding. + const assign = r.parent; + if ( + !whole.rebinds || + assign === undefined || + !ts.isBinaryExpression(assign) || + assign.parent !== whole.stmt || + assign.left !== r + ) + unsafe = true; + continue; + } + // Without the rebinding, `exports` keeps the replaced object. + if (!whole.rebinds || r.getStart(sf) < whole.stmt.getEnd()) { + unsafe = true; + continue; + } + edits.push({ start: r.getStart(sf), end: r.getEnd(), text: fnName }); + continue; + } + if (r.text === "module") { + const access = r.parent; + if (access === undefined || !isModuleExports(access as ts.Expression)) continue; + const outer = access.parent; + if (outer !== undefined && ts.isBinaryExpression(outer) && outer.left === access) { + if ( + outer.operatorToken.kind >= ts.SyntaxKind.FirstAssignment && + outer.operatorToken.kind <= ts.SyntaxKind.LastAssignment && + !( + (outer.parent === whole.stmt && !whole.rebinds) || + (whole.rebinds && outer.parent?.parent === whole.stmt) + ) + ) + unsafe = true; + continue; + } + // A member access through module.exports: a member of the function. + if ( + outer !== undefined && + (ts.isPropertyAccessExpression(outer) || ts.isElementAccessExpression(outer)) && + outer.expression === access + ) { + if (access.getStart(sf) < whole.stmt.getEnd() && !insideFunction(access)) unsafe = true; + edits.push({ start: access.getStart(sf), end: access.getEnd(), text: fnName }); + } + } + } + if (unsafe) return null; + if (whole.rebinds) { + // `exports = module.exports = W` → `module.exports = W`. + const e = unparen(whole.stmt.expression) as ts.BinaryExpression; + edits.push({ start: e.getStart(sf), end: unparen(e.right).getStart(sf), text: "" }); + } + if (edits.length === 0) return null; + return applyEdits(source, 0, source.length, edits); +} + +function insideFunction(node: ts.Node): boolean { + for (let p = node.parent; p !== undefined && !ts.isSourceFile(p); p = p.parent) { + if (ts.isFunctionLike(p)) return true; + } + return false; +} diff --git a/packages/compiler/src/frontend/npm-static.ts b/packages/compiler/src/frontend/npm-static.ts index e00122238..182c5b62b 100644 --- a/packages/compiler/src/frontend/npm-static.ts +++ b/packages/compiler/src/frontend/npm-static.ts @@ -448,7 +448,11 @@ export function npmStaticFsShadow(services: FrontendServices): NpmStaticFsShadow } else if (esm !== null) { original = esm; } - const respelled = split !== null || esm !== null; + // A module exporting one of its functions reaches its members + // through that function. + const fnExports = original === null ? null : services.cjsFunctionExports(original, path); + if (fnExports !== null) original = fnExports; + const respelled = split !== null || esm !== null || fnExports !== null; if (original !== null) { const namepaths = services.jsDocNamepaths(path, original); const source = namepaths ?? original; diff --git a/packages/compiler/src/frontend/services.ts b/packages/compiler/src/frontend/services.ts index c5dcbf513..2e3d686ab 100644 --- a/packages/compiler/src/frontend/services.ts +++ b/packages/compiler/src/frontend/services.ts @@ -2,6 +2,7 @@ import { Ts7SourceParser, type Ts7SourceKind } from "./ts7/source-parser.js"; import { Ts7Host, type Ts7ApiFactory, type Ts7HostOptions } from "./ts7/program-host.js"; import { lowerEsmInitsWithParser, rewriteBundlerCjsWithParser } from "./npm-static-rewrite-host.js"; import { isBundlerCjsCandidate } from "./npm-static-rewrite-syntax.js"; +import { normalizeCjsFunctionExports } from "./npm-static-cjs-exports-alias.js"; import { mayBundleChunks, splitBundledChunks, @@ -91,6 +92,13 @@ export class FrontendServices { if (!source.includes("__esm")) return null; return lowerEsmInitsWithParser(this.sourceParser(), source, path); } + /** A CommonJS module whose export object is one of its functions, with + * its member accesses respelled through that function (null: none). */ + cjsFunctionExports(source: string, path: string): string | null { + this.ensureOpen(); + if (!source.includes("module.exports")) return null; + return normalizeCjsFunctionExports(this.parse(path, source, "js")); + } nullableClassFields(path: string, source: string): syntax.NpmStaticOverloadRewrite | null { return syntax.applyNpmStaticNullableClassFields(this.parse(path, source, "js"), source); } diff --git a/tests/corpus/6264-required-module-truthiness.cjs b/tests/corpus/6264-required-module-truthiness.cjs new file mode 100644 index 000000000..626d1cd4d --- /dev/null +++ b/tests/corpus/6264-required-module-truthiness.cjs @@ -0,0 +1,10 @@ +// A required builtin module object in a boolean context is truthy, as any +// object: readable-stream's `process.env.READABLE_STREAM === "disable" && +// Stream` and `Stream || exports` guards. +"use strict"; +var Stream = require("stream"); +var os = require("os"); +console.log(Stream ? "stream" : "none", !!os, !os); +console.log(process.env.SCRIPTC_UNSET_FLAG === "disable" && Stream ? "disabled" : "default"); +var fallback = Stream || null; +console.log(fallback === Stream); diff --git a/tests/fixtures/npm/cases/2567-cjs-function-exports/main.ts b/tests/fixtures/npm/cases/2567-cjs-function-exports/main.ts new file mode 100644 index 000000000..29e65cbfc --- /dev/null +++ b/tests/fixtures/npm/cases/2567-cjs-function-exports/main.ts @@ -0,0 +1,12 @@ +// CommonJS modules whose export object is one of their own functions: +// semver 5/6's `exports = module.exports = Version` with members attached +// through both `exports` and `module.exports` (and read back through +// `exports` inside the module), and a constructor function exported whole +// with function and value members attached afterwards. +import Version from "gtfnexports"; +import Point from "gtfnmembers"; + +const v = Version.parse("2.4.1"); +console.log(v === null ? "null" : v.major(), String(new Version("3.0.0"))); +console.log(Version.compare("5.0.0", "3.2.0"), Version.gt("1.0.0", "2.0.0"), Version.MAX_LENGTH); +console.log(Point.origin().sum(), new Point(2, 3).sum(), Point.dimensions); diff --git a/tests/fixtures/npm/cases/2568-cjs-conditional-exports/main.ts b/tests/fixtures/npm/cases/2568-cjs-conditional-exports/main.ts new file mode 100644 index 000000000..2c117e914 --- /dev/null +++ b/tests/fixtures/npm/cases/2568-cjs-conditional-exports/main.ts @@ -0,0 +1,7 @@ +// An environment-dependent export object (readable-stream 2.x's +// READABLE_STREAM=disable switch): module.exports is replaced inside an +// if/else, so the module's exports depend on which branch ran. scriptc +// names the construct; Node picks the branch at load. +import rs from "gtcondexports"; + +console.log(rs.Readable.name); diff --git a/tests/fixtures/npm/node_modules/gtcondexports/index.d.ts b/tests/fixtures/npm/node_modules/gtcondexports/index.d.ts new file mode 100644 index 000000000..ba94e150b --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtcondexports/index.d.ts @@ -0,0 +1,2 @@ +declare const exported: { Readable: Function; Stream: unknown; name: string }; +export = exported; diff --git a/tests/fixtures/npm/node_modules/gtcondexports/index.js b/tests/fixtures/npm/node_modules/gtcondexports/index.js new file mode 100644 index 000000000..b4017d055 --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtcondexports/index.js @@ -0,0 +1,14 @@ +var Stream = require("stream"); +function Local(name) { + this.name = name; +} +if (process.env.GT_STREAMS === "native" && Stream) { + module.exports = Stream; + exports = module.exports = Stream.Readable; + exports.Readable = Stream.Readable; + exports.Stream = Stream; +} else { + exports = module.exports = Local; + exports.Stream = Stream || exports; + exports.Readable = exports; +} diff --git a/tests/fixtures/npm/node_modules/gtcondexports/package.json b/tests/fixtures/npm/node_modules/gtcondexports/package.json new file mode 100644 index 000000000..eab4647ed --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtcondexports/package.json @@ -0,0 +1,7 @@ +{ + "name": "gtcondexports", + "version": "1.0.0", + "description": "readable-stream 2.x's environment-dependent export object: module.exports is replaced (and exports rebound) inside an if/else on an environment variable", + "main": "index.js", + "types": "index.d.ts" +} diff --git a/tests/fixtures/npm/node_modules/gtfnexports/index.d.ts b/tests/fixtures/npm/node_modules/gtfnexports/index.d.ts new file mode 100644 index 000000000..c2ee082d3 --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtfnexports/index.d.ts @@ -0,0 +1,13 @@ +declare class Version { + constructor(raw: string); + raw: string; + major(): number; + toString(): string; +} +declare namespace Version { + const MAX_LENGTH: number; + function parse(raw: string): Version | null; + function compare(a: string, b: string): number; + function gt(a: string, b: string): boolean; +} +export = Version; diff --git a/tests/fixtures/npm/node_modules/gtfnexports/index.js b/tests/fixtures/npm/node_modules/gtfnexports/index.js new file mode 100644 index 000000000..d64b58f0c --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtfnexports/index.js @@ -0,0 +1,25 @@ +exports = module.exports = Version; + +var label = process.env.GT_LABEL ? "[" + process.env.GT_LABEL + "]" : ""; + +function Version(raw) { + this.raw = raw; + this.parts = raw.split("."); +} +Version.prototype.major = function () { + return Number(this.parts[0]); +}; +Version.prototype.toString = function () { + return label + this.raw; +}; + +exports.MAX_LENGTH = 256; +exports.parse = function parse(raw) { + return raw.length > exports.MAX_LENGTH ? null : new Version(raw); +}; +exports.compare = function compare(a, b) { + return exports.parse(a).major() - exports.parse(b).major(); +}; +module.exports.gt = function gt(a, b) { + return exports.compare(a, b) > 0; +}; diff --git a/tests/fixtures/npm/node_modules/gtfnexports/package.json b/tests/fixtures/npm/node_modules/gtfnexports/package.json new file mode 100644 index 000000000..4524cd1bc --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtfnexports/package.json @@ -0,0 +1,7 @@ +{ + "name": "gtfnexports", + "version": "1.0.0", + "description": "semver 5/6's export idiom: exports = module.exports = Version, then members attached through exports and module.exports", + "main": "index.js", + "types": "index.d.ts" +} diff --git a/tests/fixtures/npm/node_modules/gtfnmembers/index.d.ts b/tests/fixtures/npm/node_modules/gtfnmembers/index.d.ts new file mode 100644 index 000000000..ea1888e0b --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtfnmembers/index.d.ts @@ -0,0 +1,11 @@ +declare class Point { + constructor(x: number, y: number); + x: number; + y: number; + sum(): number; +} +declare namespace Point { + function origin(): Point; + const dimensions: number; +} +export = Point; diff --git a/tests/fixtures/npm/node_modules/gtfnmembers/index.js b/tests/fixtures/npm/node_modules/gtfnmembers/index.js new file mode 100644 index 000000000..6eaf003f7 --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtfnmembers/index.js @@ -0,0 +1,13 @@ +"use strict"; +function Point(x, y) { + this.x = x; + this.y = y; +} +Point.prototype.sum = function () { + return this.x + this.y; +}; +module.exports = Point; +module.exports.origin = function origin() { + return new Point(0, 0); +}; +module.exports.dimensions = 2; diff --git a/tests/fixtures/npm/node_modules/gtfnmembers/package.json b/tests/fixtures/npm/node_modules/gtfnmembers/package.json new file mode 100644 index 000000000..5179c6caf --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtfnmembers/package.json @@ -0,0 +1,7 @@ +{ + "name": "gtfnmembers", + "version": "1.0.0", + "description": "a constructor function as module.exports with function and value members attached through module.exports afterwards", + "main": "index.js", + "types": "index.d.ts" +} diff --git a/tests/harness/npm-cases.ts b/tests/harness/npm-cases.ts index 69ce3917f..e0f1384bb 100644 --- a/tests/harness/npm-cases.ts +++ b/tests/harness/npm-cases.ts @@ -19,12 +19,12 @@ export function npmCases(fixturesRoot: string): NpmCase[] { return [ ...globSync(join(fixturesRoot, "npm/cases/*/main.ts")) .sort() - // 2465-2469 and 2556-2566 are the --npm-static bundler-emitted-CJS + // 2465-2469 and 2556-2568 are the --npm-static bundler-emitted-CJS // cases (npm-static.test.ts drives them with the opt-in): their // consumers exercise the static rewrite's surface (lexer-visible // names the shipped .d.ts never declares, the __toESM interop // family), so they stay out of the flagless island lane by design. - .filter((entry) => !/\/(246[5-9]|255[6-9]|256[0-6])-[^/]+\/main\.ts$/.test(entry)) + .filter((entry) => !/\/(246[5-9]|255[6-9]|256[0-8])-[^/]+\/main\.ts$/.test(entry)) .map((entry) => ({ name: entry.split("/").at(-2)!, entry, diff --git a/tests/harness/npm-static.test.ts b/tests/harness/npm-static.test.ts index 3543aeb0a..02ddadb0d 100644 --- a/tests/harness/npm-static.test.ts +++ b/tests/harness/npm-static.test.ts @@ -76,7 +76,7 @@ async function buildStatic(entry: string, npmStatic: string[] | "auto"): Promise ...globSync( join(fixturesRoot, "npm/node_modules/cryptozoo/**/*.{js,mjs,cjs,json,d.ts}"), ).sort(), - // the bundler-emitted-CJS mini packages (cases 2465-2469, 2556-2566) + // the bundler-emitted-CJS mini packages (cases 2465-2469, 2556-2568) ...globSync(join(fixturesRoot, "npm/node_modules/gt*/**/*.{js,cjs,json}")).sort(), ...globSync(join(fixturesRoot, "npm/store/gt*/**/*.{js,cjs,json}")).sort(), ]; @@ -1514,6 +1514,32 @@ describe(`npm-static pilots${sanitize ? " (sanitized)" : ""}`, () => { expect(nativeRes.exitCode).toBe(nodeRes.exitCode); }, 180_000); + // 2567: a module whose export object is one of its functions (semver + // 5/6's `exports = module.exports = Version`), under both values of the + // environment variable the module reads. + test("function export objects 2567-cjs-function-exports compile statically and byte-match Node", async () => { + const entry = join(fixturesRoot, "npm/cases/2567-cjs-function-exports/main.ts"); + await expectNodeParity(entry, ["gtfnexports", "gtfnmembers"]); + const binary = await buildStatic(entry, ["gtfnexports", "gtfnmembers"]); + const env = { ...process.env, GT_LABEL: "x" }; + const [nodeRes, nativeRes] = await Promise.all([ + runBinary("node", [entry], env), + runBinary(binary, [], env), + ]); + expect(nativeRes.stdout.toString("utf8")).toBe(nodeRes.stdout.toString("utf8")); + expect(nativeRes.stdout.toString("utf8")).toContain("[x]"); + }, 180_000); + + // 2568: an export object replaced inside an environment-dependent branch + // (readable-stream 2.x) is named precisely. + test("an environment-dependent export object is refused by name", () => { + const entry = join(fixturesRoot, "npm/cases/2568-cjs-conditional-exports/main.ts"); + const { coverage } = analyze(entry, { npmStatic: ["gtcondexports"] }); + expect(JSON.stringify(coverage)).toContain( + "replacing module.exports outside a top-level statement (a conditional or environment-dependent export object)", + ); + }, 120_000); + // 2562: a file with unverifiable interop that no compiled code can load // (the program imports a leaf data module and only a type from the // main entry) leaves its package static. From 3723e05417ec5229afc23cb861da81251e26586c Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Mon, 19 Oct 2026 18:28:45 -0700 Subject: [PATCH 14/35] Resolve bundled chunks that re-export a builtin to the builtin A chunk whose whole body is `module.exports = require("")` (readable-stream's internal/streams/stream.js) is the builtin module: its calls become that require, and no chunk module is emitted. Co-Authored-By: Claude Opus 5.5 --- .../npm-static-bundled-chunks.test.ts | 18 ++++++++ .../src/frontend/npm-static-bundled-chunks.ts | 44 ++++++++++++++++++- 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/packages/compiler/src/frontend/npm-static-bundled-chunks.test.ts b/packages/compiler/src/frontend/npm-static-bundled-chunks.test.ts index 82926d2b8..7ccad33f0 100644 --- a/packages/compiler/src/frontend/npm-static-bundled-chunks.test.ts +++ b/packages/compiler/src/frontend/npm-static-bundled-chunks.test.ts @@ -182,3 +182,21 @@ export const g = import_g.g; degrade: expect.stringContaining("bundled CommonJS modules") as string, }); }); + +test("a chunk that only re-exports a builtin module becomes that module", () => { + const answer = expectSameBehavior(`${keyedHelper}var require_stream = __commonJS({ + "lib/stream.js"(exports2, module2) { + module2.exports = require("stream"); + } +}); +var require_user = __commonJS({ + "lib/user.js"(exports2) { + var Stream = require_stream(); + exports2.same = Stream === require("stream"); + } +}); +console.log(typeof require_stream().PassThrough, require_user().same); +`); + expect(answer.chunks.map((c) => basename(c.path))).toEqual(["index.js.require_user.cjs"]); + expect(answer.text).toContain('typeof require("stream").PassThrough'); +}); diff --git a/packages/compiler/src/frontend/npm-static-bundled-chunks.ts b/packages/compiler/src/frontend/npm-static-bundled-chunks.ts index 28fe4a1b9..03b639cbe 100644 --- a/packages/compiler/src/frontend/npm-static-bundled-chunks.ts +++ b/packages/compiler/src/frontend/npm-static-bundled-chunks.ts @@ -23,6 +23,7 @@ * beside it). Anything else answers a degrade reason. */ import * as ts from "./ts7/syntax.js"; import { helperTokens } from "./helper-tokens.js"; +import { canonicalBuiltinModule } from "./builtin-modules.js"; export interface BundledChunkSplit { /** The bundle with chunk declarations blanked and calls respelled. */ @@ -407,10 +408,11 @@ export function splitBundledChunks( return { degrade: `its bundled CommonJS module factory '${r.text}' is used other than as a direct call`, }; + const forwarded = forwarderSpec(chunk); callEdits.push({ start: call.getStart(sf), end: call.getEnd(), - text: `require(${JSON.stringify(chunk.spec)})`, + text: `require(${JSON.stringify(forwarded ?? chunk.spec)})`, }); } for (const name of chunks.keys()) { @@ -422,6 +424,11 @@ export function splitBundledChunks( const out: BundledChunkSplit["chunks"] = []; const bundleEdits: Edit[] = [...callEdits]; for (const chunk of chunks.values()) { + if (forwarderSpec(chunk) !== null) { + // A chunk that only re-exports a builtin module is that module. + bundleEdits.push({ start: chunk.stmt.getStart(sf), end: chunk.stmt.getEnd(), text: "" }); + continue; + } const result = chunkSource(sf, chunk, chunks, top, stable, callEdits, bundleStrict); if (typeof result !== "string") return result; out.push({ path: chunk.path, source: result }); @@ -472,6 +479,41 @@ export function applyEdits(source: string, from: number, to: number, edits: Edit return text + source.slice(at, to); } +/** The builtin module a chunk forwards, or null: a body that is exactly + * `module.exports = require("")` (readable-stream's + * internal/streams/stream.js). Requiring the chunk is then requiring the + * builtin itself (same object, and loading a builtin runs no code). */ +function forwarderSpec(chunk: Chunk): string | null { + const moduleParam = chunk.params[1]; + if (moduleParam === undefined) return null; + const stmts = chunk.body.statements.nodes.filter( + (st) => !(ts.isExpressionStatement(st) && ts.isStringLiteral(st.expression)), + ); + const only = stmts[0]; + if (stmts.length !== 1 || only === undefined || !ts.isExpressionStatement(only)) return null; + const e = unparen(only.expression); + if ( + !ts.isBinaryExpression(e) || + e.operatorToken.kind !== ts.SyntaxKind.EqualsToken || + !ts.isPropertyAccessExpression(e.left) || + !ts.isIdentifier(e.left.expression) || + e.left.expression.text !== moduleParam || + e.left.name.text !== "exports" + ) + return null; + const call = unparen(e.right); + if ( + !ts.isCallExpression(call) || + !ts.isIdentifier(call.expression) || + call.expression.text !== "require" || + call.arguments.nodes.length !== 1 || + !ts.isStringLiteral(call.arguments.nodes[0]!) + ) + return null; + const spec = (call.arguments.nodes[0] as ts.StringLiteral).text; + return canonicalBuiltinModule(spec) !== null ? spec : null; +} + /** How code in a factory body (a chunk or a lazy ES-module initializer) * observes the factory's own frame, or null: its receiver, its arguments * object, `super`, a meta property, or (when `returns`) a top-level From 2b7c892af3645f871e55aaecd590d42395ca2185 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Mon, 19 Oct 2026 18:29:54 -0700 Subject: [PATCH 15/35] Record order baselines for the new cases and corpus program Co-Authored-By: Claude Opus 5.5 --- .../tests/corpus/6264-required-module-truthiness.cjs.json | 6 ++++++ .../npm/cases/2567-cjs-function-exports/main.ts.json | 6 ++++++ .../npm/cases/2568-cjs-conditional-exports/main.ts.json | 6 ++++++ 3 files changed, 18 insertions(+) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6264-required-module-truthiness.cjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2567-cjs-function-exports/main.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2568-cjs-conditional-exports/main.ts.json diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6264-required-module-truthiness.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6264-required-module-truthiness.cjs.json new file mode 100644 index 000000000..2fa691953 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6264-required-module-truthiness.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6264-required-module-truthiness.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2567-cjs-function-exports/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2567-cjs-function-exports/main.ts.json new file mode 100644 index 000000000..63a9b7efc --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2567-cjs-function-exports/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/2567-cjs-function-exports/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2568-cjs-conditional-exports/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2568-cjs-conditional-exports/main.ts.json new file mode 100644 index 000000000..9cad58f88 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2568-cjs-conditional-exports/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/2568-cjs-conditional-exports/main.ts" + ], + "diags": [] +} From fa012f5da3ecc2fd5e818164678bdea271cb0e0f Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Mon, 19 Oct 2026 19:51:26 -0700 Subject: [PATCH 16/35] Name the corpus programs instead of numbering them The numbers collided with other open changes. Co-Authored-By: Claude Opus 5.5 --- .../tests/corpus/6260-local-class-detached-method.ts.json | 6 ------ .../tests/corpus/6262-comma-null-union-return.cjs.json | 6 ------ .../tests/corpus/6263-strict-arguments-parameters.cjs.json | 6 ------ .../tests/corpus/comma-null-union-return.cjs.json | 6 ++++++ .../tests/corpus/local-class-detached-method.ts.json | 6 ++++++ .../tests/corpus/strict-arguments-parameters.cjs.json | 6 ++++++ ...ma-null-union-return.cjs => comma-null-union-return.cjs} | 0 ...ss-detached-method.ts => local-class-detached-method.ts} | 0 ...ments-parameters.cjs => strict-arguments-parameters.cjs} | 0 9 files changed, 18 insertions(+), 18 deletions(-) delete mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6260-local-class-detached-method.ts.json delete mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6262-comma-null-union-return.cjs.json delete mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6263-strict-arguments-parameters.cjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/comma-null-union-return.cjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/local-class-detached-method.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/strict-arguments-parameters.cjs.json rename tests/corpus/{6262-comma-null-union-return.cjs => comma-null-union-return.cjs} (100%) rename tests/corpus/{6260-local-class-detached-method.ts => local-class-detached-method.ts} (100%) rename tests/corpus/{6263-strict-arguments-parameters.cjs => strict-arguments-parameters.cjs} (100%) diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6260-local-class-detached-method.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6260-local-class-detached-method.ts.json deleted file mode 100644 index d73c701d7..000000000 --- a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6260-local-class-detached-method.ts.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "order": [ - "/tests/corpus/6260-local-class-detached-method.ts" - ], - "diags": [] -} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6262-comma-null-union-return.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6262-comma-null-union-return.cjs.json deleted file mode 100644 index e167cc977..000000000 --- a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6262-comma-null-union-return.cjs.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "order": [ - "/tests/corpus/6262-comma-null-union-return.cjs" - ], - "diags": [] -} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6263-strict-arguments-parameters.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6263-strict-arguments-parameters.cjs.json deleted file mode 100644 index b7ff6a960..000000000 --- a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6263-strict-arguments-parameters.cjs.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "order": [ - "/tests/corpus/6263-strict-arguments-parameters.cjs" - ], - "diags": [] -} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/comma-null-union-return.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/comma-null-union-return.cjs.json new file mode 100644 index 000000000..1fad3c559 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/comma-null-union-return.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/comma-null-union-return.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/local-class-detached-method.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/local-class-detached-method.ts.json new file mode 100644 index 000000000..6d311fce9 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/local-class-detached-method.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/local-class-detached-method.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/strict-arguments-parameters.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/strict-arguments-parameters.cjs.json new file mode 100644 index 000000000..8c9cc9279 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/strict-arguments-parameters.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/strict-arguments-parameters.cjs" + ], + "diags": [] +} diff --git a/tests/corpus/6262-comma-null-union-return.cjs b/tests/corpus/comma-null-union-return.cjs similarity index 100% rename from tests/corpus/6262-comma-null-union-return.cjs rename to tests/corpus/comma-null-union-return.cjs diff --git a/tests/corpus/6260-local-class-detached-method.ts b/tests/corpus/local-class-detached-method.ts similarity index 100% rename from tests/corpus/6260-local-class-detached-method.ts rename to tests/corpus/local-class-detached-method.ts diff --git a/tests/corpus/6263-strict-arguments-parameters.cjs b/tests/corpus/strict-arguments-parameters.cjs similarity index 100% rename from tests/corpus/6263-strict-arguments-parameters.cjs rename to tests/corpus/strict-arguments-parameters.cjs From c1a9341739c469c1f32ed7eb07b6aa20555c4364 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Mon, 19 Oct 2026 19:51:39 -0700 Subject: [PATCH 17/35] Name the module truthiness corpus program instead of numbering it Its number collided with another open change. Co-Authored-By: Claude Opus 5.5 --- .../tests/corpus/6264-required-module-truthiness.cjs.json | 6 ------ .../tests/corpus/required-module-truthiness.cjs.json | 6 ++++++ ...module-truthiness.cjs => required-module-truthiness.cjs} | 0 3 files changed, 6 insertions(+), 6 deletions(-) delete mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6264-required-module-truthiness.cjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/required-module-truthiness.cjs.json rename tests/corpus/{6264-required-module-truthiness.cjs => required-module-truthiness.cjs} (100%) diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6264-required-module-truthiness.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6264-required-module-truthiness.cjs.json deleted file mode 100644 index 2fa691953..000000000 --- a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6264-required-module-truthiness.cjs.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "order": [ - "/tests/corpus/6264-required-module-truthiness.cjs" - ], - "diags": [] -} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/required-module-truthiness.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/required-module-truthiness.cjs.json new file mode 100644 index 000000000..101bb2d57 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/required-module-truthiness.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/required-module-truthiness.cjs" + ], + "diags": [] +} diff --git a/tests/corpus/6264-required-module-truthiness.cjs b/tests/corpus/required-module-truthiness.cjs similarity index 100% rename from tests/corpus/6264-required-module-truthiness.cjs rename to tests/corpus/required-module-truthiness.cjs From d07188d250b987fc65c5ee38f86ab0cc6cb7cf56 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Mon, 19 Oct 2026 22:00:59 -0500 Subject: [PATCH 18/35] Merge base --- .../npm-static-bundled-chunks.test.ts | 92 ++++++++ .../src/frontend/npm-static-bundled-chunks.ts | 200 +++++++++++++++++- packages/compiler/src/frontend/program.ts | 110 ++++++++-- .../2564-chunk-helper-shadowing/main.ts.json | 6 + .../2565-chunk-shared-function/main.ts.json | 6 + .../2566-cjs-cycle-indirect-call/main.ts.json | 6 + .../main.ts.json | 6 + .../cases/2564-chunk-helper-shadowing/main.ts | 7 + .../cases/2565-chunk-shared-function/main.ts | 7 + .../2566-cjs-cycle-indirect-call/main.ts | 7 + .../2571-cjs-cycle-stored-function/main.ts | 8 + .../gtchunkshadow/dist/index.d.ts | 2 + .../node_modules/gtchunkshadow/dist/index.js | 52 +++++ .../node_modules/gtchunkshadow/package.json | 7 + .../gtchunkshared/dist/index.d.ts | 2 + .../node_modules/gtchunkshared/dist/index.js | 56 +++++ .../node_modules/gtchunkshared/package.json | 7 + .../node_modules/gtcyclealias/dist/index.d.ts | 3 + .../node_modules/gtcyclealias/dist/index.js | 37 ++++ .../node_modules/gtcyclealias/dist/reader.js | 34 +++ .../node_modules/gtcyclealias/package.json | 7 + .../node_modules/gtcyclebox/dist/index.d.ts | 3 + .../npm/node_modules/gtcyclebox/dist/index.js | 37 ++++ .../node_modules/gtcyclebox/dist/reader.js | 34 +++ .../npm/node_modules/gtcyclebox/package.json | 7 + .../node_modules/gtcyclecall/dist/index.d.ts | 3 + .../node_modules/gtcyclecall/dist/index.js | 37 ++++ .../node_modules/gtcyclecall/dist/reader.js | 33 +++ .../npm/node_modules/gtcyclecall/package.json | 7 + tests/harness/npm-cases.ts | 4 +- tests/harness/npm-static.test.ts | 57 ++++- 31 files changed, 860 insertions(+), 24 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2564-chunk-helper-shadowing/main.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2565-chunk-shared-function/main.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2566-cjs-cycle-indirect-call/main.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2571-cjs-cycle-stored-function/main.ts.json create mode 100644 tests/fixtures/npm/cases/2564-chunk-helper-shadowing/main.ts create mode 100644 tests/fixtures/npm/cases/2565-chunk-shared-function/main.ts create mode 100644 tests/fixtures/npm/cases/2566-cjs-cycle-indirect-call/main.ts create mode 100644 tests/fixtures/npm/cases/2571-cjs-cycle-stored-function/main.ts create mode 100644 tests/fixtures/npm/node_modules/gtchunkshadow/dist/index.d.ts create mode 100644 tests/fixtures/npm/node_modules/gtchunkshadow/dist/index.js create mode 100644 tests/fixtures/npm/node_modules/gtchunkshadow/package.json create mode 100644 tests/fixtures/npm/node_modules/gtchunkshared/dist/index.d.ts create mode 100644 tests/fixtures/npm/node_modules/gtchunkshared/dist/index.js create mode 100644 tests/fixtures/npm/node_modules/gtchunkshared/package.json create mode 100644 tests/fixtures/npm/node_modules/gtcyclealias/dist/index.d.ts create mode 100644 tests/fixtures/npm/node_modules/gtcyclealias/dist/index.js create mode 100644 tests/fixtures/npm/node_modules/gtcyclealias/dist/reader.js create mode 100644 tests/fixtures/npm/node_modules/gtcyclealias/package.json create mode 100644 tests/fixtures/npm/node_modules/gtcyclebox/dist/index.d.ts create mode 100644 tests/fixtures/npm/node_modules/gtcyclebox/dist/index.js create mode 100644 tests/fixtures/npm/node_modules/gtcyclebox/dist/reader.js create mode 100644 tests/fixtures/npm/node_modules/gtcyclebox/package.json create mode 100644 tests/fixtures/npm/node_modules/gtcyclecall/dist/index.d.ts create mode 100644 tests/fixtures/npm/node_modules/gtcyclecall/dist/index.js create mode 100644 tests/fixtures/npm/node_modules/gtcyclecall/dist/reader.js create mode 100644 tests/fixtures/npm/node_modules/gtcyclecall/package.json diff --git a/packages/compiler/src/frontend/npm-static-bundled-chunks.test.ts b/packages/compiler/src/frontend/npm-static-bundled-chunks.test.ts index 82926d2b8..f9463ea1e 100644 --- a/packages/compiler/src/frontend/npm-static-bundled-chunks.test.ts +++ b/packages/compiler/src/frontend/npm-static-bundled-chunks.test.ts @@ -111,6 +111,98 @@ console.log(require_named().name); expect(chunk).toContain("var __defProp = Object.defineProperty;"); }); +test("a helper travels with a chunk whose nested scopes declare its name", () => { + const answer = expectSameBehavior(`${keyedHelper}function label(n) { + return "label " + n; +} +function wrap(n) { + function inner() { + var label = "inner"; + return label; + } + return label(n) + " " + inner(); +} +var require_e = __commonJS({ + "e.js"(exports2, module2) { + function nested() { + var label = "nested"; + return label; + } + var seen = []; + for (let label of ["loop"]) seen.push(label); + try { + throw "caught"; + } catch (label) { + seen.push(label); + } + { + const label = "block"; + seen.push(label); + } + module2.exports = { outer: label(1), indirect: (0, label)(3), wrapped: wrap(2), nested: nested(), seen }; + } +}); +var e = require_e(); +console.log(e.outer, e.indirect, e.wrapped, e.nested, e.seen.join(",")); +`); + const chunk = answer.chunks[0]!.source; + expect(chunk).toContain("function label(n)"); + expect(chunk).toContain("function wrap(n)"); +}); + +test("a named function expression that only calls itself travels with a chunk", () => { + const answer = expectSameBehavior(`${keyedHelper}var fact = function self(n) { + return n <= 1 ? 1 : n * self(n - 1); +}; +var kind = function self(self) { + return typeof self; +}; +var outer = function self() { + var self = 2; + return self; +}; +var require_g = __commonJS({ + "g.js"(exports2, module2) { + module2.exports = [fact(5), kind(1), outer()]; + } +}); +console.log(require_g().join(",")); +`); + const chunk = answer.chunks[0]!.source; + expect(chunk).toContain("var fact = function self(n)"); +}); + +test.each([ + ["exports it", `function tally() {}`, `module2.exports = { tally };`], + ["reads its properties", `function tally() {}`, `exports2.calls = tally.calls;`], + ["constructs it", `function tally() {}`, `exports2.made = new tally();`], + ["stores an arrow helper", `var tally = () => 1;`, `exports2.fn = tally;`], + [ + "calls a function that reads its own name as a value", + `var tally = function self() { self.n = (self.n || 0) + 1; return self.n; };`, + `exports2.count = tally();`, + ], + [ + "calls a function whose parameter default reads its own name", + `var tally = function self(box = self) { box.n = (box.n || 0) + 1; return box.n; };`, + `exports2.count = tally();`, + ], +])("a chunk that %s keeps the package refused", (_, declaration, statement) => { + const result = split(`${keyedHelper}${declaration} +var require_f = __commonJS({ + "f.js"(exports2, module2) { + ${statement} + } +}); +require_f(); +`); + expect(result).toEqual({ + degrade: expect.stringContaining( + "uses the bundle's top-level function 'tally' as a value", + ) as string, + }); +}); + test.each([ ["reads its factory's receiver", `console.log(this);`], ["reads its factory's arguments object", `console.log(arguments.length);`], diff --git a/packages/compiler/src/frontend/npm-static-bundled-chunks.ts b/packages/compiler/src/frontend/npm-static-bundled-chunks.ts index 28fe4a1b9..2a3ad0b53 100644 --- a/packages/compiler/src/frontend/npm-static-bundled-chunks.ts +++ b/packages/compiler/src/frontend/npm-static-bundled-chunks.ts @@ -105,6 +105,128 @@ export function declaredNames(root: ts.Node): Set { return names; } +/** Adds the identifiers a binding name (an identifier or a destructuring + * pattern) declares. */ +function addBindingNames(name: ts.BindingName, out: Set): void { + if (ts.isIdentifier(name)) { + out.add(name.text); + return; + } + for (const element of name.elements.nodes) { + if (!ts.isOmittedExpression(element) && element.name !== undefined) + addBindingNames(element.name, out); + } +} + +function isFunctionScope(n: ts.Node): n is ts.SignatureDeclaration { + return ( + ts.isFunctionDeclaration(n) || + ts.isFunctionExpression(n) || + ts.isArrowFunction(n) || + ts.isMethodDeclaration(n) || + ts.isGetAccessorDeclaration(n) || + ts.isSetAccessorDeclaration(n) || + ts.isConstructorDeclaration(n) + ); +} + +/** The `var` names a function body declares, without entering nested + * functions. Function declarations in nested blocks count as block-scoped + * here; treating one as free only adds an unused helper copy. */ +function varScopedNames(body: ts.Node): Set { + const out = new Set(); + ts.walkPreorder(body, (n) => { + if (n !== body && (isFunctionScope(n) || ts.isClassDeclaration(n) || ts.isClassExpression(n))) + return "skip"; + if (ts.isVariableDeclarationList(n) && (n.flags & ts.NodeFlags.BlockScoped) === 0) { + for (const d of n.declarations.nodes) addBindingNames(d.name, out); + } + return undefined; + }); + return out; +} + +/** The names one scope node binds for the references inside it. */ +function scopeNames(scope: ts.Node): Set { + const out = new Set(); + if (isFunctionScope(scope)) { + for (const param of scope.parameters.nodes) addBindingNames(param.name, out); + if (ts.isFunctionExpression(scope) && scope.name !== undefined) out.add(scope.name.text); + if (scope.body !== undefined) for (const name of varScopedNames(scope.body)) out.add(name); + } else if (ts.isBlock(scope) || ts.isSourceFile(scope) || ts.isCaseBlock(scope)) { + const statements = ts.isCaseBlock(scope) + ? scope.clauses.nodes.flatMap((clause) => clause.statements.nodes) + : scope.statements.nodes; + for (const stmt of statements) { + if ((ts.isFunctionDeclaration(stmt) || ts.isClassDeclaration(stmt)) && stmt.name) + out.add(stmt.name.text); + else if ( + ts.isVariableStatement(stmt) && + (stmt.declarationList.flags & ts.NodeFlags.BlockScoped) !== 0 + ) { + for (const d of stmt.declarationList.declarations.nodes) addBindingNames(d.name, out); + } + } + } else if (ts.isCatchClause(scope) && scope.variableDeclaration !== undefined) { + addBindingNames(scope.variableDeclaration.name, out); + } else if ( + (ts.isForStatement(scope) || ts.isForInStatement(scope) || ts.isForOfStatement(scope)) && + scope.initializer !== undefined && + ts.isVariableDeclarationList(scope.initializer) && + (scope.initializer.flags & ts.NodeFlags.BlockScoped) !== 0 + ) { + for (const d of scope.initializer.declarations.nodes) addBindingNames(d.name, out); + } else if ( + (ts.isClassDeclaration(scope) || ts.isClassExpression(scope)) && + scope.name !== undefined + ) { + out.add(scope.name.text); + } + return out; +} + +/** Resolves references against one bundle's scopes, keeping each scope's + * names for the bundle's split. */ +class BundleScopes { + private readonly names = new Map>(); + + private of(scope: ts.Node): Set { + let names = this.names.get(scope); + if (names === undefined) { + names = scopeNames(scope); + this.names.set(scope, names); + } + return names; + } + + /** True when a scope between the reference and `root` (inclusive) + * binds its name. */ + boundWithin(id: ts.Identifier, root: ts.Node): boolean { + for (let n = id.parent; n !== undefined; n = n.parent) { + if (this.of(n).has(id.text)) return true; + if (n === root) return false; + } + return false; + } + + /** True when a scope inside `root` (exclusive) binds the reference's + * name. */ + boundInside(id: ts.Identifier, root: ts.Node): boolean { + for (let n = id.parent; n !== undefined && n !== root; n = n.parent) { + if (this.of(n).has(id.text)) return true; + } + return false; + } + + /** True when the reference resolves to a top-level binding of `sf`. */ + topLevel(id: ts.Identifier, sf: ts.SourceFile): boolean { + for (let n = id.parent; n !== undefined && n !== sf; n = n.parent) { + if (this.of(n).has(id.text)) return false; + } + return true; + } +} + /** Every binding reference inside a subtree. */ export function references(root: ts.Node): ts.Identifier[] { const out: ts.Identifier[] = []; @@ -230,6 +352,52 @@ function copyableInitializer(init: ts.Expression | undefined): boolean { return depth > 0 && ts.isIdentifier(walk) && walk.text === "Object"; } +/** A top-level declaration whose value is a function. */ +function isFunctionValued(node: ts.VariableDeclaration | ts.FunctionDeclaration): boolean { + if (ts.isFunctionDeclaration(node)) return true; + const init = node.initializer && unparen(node.initializer); + return init !== undefined && (ts.isArrowFunction(init) || ts.isFunctionExpression(init)); +} + +/** True when a named function expression initializer (`var f = function + * self() {}`) uses its own name only as a callee: `self` is the copied + * function, so it is as observable as the top-level binding. */ +function selfCalledOnly( + node: ts.VariableDeclaration | ts.FunctionDeclaration, + scopes: BundleScopes, +): boolean { + if (!ts.isVariableDeclaration(node) || node.initializer === undefined) return true; + const fn = unparen(node.initializer); + if (!ts.isFunctionExpression(fn) || fn.name === undefined || fn.body === undefined) return true; + const self = fn.name.text; + // A parameter or `var` of the same name hides the function's own name. + const own = varScopedNames(fn.body); + for (const param of fn.parameters.nodes) addBindingNames(param.name, own); + if (own.has(self)) return true; + // Its parameter initializers see the name too. + return references(fn).every((r) => r.text !== self || scopes.boundInside(r, fn) || isCallee(r)); +} + +/** True when a reference is the function a call invokes: `f(...)`, or the + * `(0, f)(...)` form that drops the receiver. */ +function isCallee(id: ts.Identifier): boolean { + let callee: ts.Node = id; + const p = id.parent; + if ( + p !== undefined && + ts.isBinaryExpression(p) && + p.operatorToken.kind === ts.SyntaxKind.CommaToken && + p.right === id && + p.parent !== undefined && + ts.isParenthesizedExpression(p.parent) + ) + callee = p.parent; + while (callee.parent !== undefined && ts.isParenthesizedExpression(callee.parent)) + callee = callee.parent; + const call = callee.parent; + return call !== undefined && ts.isCallExpression(call) && call.expression === callee; +} + /** One recognized chunk declaration. */ interface Chunk { name: string; @@ -302,6 +470,12 @@ export function splitBundledChunks( if (declaredAnywhere.has("Object") || written.has("Object")) return { degrade: CHUNK_SHAPE_DEGRADE }; const stable = (name: string): boolean => !duplicate.has(name) && !written.has(name); + // Names whose every reference in the bundle is a callee: `f(...)` or the + // `(0, f)(...)` form. + const scopes = new BundleScopes(); + const valueUsed = new Set(); + for (const r of allRefs) if (!isCallee(r) && scopes.topLevel(r, sf)) valueUsed.add(r.text); + const calledOnly = (name: string): boolean => !valueUsed.has(name); const ownNames = (name: string): boolean => { const t = top.get(name); if (t === undefined || !stable(name) || !ts.isVariableDeclaration(t.node)) return false; @@ -422,7 +596,17 @@ export function splitBundledChunks( const out: BundledChunkSplit["chunks"] = []; const bundleEdits: Edit[] = [...callEdits]; for (const chunk of chunks.values()) { - const result = chunkSource(sf, chunk, chunks, top, stable, callEdits, bundleStrict); + const result = chunkSource( + sf, + chunk, + chunks, + top, + stable, + calledOnly, + scopes, + callEdits, + bundleStrict, + ); if (typeof result !== "string") return result; out.push({ path: chunk.path, source: result }); bundleEdits.push({ start: chunk.stmt.getStart(sf), end: chunk.stmt.getEnd(), text: "" }); @@ -524,6 +708,8 @@ function chunkSource( chunks: ReadonlyMap, top: ReadonlyMap, stable: (name: string) => boolean, + calledOnly: (name: string) => boolean, + scopes: BundleScopes, callEdits: readonly Edit[], bundleStrict: boolean, ): string | { degrade: string } { @@ -570,7 +756,8 @@ function chunkSource( } continue; } - if (local.has(name)) continue; + // The factory function itself binds the parameters and `var`s. + if (scopes.boundWithin(r, body.parent ?? body)) continue; if (name === "exports" || name === "module") return why(`reads the bundle's own '${name}' binding`); if (name === "__filename") return why("reads __filename, which names the bundle file"); @@ -594,9 +781,14 @@ function chunkSource( (ts.isVariableDeclaration(node) && copyableInitializer(node.initializer))); if (!pure || chunks.has(name)) return why(`reads the bundle's top-level binding '${name}', which it cannot carry along`); - const own = declaredNames(node); + // Each chunk gets its own copy, so a function's identity and its + // properties must stay unobservable: the bundle only ever calls it. + if (isFunctionValued(node) && (!calledOnly(name) || !selfCalledOnly(node, scopes))) + return why( + `uses the bundle's top-level function '${name}' as a value, which its chunks cannot share`, + ); for (const r of references(node)) { - if (own.has(r.text) || !top.has(r.text) || copied.has(r.text)) continue; + if (scopes.boundWithin(r, node) || !top.has(r.text) || copied.has(r.text)) continue; copied.add(r.text); copyQueue.push(r.text); } diff --git a/packages/compiler/src/frontend/program.ts b/packages/compiler/src/frontend/program.ts index 908a587ea..79978abee 100644 --- a/packages/compiler/src/frontend/program.ts +++ b/packages/compiler/src/frontend/program.ts @@ -3199,29 +3199,105 @@ function cjsCycleRefusal7( return null; } -/** True for a plain read `binding.name` (not a call, not a write) of the - * cycle member `dep` whose export `name` is a top-level function - * declaration nothing reassigns. Every member settles its exports before - * entering the cycle, so Node reads that same function whenever the read - * runs, and the compiled read resolves to the declaration itself. */ +/** True when `use` initializes a top-level const whose every other + * reference is a value of an export statement (`module.exports = { … }` + * or `exports.n = …`). */ +function publishedAlias7(use: ts.Node, decl: ts.Node | undefined): boolean { + if ( + decl === undefined || + !ts.isVariableDeclaration(decl) || + decl.initializer !== use || + !ts.isIdentifier(decl.name) + ) + return false; + const list = decl.parent; + const stmt = list?.parent; + const sf = stmt?.parent; + if ( + list === undefined || + (list.flags & ts.NodeFlags.Const) === 0 || + stmt === undefined || + !ts.isVariableStatement(stmt) || + sf === undefined || + !ts.isSourceFile(sf) + ) + return false; + const alias = decl.name; + const published = new Set(); + for (const st of sf.statements.nodes) { + const cjs = cjsExportAssignmentOf(st); + if (cjs === null) continue; + if (cjs.kind === "member") published.add(cjs.value); + else if (cjs.obj !== null) { + for (const p of cjs.obj.properties.nodes) { + if (ts.isPropertyAssignment(p) && !ts.isComputedPropertyName(p.name)) + published.add(p.initializer); + else if (ts.isShorthandPropertyAssignment(p)) published.add(p.name); + } + } + } + let kept = false; + ts.walkPreorder(sf, (n) => { + if (kept) return "stop"; + if (!ts.isIdentifier(n) || n === alias || n.text !== alias.text) return undefined; + const p = n.parent; + if ( + p !== undefined && + ((ts.isPropertyAccessExpression(p) && p.name === n) || + (ts.isPropertyAssignment(p) && p.name === n)) + ) + return undefined; + if (!published.has(n)) kept = true; + return undefined; + }); + return !kept; +} + +/** True for a plain read `binding.name` (not a call, not a write, not a + * value that is stored or passed on) of the cycle member `dep` whose + * export `name` is a top-level function declaration nothing reassigns. + * Every member settles its exports before entering the cycle, so Node + * reads that same function whenever the read runs, and the compiled read + * resolves to the declaration itself. */ function functionExportRead7(id: ts.Identifier, dep: ts.SourceFile): boolean { const access = id.parent; if (!ts.isPropertyAccessExpression(access) || access.expression !== id) return false; + // The value the read yields: through parentheses and the right operand + // of a comma, so `(0, binding.name)(...)` is the call it spells. let use: ts.Node = access; - while (ts.isParenthesizedExpression(use.parent)) use = use.parent; + for (;;) { + const p = use.parent; + if (ts.isParenthesizedExpression(p)) use = p; + else if ( + ts.isBinaryExpression(p) && + p.operatorToken.kind === ts.SyntaxKind.CommaToken && + p.right === use + ) + use = p; + else break; + } const parent = use.parent; - if ( - (ts.isCallExpression(parent) && parent.expression === use) || - ts.isDeleteExpression(parent) || - ((ts.isPrefixUnaryExpression(parent) || ts.isPostfixUnaryExpression(parent)) && - (parent.operator === ts.SyntaxKind.PlusPlusToken || - parent.operator === ts.SyntaxKind.MinusMinusToken)) || + // Only a use that neither runs the function nor keeps it is a read: a + // stored alias (`const f = binding.name`, `{ f: binding.name }`) or an + // argument can be called while the cycle initializes. A top-level const + // that only publishes it (`module.exports = { name: f }`, the static + // rewrite's re-export spelling) runs nothing: a module that reads the + // export back checks that read itself. + const observes = + publishedAlias7(use, parent) || + ts.isExpressionStatement(parent) || + ts.isTypeOfExpression(parent) || + ts.isVoidExpression(parent) || + (ts.isPrefixUnaryExpression(parent) && parent.operator === ts.SyntaxKind.ExclamationToken) || (ts.isBinaryExpression(parent) && - parent.left === use && - parent.operatorToken.kind >= ts.SyntaxKind.FirstAssignment && - parent.operatorToken.kind <= ts.SyntaxKind.LastAssignment) - ) - return false; + (parent.operatorToken.kind === ts.SyntaxKind.EqualsEqualsEqualsToken || + parent.operatorToken.kind === ts.SyntaxKind.ExclamationEqualsEqualsToken || + parent.operatorToken.kind === ts.SyntaxKind.EqualsEqualsToken || + parent.operatorToken.kind === ts.SyntaxKind.ExclamationEqualsToken)) || + ((ts.isIfStatement(parent) || ts.isWhileStatement(parent) || ts.isDoStatement(parent)) && + parent.expression === use) || + (ts.isConditionalExpression(parent) && parent.condition === use); + if (!observes) return false; const name = access.name.text; // The exported value's identifier: a table entry or a member export. const values: ts.Expression[] = []; diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2564-chunk-helper-shadowing/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2564-chunk-helper-shadowing/main.ts.json new file mode 100644 index 000000000..90e304cbe --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2564-chunk-helper-shadowing/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/2564-chunk-helper-shadowing/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2565-chunk-shared-function/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2565-chunk-shared-function/main.ts.json new file mode 100644 index 000000000..6443d170b --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2565-chunk-shared-function/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/2565-chunk-shared-function/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2566-cjs-cycle-indirect-call/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2566-cjs-cycle-indirect-call/main.ts.json new file mode 100644 index 000000000..f51a29951 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2566-cjs-cycle-indirect-call/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/2566-cjs-cycle-indirect-call/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2571-cjs-cycle-stored-function/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2571-cjs-cycle-stored-function/main.ts.json new file mode 100644 index 000000000..c3e0442f9 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/2571-cjs-cycle-stored-function/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/2571-cjs-cycle-stored-function/main.ts" + ], + "diags": [] +} diff --git a/tests/fixtures/npm/cases/2564-chunk-helper-shadowing/main.ts b/tests/fixtures/npm/cases/2564-chunk-helper-shadowing/main.ts new file mode 100644 index 000000000..d52b3731d --- /dev/null +++ b/tests/fixtures/npm/cases/2564-chunk-helper-shadowing/main.ts @@ -0,0 +1,7 @@ +// A bundled CommonJS chunk that calls the bundle's top-level `label` while +// a nested function of the chunk declares its own `label`: the chunk +// carries the top-level function along, and each reference keeps its own +// binding. +import { inner, outer } from "gtchunkshadow"; + +console.log(outer(), inner()); diff --git a/tests/fixtures/npm/cases/2565-chunk-shared-function/main.ts b/tests/fixtures/npm/cases/2565-chunk-shared-function/main.ts new file mode 100644 index 000000000..21ca81b40 --- /dev/null +++ b/tests/fixtures/npm/cases/2565-chunk-shared-function/main.ts @@ -0,0 +1,7 @@ +// Two bundled CommonJS chunks export the bundle's top-level `tally` and +// count calls on a property of it. Node prints "1 2 true": both chunks +// share one function. A per-chunk copy would split its identity and its +// properties, so the package stays refused. +import { counts, same } from "gtchunkshared"; + +console.log(counts(), same()); diff --git a/tests/fixtures/npm/cases/2566-cjs-cycle-indirect-call/main.ts b/tests/fixtures/npm/cases/2566-cjs-cycle-indirect-call/main.ts new file mode 100644 index 000000000..3cca145e0 --- /dev/null +++ b/tests/fixtures/npm/cases/2566-cjs-cycle-indirect-call/main.ts @@ -0,0 +1,7 @@ +// A require cycle whose partner calls an export of index.js with esbuild's +// `(0, import_index.peek)()` spelling while index is still initializing: +// peek reads a const in its temporal dead zone, so Node throws a +// ReferenceError, and scriptc refuses the cycle. +import { show } from "gtcyclecall"; + +console.log(show()); diff --git a/tests/fixtures/npm/cases/2571-cjs-cycle-stored-function/main.ts b/tests/fixtures/npm/cases/2571-cjs-cycle-stored-function/main.ts new file mode 100644 index 000000000..ae2be341a --- /dev/null +++ b/tests/fixtures/npm/cases/2571-cjs-cycle-stored-function/main.ts @@ -0,0 +1,8 @@ +// Require cycles whose partner stores an export of index.js (in a const, +// or in an object literal) and calls it while index is still +// initializing: the export reads a const in its temporal dead zone, so +// Node throws a ReferenceError, and scriptc refuses both cycles. +import { show } from "gtcyclealias"; +import { show as showBox } from "gtcyclebox"; + +console.log(show(), showBox()); diff --git a/tests/fixtures/npm/node_modules/gtchunkshadow/dist/index.d.ts b/tests/fixtures/npm/node_modules/gtchunkshadow/dist/index.d.ts new file mode 100644 index 000000000..238cf1744 --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtchunkshadow/dist/index.d.ts @@ -0,0 +1,2 @@ +export declare function outer(): string; +export declare function inner(): string; diff --git a/tests/fixtures/npm/node_modules/gtchunkshadow/dist/index.js b/tests/fixtures/npm/node_modules/gtchunkshadow/dist/index.js new file mode 100644 index 000000000..0f89c7ec5 --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtchunkshadow/dist/index.js @@ -0,0 +1,52 @@ +var __defProp = Object.defineProperty; +var __getOwnPropDesc = Object.getOwnPropertyDescriptor; +var __getOwnPropNames = Object.getOwnPropertyNames; +var __hasOwnProp = Object.prototype.hasOwnProperty; +var __commonJS = (cb, mod) => function __require() { + return mod || (0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports; +}; +var __export = (target, all) => { + for (var name in all) + __defProp(target, name, { get: all[name], enumerable: true }); +}; +var __copyProps = (to, from, except, desc) => { + if (from && typeof from === "object" || typeof from === "function") { + for (let key of __getOwnPropNames(from)) + if (!__hasOwnProp.call(to, key) && key !== except) + __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable }); + } + return to; +}; +var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod); +function label(n) { + return "label " + n; +} + +// chunk/vendor/shadow.js +var require_shadow = __commonJS({ + "chunk/vendor/shadow.js"(exports2, module2) { + function nested() { + var label = "nested"; + return label; + } + module2.exports = { outer: label(1), inner: nested() }; + } +}); + +// chunk/src/index.ts +var index_exports = {}; +__export(index_exports, { + inner: () => inner, + outer: () => outer +}); +module.exports = __toCommonJS(index_exports); +function outer() { + return require_shadow().outer; +} +function inner() { + return require_shadow().inner; +} +0 && (module.exports = { + inner, + outer +}); diff --git a/tests/fixtures/npm/node_modules/gtchunkshadow/package.json b/tests/fixtures/npm/node_modules/gtchunkshadow/package.json new file mode 100644 index 000000000..6830fbd2c --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtchunkshadow/package.json @@ -0,0 +1,7 @@ +{ + "name": "gtchunkshadow", + "version": "1.0.0", + "description": "esbuild's CommonJS bundle shape: a chunk calls a top-level function whose name a nested scope of the chunk also declares", + "main": "./dist/index.js", + "types": "./dist/index.d.ts" +} diff --git a/tests/fixtures/npm/node_modules/gtchunkshared/dist/index.d.ts b/tests/fixtures/npm/node_modules/gtchunkshared/dist/index.d.ts new file mode 100644 index 000000000..3a47a0495 --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtchunkshared/dist/index.d.ts @@ -0,0 +1,2 @@ +export declare function same(): boolean; +export declare function counts(): string; diff --git a/tests/fixtures/npm/node_modules/gtchunkshared/dist/index.js b/tests/fixtures/npm/node_modules/gtchunkshared/dist/index.js new file mode 100644 index 000000000..0d3aacec8 --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtchunkshared/dist/index.js @@ -0,0 +1,56 @@ +var __defProp = Object.defineProperty; +var __getOwnPropDesc = Object.getOwnPropertyDescriptor; +var __getOwnPropNames = Object.getOwnPropertyNames; +var __hasOwnProp = Object.prototype.hasOwnProperty; +var __commonJS = (cb, mod) => function __require() { + return mod || (0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports; +}; +var __export = (target, all) => { + for (var name in all) + __defProp(target, name, { get: all[name], enumerable: true }); +}; +var __copyProps = (to, from, except, desc) => { + if (from && typeof from === "object" || typeof from === "function") { + for (let key of __getOwnPropNames(from)) + if (!__hasOwnProp.call(to, key) && key !== except) + __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable }); + } + return to; +}; +var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod); +function tally() { + tally.calls = (tally.calls || 0) + 1; + return tally.calls; +} + +// chunk/vendor/a.js +var require_a = __commonJS({ + "chunk/vendor/a.js"(exports2, module2) { + module2.exports = { tally, first: tally() }; + } +}); + +// chunk/vendor/b.js +var require_b = __commonJS({ + "chunk/vendor/b.js"(exports2, module2) { + module2.exports = { tally, second: tally() }; + } +}); + +// chunk/src/index.ts +var index_exports = {}; +__export(index_exports, { + counts: () => counts, + same: () => same +}); +module.exports = __toCommonJS(index_exports); +function same() { + return require_a().tally === require_b().tally; +} +function counts() { + return require_a().first + " " + require_b().second; +} +0 && (module.exports = { + counts, + same +}); diff --git a/tests/fixtures/npm/node_modules/gtchunkshared/package.json b/tests/fixtures/npm/node_modules/gtchunkshared/package.json new file mode 100644 index 000000000..bfc90c5d2 --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtchunkshared/package.json @@ -0,0 +1,7 @@ +{ + "name": "gtchunkshared", + "version": "1.0.0", + "description": "esbuild's CommonJS bundle shape: two chunks export one top-level function and count calls on a property of it", + "main": "./dist/index.js", + "types": "./dist/index.d.ts" +} diff --git a/tests/fixtures/npm/node_modules/gtcyclealias/dist/index.d.ts b/tests/fixtures/npm/node_modules/gtcyclealias/dist/index.d.ts new file mode 100644 index 000000000..5f4373e2f --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtcyclealias/dist/index.d.ts @@ -0,0 +1,3 @@ +export declare const LIMIT = 3; +export declare function show(): string; +export declare function peek(): number; diff --git a/tests/fixtures/npm/node_modules/gtcyclealias/dist/index.js b/tests/fixtures/npm/node_modules/gtcyclealias/dist/index.js new file mode 100644 index 000000000..77aca3c9b --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtcyclealias/dist/index.js @@ -0,0 +1,37 @@ +var __defProp = Object.defineProperty; +var __getOwnPropDesc = Object.getOwnPropertyDescriptor; +var __getOwnPropNames = Object.getOwnPropertyNames; +var __hasOwnProp = Object.prototype.hasOwnProperty; +var __export = (target, all) => { + for (var name in all) + __defProp(target, name, { get: all[name], enumerable: true }); +}; +var __copyProps = (to, from, except, desc) => { + if (from && typeof from === "object" || typeof from === "function") { + for (let key of __getOwnPropNames(from)) + if (!__hasOwnProp.call(to, key) && key !== except) + __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable }); + } + return to; +}; +var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod); +var index_exports = {}; +__export(index_exports, { + LIMIT: () => LIMIT, + peek: () => peek, + show: () => show +}); +module.exports = __toCommonJS(index_exports); +var import_reader = require("./reader"); +const LIMIT = 3; +function peek() { + return LIMIT; +} +function show() { + return (0, import_reader.describe)(); +} +// Annotate the CommonJS export names for ESM import in node: +0 && (module.exports = { + LIMIT, + show +}); diff --git a/tests/fixtures/npm/node_modules/gtcyclealias/dist/reader.js b/tests/fixtures/npm/node_modules/gtcyclealias/dist/reader.js new file mode 100644 index 000000000..943e61a0a --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtcyclealias/dist/reader.js @@ -0,0 +1,34 @@ +var __defProp = Object.defineProperty; +var __getOwnPropDesc = Object.getOwnPropertyDescriptor; +var __getOwnPropNames = Object.getOwnPropertyNames; +var __hasOwnProp = Object.prototype.hasOwnProperty; +var __export = (target, all) => { + for (var name in all) + __defProp(target, name, { get: all[name], enumerable: true }); +}; +var __copyProps = (to, from, except, desc) => { + if (from && typeof from === "object" || typeof from === "function") { + for (let key of __getOwnPropNames(from)) + if (!__hasOwnProp.call(to, key) && key !== except) + __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable }); + } + return to; +}; +var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod); +var reader_exports = {}; +__export(reader_exports, { + captured: () => captured, + describe: () => describe +}); +module.exports = __toCommonJS(reader_exports); +var import_index = require("./index"); +const read = import_index.peek; +const captured = read(); +function describe() { + return "limit " + captured; +} +// Annotate the CommonJS export names for ESM import in node: +0 && (module.exports = { + captured, + describe +}); diff --git a/tests/fixtures/npm/node_modules/gtcyclealias/package.json b/tests/fixtures/npm/node_modules/gtcyclealias/package.json new file mode 100644 index 000000000..9ee7441e0 --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtcyclealias/package.json @@ -0,0 +1,7 @@ +{ + "name": "gtcyclealias", + "version": "1.0.0", + "description": "esbuild 0.25.10 per-file CommonJS output shape of a cycle whose partner stores an exported function in a const and calls it while that function's const binding is uninitialized", + "main": "./dist/index.js", + "types": "./dist/index.d.ts" +} diff --git a/tests/fixtures/npm/node_modules/gtcyclebox/dist/index.d.ts b/tests/fixtures/npm/node_modules/gtcyclebox/dist/index.d.ts new file mode 100644 index 000000000..5f4373e2f --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtcyclebox/dist/index.d.ts @@ -0,0 +1,3 @@ +export declare const LIMIT = 3; +export declare function show(): string; +export declare function peek(): number; diff --git a/tests/fixtures/npm/node_modules/gtcyclebox/dist/index.js b/tests/fixtures/npm/node_modules/gtcyclebox/dist/index.js new file mode 100644 index 000000000..77aca3c9b --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtcyclebox/dist/index.js @@ -0,0 +1,37 @@ +var __defProp = Object.defineProperty; +var __getOwnPropDesc = Object.getOwnPropertyDescriptor; +var __getOwnPropNames = Object.getOwnPropertyNames; +var __hasOwnProp = Object.prototype.hasOwnProperty; +var __export = (target, all) => { + for (var name in all) + __defProp(target, name, { get: all[name], enumerable: true }); +}; +var __copyProps = (to, from, except, desc) => { + if (from && typeof from === "object" || typeof from === "function") { + for (let key of __getOwnPropNames(from)) + if (!__hasOwnProp.call(to, key) && key !== except) + __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable }); + } + return to; +}; +var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod); +var index_exports = {}; +__export(index_exports, { + LIMIT: () => LIMIT, + peek: () => peek, + show: () => show +}); +module.exports = __toCommonJS(index_exports); +var import_reader = require("./reader"); +const LIMIT = 3; +function peek() { + return LIMIT; +} +function show() { + return (0, import_reader.describe)(); +} +// Annotate the CommonJS export names for ESM import in node: +0 && (module.exports = { + LIMIT, + show +}); diff --git a/tests/fixtures/npm/node_modules/gtcyclebox/dist/reader.js b/tests/fixtures/npm/node_modules/gtcyclebox/dist/reader.js new file mode 100644 index 000000000..b8f8a927e --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtcyclebox/dist/reader.js @@ -0,0 +1,34 @@ +var __defProp = Object.defineProperty; +var __getOwnPropDesc = Object.getOwnPropertyDescriptor; +var __getOwnPropNames = Object.getOwnPropertyNames; +var __hasOwnProp = Object.prototype.hasOwnProperty; +var __export = (target, all) => { + for (var name in all) + __defProp(target, name, { get: all[name], enumerable: true }); +}; +var __copyProps = (to, from, except, desc) => { + if (from && typeof from === "object" || typeof from === "function") { + for (let key of __getOwnPropNames(from)) + if (!__hasOwnProp.call(to, key) && key !== except) + __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable }); + } + return to; +}; +var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod); +var reader_exports = {}; +__export(reader_exports, { + captured: () => captured, + describe: () => describe +}); +module.exports = __toCommonJS(reader_exports); +var import_index = require("./index"); +const box = { read: import_index.peek }; +const captured = box.read(); +function describe() { + return "limit " + captured; +} +// Annotate the CommonJS export names for ESM import in node: +0 && (module.exports = { + captured, + describe +}); diff --git a/tests/fixtures/npm/node_modules/gtcyclebox/package.json b/tests/fixtures/npm/node_modules/gtcyclebox/package.json new file mode 100644 index 000000000..67f943195 --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtcyclebox/package.json @@ -0,0 +1,7 @@ +{ + "name": "gtcyclebox", + "version": "1.0.0", + "description": "esbuild 0.25.10 per-file CommonJS output shape of a cycle whose partner stores an exported function in an object literal and calls it while that function's const binding is uninitialized", + "main": "./dist/index.js", + "types": "./dist/index.d.ts" +} diff --git a/tests/fixtures/npm/node_modules/gtcyclecall/dist/index.d.ts b/tests/fixtures/npm/node_modules/gtcyclecall/dist/index.d.ts new file mode 100644 index 000000000..5f4373e2f --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtcyclecall/dist/index.d.ts @@ -0,0 +1,3 @@ +export declare const LIMIT = 3; +export declare function show(): string; +export declare function peek(): number; diff --git a/tests/fixtures/npm/node_modules/gtcyclecall/dist/index.js b/tests/fixtures/npm/node_modules/gtcyclecall/dist/index.js new file mode 100644 index 000000000..77aca3c9b --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtcyclecall/dist/index.js @@ -0,0 +1,37 @@ +var __defProp = Object.defineProperty; +var __getOwnPropDesc = Object.getOwnPropertyDescriptor; +var __getOwnPropNames = Object.getOwnPropertyNames; +var __hasOwnProp = Object.prototype.hasOwnProperty; +var __export = (target, all) => { + for (var name in all) + __defProp(target, name, { get: all[name], enumerable: true }); +}; +var __copyProps = (to, from, except, desc) => { + if (from && typeof from === "object" || typeof from === "function") { + for (let key of __getOwnPropNames(from)) + if (!__hasOwnProp.call(to, key) && key !== except) + __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable }); + } + return to; +}; +var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod); +var index_exports = {}; +__export(index_exports, { + LIMIT: () => LIMIT, + peek: () => peek, + show: () => show +}); +module.exports = __toCommonJS(index_exports); +var import_reader = require("./reader"); +const LIMIT = 3; +function peek() { + return LIMIT; +} +function show() { + return (0, import_reader.describe)(); +} +// Annotate the CommonJS export names for ESM import in node: +0 && (module.exports = { + LIMIT, + show +}); diff --git a/tests/fixtures/npm/node_modules/gtcyclecall/dist/reader.js b/tests/fixtures/npm/node_modules/gtcyclecall/dist/reader.js new file mode 100644 index 000000000..c98d1ae46 --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtcyclecall/dist/reader.js @@ -0,0 +1,33 @@ +var __defProp = Object.defineProperty; +var __getOwnPropDesc = Object.getOwnPropertyDescriptor; +var __getOwnPropNames = Object.getOwnPropertyNames; +var __hasOwnProp = Object.prototype.hasOwnProperty; +var __export = (target, all) => { + for (var name in all) + __defProp(target, name, { get: all[name], enumerable: true }); +}; +var __copyProps = (to, from, except, desc) => { + if (from && typeof from === "object" || typeof from === "function") { + for (let key of __getOwnPropNames(from)) + if (!__hasOwnProp.call(to, key) && key !== except) + __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable }); + } + return to; +}; +var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod); +var reader_exports = {}; +__export(reader_exports, { + captured: () => captured, + describe: () => describe +}); +module.exports = __toCommonJS(reader_exports); +var import_index = require("./index"); +const captured = (0, import_index.peek)(); +function describe() { + return "limit " + captured; +} +// Annotate the CommonJS export names for ESM import in node: +0 && (module.exports = { + captured, + describe +}); diff --git a/tests/fixtures/npm/node_modules/gtcyclecall/package.json b/tests/fixtures/npm/node_modules/gtcyclecall/package.json new file mode 100644 index 000000000..cc1069a6f --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtcyclecall/package.json @@ -0,0 +1,7 @@ +{ + "name": "gtcyclecall", + "version": "1.0.0", + "description": "esbuild 0.25.10 per-file CommonJS output shape of a cycle whose partner calls an exported function through (0, binding.fn)() while that function's const binding is uninitialized", + "main": "./dist/index.js", + "types": "./dist/index.d.ts" +} diff --git a/tests/harness/npm-cases.ts b/tests/harness/npm-cases.ts index 35a9df4a6..9ca3c0df0 100644 --- a/tests/harness/npm-cases.ts +++ b/tests/harness/npm-cases.ts @@ -19,12 +19,12 @@ export function npmCases(fixturesRoot: string): NpmCase[] { return [ ...globSync(join(fixturesRoot, "npm/cases/*/main.ts")) .sort() - // 2465-2469, 2556-2563 and 2567 are the --npm-static bundler-emitted-CJS + // 2465-2469, 2556-2567 and 2571 are the --npm-static bundler-emitted-CJS // cases (npm-static.test.ts drives them with the opt-in): their // consumers exercise the static rewrite's surface (lexer-visible // names the shipped .d.ts never declares, the __toESM interop // family), so they stay out of the flagless island lane by design. - .filter((entry) => !/\/(246[5-9]|255[6-9]|256[0-37])-[^/]+\/main\.ts$/.test(entry)) + .filter((entry) => !/\/(246[5-9]|255[6-9]|256[0-7]|2571)-[^/]+\/main\.ts$/.test(entry)) .map((entry) => ({ name: entry.split("/").at(-2)!, entry, diff --git a/tests/harness/npm-static.test.ts b/tests/harness/npm-static.test.ts index 443c0c72d..32639e0d1 100644 --- a/tests/harness/npm-static.test.ts +++ b/tests/harness/npm-static.test.ts @@ -76,7 +76,7 @@ async function buildStatic(entry: string, npmStatic: string[] | "auto"): Promise ...globSync( join(fixturesRoot, "npm/node_modules/cryptozoo/**/*.{js,mjs,cjs,json,d.ts}"), ).sort(), - // the bundler-emitted-CJS mini packages (cases 2465-2469, 2556-2563, 2567) + // the bundler-emitted-CJS mini packages (cases 2465-2469, 2556-2567, 2571) ...globSync(join(fixturesRoot, "npm/node_modules/gt*/**/*.{js,cjs,json}")).sort(), ]; for (const f of inputs) hash.update(f).update(readFileSync(f)); @@ -1502,6 +1502,61 @@ describe(`npm-static pilots${sanitize ? " (sanitized)" : ""}`, () => { ]); }, 180_000); + // 2566: the same early read through esbuild's `(0, binding.fn)()` call + // spelling also keeps SC1016. + test("a CommonJS cycle that calls an export during initialization stays refused", () => { + const entry = join(fixturesRoot, "npm/cases/2566-cjs-cycle-indirect-call/main.ts"); + const { coverage } = analyze(entry, { npmStatic: ["gtcyclecall"] }); + expect(coverage.npmStatic).toEqual([ + { + package: "gtcyclecall", + status: "refused", + detail: expect.stringMatching( + /circular imports .*can use its binding 'import_index' of a module of the cycle/, + ) as string, + }, + ]); + }, 120_000); + + // 2571: the partner stores the export (in a const, or in an object + // literal) and calls it while the cycle initializes; both keep SC1016. + test("a CommonJS cycle that stores an export and calls it stays refused", () => { + const entry = join(fixturesRoot, "npm/cases/2571-cjs-cycle-stored-function/main.ts"); + const { coverage } = analyze(entry, { npmStatic: ["gtcyclealias", "gtcyclebox"] }); + for (const pkg of ["gtcyclealias", "gtcyclebox"]) + expect(coverage.npmStatic).toContainEqual({ + package: pkg, + status: "refused", + detail: expect.stringMatching( + /circular imports .*can use its binding 'import_index' of a module of the cycle/, + ) as string, + }); + }, 120_000); + + // 2564: a chunk reads a top-level function whose name a nested scope of + // the chunk also declares; the function still travels with the chunk. + test("a chunk carries a top-level function its nested scopes shadow", async () => { + await expectNodeParity(join(fixturesRoot, "npm/cases/2564-chunk-helper-shadowing/main.ts"), [ + "gtchunkshadow", + ]); + }, 180_000); + + // 2565: Node shares one function between the chunks that export it; a + // per-chunk copy would not, so the package stays refused. + test("chunks that export a shared top-level function stay refused", () => { + const entry = join(fixturesRoot, "npm/cases/2565-chunk-shared-function/main.ts"); + const { coverage } = analyze(entry, { npmStatic: ["gtchunkshared"] }); + expect(coverage.npmStatic).toEqual([ + { + package: "gtchunkshared", + status: "refused", + detail: expect.stringMatching( + /uses the bundle's top-level function 'tally' as a value, which its chunks cannot share/, + ) as string, + }, + ]); + }, 120_000); + test("published type-only declarations coexist with a static runtime bundle", async () => { const entry = join(fixturesRoot, "npm/cases/2469-bundle-offender/main.ts"); const { coverage } = analyze(entry, { npmStatic: ["gtghost"] }); From 72e22ed0edfb0c29d3cc68fef5d53ea338bbae28 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Mon, 19 Oct 2026 23:22:23 -0500 Subject: [PATCH 19/35] Run SHA-384 and SHA-512 blocks with AVX2, AVX-512VL or the arm64 SHA-512 instructions SHA-384, SHA-512, SHA-512/224 and SHA-512/256 (createHash, createHmac, crypto.hash, pbkdf2, hkdf) process their blocks with the fastest kernel the CPU supports: - x86-64 with AVX2 and BMI2: the message schedule is computed two words at a time in vector registers, interleaved with the scalar rounds, which rotate with RORX. - x86-64 with AVX-512VL: the same, with vector rotates and three-input XOR in the schedule. - arm64 with the SHA-512 extension: SHA512H, SHA512H2, SHA512SU0 and SHA512SU1. - Otherwise the portable blocks, now unrolled over a 16-word schedule ring. The CPU and OS checks run once and are cached. The update and final paths pass all their whole blocks to the kernel in one call. test_sha_kernels.c checks the FIPS 180-4 SHA-384 and SHA-512 vectors through every kernel the CPU supports and compares each kernel with the portable blocks over every length from 0 to 1100 bytes and 1 MiB, whole and in 1-, 7-, 64- and 100-byte updates. Corpus 5779 compares SHA-384, SHA-512 and HMAC-SHA512 digests over many lengths and update sizes with Node; tests/harness/corpus-costs/ records its plain and sanitized costs. --- .../tests/corpus/5779-sha512-lengths.ts.json | 6 + packages/runtime/src/scr_lib.c | 344 ++++++++++++++++-- packages/runtime/test/sha-kernels.test.ts | 4 +- packages/runtime/test/test_sha_kernels.c | 100 ++++- tests/corpus/5779-sha512-lengths.ts | 35 ++ .../corpus-costs/5779-sha512-lengths.ts.json | 1 + 6 files changed, 456 insertions(+), 34 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5779-sha512-lengths.ts.json create mode 100644 tests/corpus/5779-sha512-lengths.ts create mode 100644 tests/harness/corpus-costs/5779-sha512-lengths.ts.json diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5779-sha512-lengths.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5779-sha512-lengths.ts.json new file mode 100644 index 000000000..60582ec60 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5779-sha512-lengths.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5779-sha512-lengths.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_lib.c b/packages/runtime/src/scr_lib.c index 930d44818..99c5a8f63 100644 --- a/packages/runtime/src/scr_lib.c +++ b/packages/runtime/src/scr_lib.c @@ -7405,29 +7405,319 @@ static uint64_t scr_sha512_rotr(uint64_t x, unsigned n) { return (x >> n) | (x << (64 - n)); } -SCR_DIGEST_ALIGNED static void scr_sha512_block(uint64_t h[8], const unsigned char *data) { - uint64_t w[80]; - for (size_t i = 0; i < 16; i++) { - w[i] = 0; - for (size_t b = 0; b < 8; b++) w[i] = (w[i] << 8) | data[i * 8 + b]; +#define SCR_SHA512_BSIG0(x) \ + (scr_sha512_rotr(x, 28) ^ scr_sha512_rotr(x, 34) ^ scr_sha512_rotr(x, 39)) +#define SCR_SHA512_BSIG1(x) \ + (scr_sha512_rotr(x, 14) ^ scr_sha512_rotr(x, 18) ^ scr_sha512_rotr(x, 41)) +#define SCR_SHA512_SSIG0(x) (scr_sha512_rotr(x, 1) ^ scr_sha512_rotr(x, 8) ^ ((x) >> 7)) +#define SCR_SHA512_SSIG1(x) (scr_sha512_rotr(x, 19) ^ scr_sha512_rotr(x, 61) ^ ((x) >> 6)) + +/* One round with the eight working variables passed in rotated order, so + * an unrolled sequence of rounds needs no register moves. */ +#define SCR_SHA512_ROUND(a, b, c, d, e, f, g, h, k, w) \ + do { \ + uint64_t t1 = (h) + SCR_SHA512_BSIG1(e) + ((g) ^ ((e) & ((f) ^ (g)))) + (k) + (w); \ + (d) += t1; \ + (h) = t1 + SCR_SHA512_BSIG0(a) + (((a) & (b)) | ((c) & ((a) | (b)))); \ + } while (0) + +static uint64_t scr_sha512_load(const unsigned char *p) { + uint64_t v = 0; + for (size_t b = 0; b < 8; b++) v = (v << 8) | p[b]; + return v; +} + +/* The message schedule is kept as a 16-word ring and extended 16 words + * at a time between groups of 16 rounds. */ +SCR_DIGEST_ALIGNED static void scr_sha512_blocks_portable(uint64_t state[8], const unsigned char *p, + size_t blocks) { + uint64_t a = state[0], b = state[1], c = state[2], d = state[3]; + uint64_t e = state[4], f = state[5], g = state[6], h = state[7]; + for (; blocks != 0; blocks--, p += 128) { + uint64_t w[16]; + for (size_t i = 0; i < 16; i++) w[i] = scr_sha512_load(p + i * 8); + for (size_t i = 0; i < 80; i += 16) { + if (i != 0) { + for (size_t j = 0; j < 16; j++) { + w[j] += SCR_SHA512_SSIG1(w[(j + 14) & 15]) + w[(j + 9) & 15] + + SCR_SHA512_SSIG0(w[(j + 1) & 15]); + } + } + const uint64_t *k = scr_sha512_k + i; + SCR_SHA512_ROUND(a, b, c, d, e, f, g, h, k[0], w[0]); + SCR_SHA512_ROUND(h, a, b, c, d, e, f, g, k[1], w[1]); + SCR_SHA512_ROUND(g, h, a, b, c, d, e, f, k[2], w[2]); + SCR_SHA512_ROUND(f, g, h, a, b, c, d, e, k[3], w[3]); + SCR_SHA512_ROUND(e, f, g, h, a, b, c, d, k[4], w[4]); + SCR_SHA512_ROUND(d, e, f, g, h, a, b, c, k[5], w[5]); + SCR_SHA512_ROUND(c, d, e, f, g, h, a, b, k[6], w[6]); + SCR_SHA512_ROUND(b, c, d, e, f, g, h, a, k[7], w[7]); + SCR_SHA512_ROUND(a, b, c, d, e, f, g, h, k[8], w[8]); + SCR_SHA512_ROUND(h, a, b, c, d, e, f, g, k[9], w[9]); + SCR_SHA512_ROUND(g, h, a, b, c, d, e, f, k[10], w[10]); + SCR_SHA512_ROUND(f, g, h, a, b, c, d, e, k[11], w[11]); + SCR_SHA512_ROUND(e, f, g, h, a, b, c, d, k[12], w[12]); + SCR_SHA512_ROUND(d, e, f, g, h, a, b, c, k[13], w[13]); + SCR_SHA512_ROUND(c, d, e, f, g, h, a, b, k[14], w[14]); + SCR_SHA512_ROUND(b, c, d, e, f, g, h, a, k[15], w[15]); + } + a = state[0] += a; b = state[1] += b; c = state[2] += c; d = state[3] += d; + e = state[4] += e; f = state[5] += f; g = state[6] += g; h = state[7] += h; } - for (size_t i = 16; i < 80; i++) { - uint64_t s0 = scr_sha512_rotr(w[i - 15], 1) ^ scr_sha512_rotr(w[i - 15], 8) ^ (w[i - 15] >> 7); - uint64_t s1 = scr_sha512_rotr(w[i - 2], 19) ^ scr_sha512_rotr(w[i - 2], 61) ^ (w[i - 2] >> 6); - w[i] = w[i - 16] + s0 + w[i - 7] + s1; +} + +#ifdef SCR_SHA_NI +/* ── SHA-512 kernels (x86-64) ───────────────────────────────────────── + * With AVX2 and BMI2, the message schedule is computed two words at a + * time in vector registers, interleaved with the scalar rounds, and the + * rounds rotate with RORX. With AVX-512VL the schedule uses vector + * rotates and three-input XOR. The CPU and OS state checks run once and + * are cached. */ +static int scr_sha512_x86_state; /* 0 unknown, 1 portable, 2 AVX2, 3 AVX-512VL */ + +static int scr_sha512_x86(void) { + if (scr_sha512_x86_state == 0) { + unsigned a, b, c, d; + int kind = 1; + if (__get_cpuid(1, &a, &b, &c, &d) && (c & bit_OSXSAVE) && (c & bit_AVX)) { + unsigned lo, hi; + __asm__ volatile("xgetbv" : "=a"(lo), "=d"(hi) : "c"(0)); + (void)hi; + /* XMM and YMM state enabled. */ + if ((lo & 6) == 6 && __get_cpuid_count(7, 0, &a, &b, &c, &d) && (b & bit_AVX2) && + (b & bit_BMI2)) { + kind = 2; + /* Opmask, ZMM0-15 upper halves and ZMM16-31 state enabled. */ + if ((b & bit_AVX512F) && (b & bit_AVX512VL) && (lo & 0xe0) == 0xe0) kind = 3; + } + } + scr_sha512_x86_state = kind; } - uint64_t a = h[0], b = h[1], c = h[2], d = h[3]; - uint64_t e = h[4], f = h[5], g = h[6], v = h[7]; - for (size_t i = 0; i < 80; i++) { - uint64_t s1 = scr_sha512_rotr(e, 14) ^ scr_sha512_rotr(e, 18) ^ scr_sha512_rotr(e, 41); - uint64_t t1 = v + s1 + ((e & f) ^ (~e & g)) + scr_sha512_k[i] + w[i]; - uint64_t s0 = scr_sha512_rotr(a, 28) ^ scr_sha512_rotr(a, 34) ^ scr_sha512_rotr(a, 39); - uint64_t t2 = s0 + ((a & b) ^ (a & c) ^ (b & c)); - v = g; g = f; f = e; e = d + t1; - d = c; c = b; b = a; a = t1 + t2; + return scr_sha512_x86_state; +} + +__attribute__((target("avx2,bmi2"))) static inline __m128i scr_sha512_ssig0_avx2(__m128i x) { + return _mm_xor_si128(_mm_xor_si128(_mm_or_si128(_mm_srli_epi64(x, 1), _mm_slli_epi64(x, 63)), + _mm_or_si128(_mm_srli_epi64(x, 8), _mm_slli_epi64(x, 56))), + _mm_srli_epi64(x, 7)); +} + +__attribute__((target("avx2,bmi2"))) static inline __m128i scr_sha512_ssig1_avx2(__m128i x) { + return _mm_xor_si128(_mm_xor_si128(_mm_or_si128(_mm_srli_epi64(x, 19), _mm_slli_epi64(x, 45)), + _mm_or_si128(_mm_srli_epi64(x, 61), _mm_slli_epi64(x, 3))), + _mm_srli_epi64(x, 6)); +} + +__attribute__((target("avx512f,avx512vl,bmi2"))) static inline __m128i +scr_sha512_ssig0_avx512(__m128i x) { + return _mm_ternarylogic_epi64(_mm_ror_epi64(x, 1), _mm_ror_epi64(x, 8), _mm_srli_epi64(x, 7), + 0x96); +} + +__attribute__((target("avx512f,avx512vl,bmi2"))) static inline __m128i +scr_sha512_ssig1_avx512(__m128i x) { + return _mm_ternarylogic_epi64(_mm_ror_epi64(x, 19), _mm_ror_epi64(x, 61), _mm_srli_epi64(x, 6), + 0x96); +} + +/* A round for the vector kernels. wk is the schedule word plus its + * constant. Maj(a, b, c) is ((a ^ b) & (b ^ c)) ^ b, where b ^ c is the + * previous round's a ^ b. The empty asm statements keep the compiler from + * reassociating the additions: adding Sigma1(e) last keeps the sum of h, + * wk and Ch off the dependency chain through e. */ +#define SCR_SHA512_PIN(v) __asm__("" : "+r"(v)) +#define SCR_SHA512_ROUND_WK(a, b, c, d, e, f, g, h, wk, ab, bc) \ + do { \ + uint64_t t1 = (h) + (wk); \ + SCR_SHA512_PIN(t1); \ + t1 += (g) ^ ((e) & ((f) ^ (g))); \ + SCR_SHA512_PIN(t1); \ + t1 += SCR_SHA512_BSIG1(e); \ + SCR_SHA512_PIN(t1); \ + (d) += t1; \ + (ab) = (a) ^ (b); \ + uint64_t m = t1 + (((ab) & (bc)) ^ (b)); \ + SCR_SHA512_PIN(m); \ + (h) = m + SCR_SHA512_BSIG0(a); \ + } while (0) + +/* Words 2j and 2j+1 of the schedule, in the ring slot of words 2j-16 and + * 2j-15, and their sum with the round constants into wk. */ +#define SCR_SHA512_SCHEDULE(j, ssig0, ssig1) \ + do { \ + if ((j) < 40) { \ + __m128i s15 = _mm_alignr_epi8(w[((j) + 1) & 7], w[(j) & 7], 8); \ + __m128i s7 = _mm_alignr_epi8(w[((j) + 5) & 7], w[((j) + 4) & 7], 8); \ + __m128i next = _mm_add_epi64(_mm_add_epi64(w[(j) & 7], ssig0(s15)), \ + _mm_add_epi64(s7, ssig1(w[((j) + 7) & 7]))); \ + w[(j) & 7] = next; \ + _mm_store_si128((__m128i *)&wk[(j) * 2], \ + _mm_add_epi64(next, _mm_loadu_si128((const __m128i *)&scr_sha512_k[(j) * 2]))); \ + } \ + } while (0) + +/* Rounds 2i to 2i+7 compute the schedule words 2i+16 to 2i+23. */ +#define SCR_SHA512_VECTOR_BODY(ssig0, ssig1) \ + const __m128i bswap = _mm_set_epi64x(0x08090a0b0c0d0e0fULL, 0x0001020304050607ULL); \ + uint64_t a = state[0], b = state[1], c = state[2], d = state[3]; \ + uint64_t e = state[4], f = state[5], g = state[6], h = state[7]; \ + for (; blocks != 0; blocks--, p += 128) { \ + __m128i w[8]; \ + uint64_t wk[80] __attribute__((aligned(16))); \ + _Pragma("GCC unroll 8") for (int i = 0; i < 8; i++) { \ + w[i] = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *)(p + i * 16)), bswap); \ + _mm_store_si128((__m128i *)&wk[i * 2], \ + _mm_add_epi64(w[i], _mm_loadu_si128((const __m128i *)&scr_sha512_k[i * 2]))); \ + } \ + uint64_t x = b ^ c, y; \ + _Pragma("GCC unroll 10") for (int i = 0; i < 40; i += 4) { \ + SCR_SHA512_SCHEDULE(i + 8, ssig0, ssig1); \ + SCR_SHA512_ROUND_WK(a, b, c, d, e, f, g, h, wk[2 * i + 0], y, x); \ + SCR_SHA512_ROUND_WK(h, a, b, c, d, e, f, g, wk[2 * i + 1], x, y); \ + SCR_SHA512_SCHEDULE(i + 9, ssig0, ssig1); \ + SCR_SHA512_ROUND_WK(g, h, a, b, c, d, e, f, wk[2 * i + 2], y, x); \ + SCR_SHA512_ROUND_WK(f, g, h, a, b, c, d, e, wk[2 * i + 3], x, y); \ + SCR_SHA512_SCHEDULE(i + 10, ssig0, ssig1); \ + SCR_SHA512_ROUND_WK(e, f, g, h, a, b, c, d, wk[2 * i + 4], y, x); \ + SCR_SHA512_ROUND_WK(d, e, f, g, h, a, b, c, wk[2 * i + 5], x, y); \ + SCR_SHA512_SCHEDULE(i + 11, ssig0, ssig1); \ + SCR_SHA512_ROUND_WK(c, d, e, f, g, h, a, b, wk[2 * i + 6], y, x); \ + SCR_SHA512_ROUND_WK(b, c, d, e, f, g, h, a, wk[2 * i + 7], x, y); \ + } \ + a = state[0] += a; b = state[1] += b; c = state[2] += c; d = state[3] += d; \ + e = state[4] += e; f = state[5] += f; g = state[6] += g; h = state[7] += h; \ } - h[0] += a; h[1] += b; h[2] += c; h[3] += d; - h[4] += e; h[5] += f; h[6] += g; h[7] += v; + +__attribute__((target("avx2,bmi2"), aligned(64))) static void +scr_sha512_blocks_avx2(uint64_t state[8], const unsigned char *p, size_t blocks) { + SCR_SHA512_VECTOR_BODY(scr_sha512_ssig0_avx2, scr_sha512_ssig1_avx2) +} + +__attribute__((target("avx512f,avx512vl,bmi2"), aligned(64))) static void +scr_sha512_blocks_avx512(uint64_t state[8], const unsigned char *p, size_t blocks) { + SCR_SHA512_VECTOR_BODY(scr_sha512_ssig0_avx512, scr_sha512_ssig1_avx512) +} +#endif + + +#if defined(__aarch64__) && (defined(__clang__) || defined(__GNUC__)) && !defined(_WIN32) +/* ── SHA-512 kernel (arm64) ─────────────────────────────────────────── + * On CPUs with the SHA-512 extension (FEAT_SHA512), the blocks run with + * SHA512H, SHA512H2, SHA512SU0 and SHA512SU1; other CPUs use the portable + * blocks. The CPU check runs once and is cached. */ +#include +#if defined(__APPLE__) +#include +#else +#include +#endif +#define SCR_SHA512_ARM 1 +#if defined(__clang__) +#define SCR_SHA512_ARM_TARGET __attribute__((target("sha3"))) +#else +#define SCR_SHA512_ARM_TARGET __attribute__((target("+sha3"))) +#endif + +static int scr_sha512_arm_state; /* 0 unknown, 1 available, 2 absent */ + +static bool scr_sha512_arm(void) { + if (scr_sha512_arm_state == 0) { + bool ok; +#if defined(__APPLE__) + int value = 0; + size_t size = sizeof value; + ok = sysctlbyname("hw.optional.armv8_2_sha512", &value, &size, NULL, 0) == 0 && value != 0; +#else + ok = (getauxval(AT_HWCAP) & (1ul << 21)) != 0; /* HWCAP_SHA512 */ +#endif + scr_sha512_arm_state = ok ? 1 : 2; + } + return scr_sha512_arm_state == 1; +} + +/* Two rounds per step. With the state pairs in the roles (A, B, C, D) = + * (ab, cd, ef, gh), a step computes I = SHA512H(swap(K + W) + D, ext(C, D), + * ext(B, C)), D = SHA512H2(I, B, A) and B += I; the roles then rotate to + * (D, A, B, C). */ +#define SCR_SHA512_ARM_STEP(A, B, C, D, msg, k) \ + do { \ + uint64x2_t kw = vaddq_u64((msg), vld1q_u64(k)); \ + uint64x2_t sum = vaddq_u64(vextq_u64(kw, kw, 1), (D)); \ + uint64x2_t i = vsha512hq_u64(sum, vextq_u64((C), (D), 1), vextq_u64((B), (C), 1)); \ + (D) = vsha512h2q_u64(i, (B), (A)); \ + (B) = vaddq_u64((B), i); \ + } while (0) + +/* Message words 2j and 2j+1 from the eight pairs before them. */ +#define SCR_SHA512_ARM_SCHEDULE(m0, m1, m4, m5, m7) \ + (m0) = vsha512su1q_u64(vsha512su0q_u64((m0), (m1)), (m7), vextq_u64((m4), (m5), 1)) + +SCR_SHA512_ARM_TARGET static void scr_sha512_blocks_arm(uint64_t state[8], const unsigned char *p, + size_t blocks) { + uint64x2_t ab = vld1q_u64(&state[0]), cd = vld1q_u64(&state[2]); + uint64x2_t ef = vld1q_u64(&state[4]), gh = vld1q_u64(&state[6]); + for (; blocks != 0; blocks--, p += 128) { + uint64x2_t ab0 = ab, cd0 = cd, ef0 = ef, gh0 = gh; + uint64x2_t m0 = vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(p + 0))); + uint64x2_t m1 = vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(p + 16))); + uint64x2_t m2 = vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(p + 32))); + uint64x2_t m3 = vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(p + 48))); + uint64x2_t m4 = vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(p + 64))); + uint64x2_t m5 = vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(p + 80))); + uint64x2_t m6 = vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(p + 96))); + uint64x2_t m7 = vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(p + 112))); + const uint64_t *k = scr_sha512_k; + for (int i = 0; i < 5; i++, k += 16) { + SCR_SHA512_ARM_STEP(ab, cd, ef, gh, m0, k + 0); + SCR_SHA512_ARM_STEP(gh, ab, cd, ef, m1, k + 2); + SCR_SHA512_ARM_STEP(ef, gh, ab, cd, m2, k + 4); + SCR_SHA512_ARM_STEP(cd, ef, gh, ab, m3, k + 6); + SCR_SHA512_ARM_STEP(ab, cd, ef, gh, m4, k + 8); + SCR_SHA512_ARM_STEP(gh, ab, cd, ef, m5, k + 10); + SCR_SHA512_ARM_STEP(ef, gh, ab, cd, m6, k + 12); + SCR_SHA512_ARM_STEP(cd, ef, gh, ab, m7, k + 14); + if (i == 4) break; + SCR_SHA512_ARM_SCHEDULE(m0, m1, m4, m5, m7); + SCR_SHA512_ARM_SCHEDULE(m1, m2, m5, m6, m0); + SCR_SHA512_ARM_SCHEDULE(m2, m3, m6, m7, m1); + SCR_SHA512_ARM_SCHEDULE(m3, m4, m7, m0, m2); + SCR_SHA512_ARM_SCHEDULE(m4, m5, m0, m1, m3); + SCR_SHA512_ARM_SCHEDULE(m5, m6, m1, m2, m4); + SCR_SHA512_ARM_SCHEDULE(m6, m7, m2, m3, m5); + SCR_SHA512_ARM_SCHEDULE(m7, m0, m3, m4, m6); + } + ab = vaddq_u64(ab, ab0); + cd = vaddq_u64(cd, cd0); + ef = vaddq_u64(ef, ef0); + gh = vaddq_u64(gh, gh0); + } + vst1q_u64(&state[0], ab); + vst1q_u64(&state[2], cd); + vst1q_u64(&state[4], ef); + vst1q_u64(&state[6], gh); +} +#endif + +static void scr_sha512_blocks(uint64_t h[8], const unsigned char *p, size_t blocks) { +#ifdef SCR_SHA_NI + switch (scr_sha512_x86()) { + case 3: + scr_sha512_blocks_avx512(h, p, blocks); + return; + case 2: + scr_sha512_blocks_avx2(h, p, blocks); + return; + default: + break; + } +#endif +#ifdef SCR_SHA512_ARM + if (scr_sha512_arm()) { + scr_sha512_blocks_arm(h, p, blocks); + return; + } +#endif + scr_sha512_blocks_portable(h, p, blocks); } static void scr_sha512_init(ScrSha512Ctx *ctx, bool sha384) { @@ -7456,14 +7746,15 @@ static void scr_sha512_update(ScrSha512Ctx *ctx, const unsigned char *data, size data += take; len -= take; if (ctx->tail_len == 128) { - scr_sha512_block(ctx->h, ctx->tail); + scr_sha512_blocks(ctx->h, ctx->tail, 1); ctx->tail_len = 0; } } - while (len >= 128) { - scr_sha512_block(ctx->h, data); - data += 128; - len -= 128; + if (len >= 128) { + size_t blocks = len / 128; + scr_sha512_blocks(ctx->h, data, blocks); + data += blocks * 128; + len -= blocks * 128; } if (len != 0) { memcpy(ctx->tail, data, len); @@ -7482,8 +7773,7 @@ static size_t scr_sha512_final(const ScrSha512Ctx *source, unsigned char out[64] tail[pad - 1 - b] = (unsigned char)(low >> (8 * b)); tail[pad - 9 - b] = (unsigned char)(high >> (8 * b)); } - scr_sha512_block(ctx.h, tail); - if (pad == 256) scr_sha512_block(ctx.h, tail + 128); + scr_sha512_blocks(ctx.h, tail, pad / 128); size_t len = sha384 ? 48 : 64; for (size_t i = 0; i < len; i++) out[i] = (unsigned char)(ctx.h[i / 8] >> (56 - 8 * (i % 8))); return len; diff --git a/packages/runtime/test/sha-kernels.test.ts b/packages/runtime/test/sha-kernels.test.ts index 724841fb1..6f7759348 100644 --- a/packages/runtime/test/sha-kernels.test.ts +++ b/packages/runtime/test/sha-kernels.test.ts @@ -42,7 +42,7 @@ const runtimeSources = [ ]; // Compiled once with ASan, UBSan and the RC audit. The test runs the SHA-NI and portable -// SHA-1/SHA-256 blocks over the same inputs and checks known vectors. +// SHA-1/SHA-256 blocks and the SHA-512 kernels over the same inputs and checks known vectors. beforeAll(async () => { await mkdir(join(testDir, "build"), { recursive: true }); await execFileAsync("clang", [ @@ -64,7 +64,7 @@ beforeAll(async () => { ]); }, 300_000); -test("SHA-1 and SHA-256: the SHA-NI blocks match the portable blocks", async () => { +test("SHA-1, SHA-256, SHA-384 and SHA-512: the CPU kernels match the portable blocks", async () => { const { stderr } = await execFileAsync(bin, [], { env: { ...process.env, UBSAN_OPTIONS: "halt_on_error=1" }, }); diff --git a/packages/runtime/test/test_sha_kernels.c b/packages/runtime/test/test_sha_kernels.c index 7f854216e..f9f9b9df7 100644 --- a/packages/runtime/test/test_sha_kernels.c +++ b/packages/runtime/test/test_sha_kernels.c @@ -1,9 +1,11 @@ -/* SHA-1 and SHA-256 block kernels (scr_lib.c), built by +/* SHA-1, SHA-256, SHA-384 and SHA-512 block kernels (scr_lib.c), built by * sha-kernels.test.ts. The unit is included directly so the test can run - * the SHA-NI blocks and the portable blocks over the same inputs: known - * vectors, every length from 0 to 1100 bytes, and 1 MiB, each fed whole - * and in 1-, 7-, 64- and 100-byte updates. On CPUs without the SHA - * extensions only the portable blocks run. */ + * each kernel the CPU supports and the portable blocks over the same + * inputs: known vectors, every length from 0 to 1100 bytes, and 1 MiB, + * each fed whole and in 1-, 7-, 64- and 100-byte updates. The SHA-1 and + * SHA-256 kernels use the SHA extensions; the SHA-512 kernels use AVX2 or + * AVX-512VL on x86-64 and the SHA-512 extension on arm64. On CPUs without + * them only the portable blocks run. */ #include "../src/scr_lib.c" #include @@ -45,6 +47,71 @@ static void sha1_of(const unsigned char *data, size_t len, size_t chunk, unsigne scr_sha1_final(&ctx, out); } +static void sha512_of(const unsigned char *data, size_t len, size_t chunk, bool sha384, + unsigned char out[64]) { + ScrSha512Ctx ctx; + scr_sha512_init(&ctx, sha384); + for (size_t at = 0; at < len;) { + size_t take = chunk == 0 || chunk > len - at ? len - at : chunk; + scr_sha512_update(&ctx, data + at, take); + at += take; + } + scr_sha512_final(&ctx, out, sha384); +} + +/* FIPS 180-4 example vectors for SHA-384 and SHA-512. */ +static void vectors512(const char *path) { + unsigned char d[64]; + char text[129]; + static const char two_block[] = + "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"; + sha512_of((const unsigned char *)"abc", 3, 0, false, d); + hex(d, 64, text); + check(strcmp(text, "ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f") == 0, path, 3, 0); + sha512_of((const unsigned char *)"", 0, 0, false, d); + hex(d, 64, text); + check(strcmp(text, "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e") == 0, path, 0, 0); + sha512_of((const unsigned char *)two_block, sizeof two_block - 1, 0, false, d); + hex(d, 64, text); + check(strcmp(text, "8e959b75dae313da8cf4f72814fc143f8f7779c6eb9f7fa17299aeadb6889018501d289e4900f7e4331b99dec4b5433ac7d329eeb6dd26545e96e55b874be909") == 0, path, sizeof two_block - 1, 0); + sha512_of((const unsigned char *)"abc", 3, 0, true, d); + hex(d, 48, text); + check(strcmp(text, "cb00753f45a35e8bb5a03d699ac65007272c32ab0eded1631a8b605a43ff5bed8086072ba1e7cc2358baeca134c825a7") == 0, path, 3, 0); + sha512_of((const unsigned char *)two_block, sizeof two_block - 1, 0, true, d); + hex(d, 48, text); + check(strcmp(text, "09330c33f71147e83d192fc782cd1b4753111b173b3b05d22fa08086e3b0f712fcc7c71a557e2db966c3e9fa91746039") == 0, path, sizeof two_block - 1, 0); + static unsigned char million[1000000]; + memset(million, 'a', sizeof million); + sha512_of(million, sizeof million, 0, false, d); + hex(d, 64, text); + check(strcmp(text, "e718483d0ce769644e2e42c7bc15b4638e1f98b13b2044285632a803afa973ebde0ff244877ea60a4cb0432ce577c31beb009c5c2c49aa2e4eadb217ad8cc09b") == 0, path, sizeof million, 0); + sha512_of(million, sizeof million, 0, true, d); + hex(d, 48, text); + check(strcmp(text, "9d0e1809716474cb086e834e310a4a1ced149e9c00f248527972cec5704c2a5b07b8b3dc38ecc4ebae97ddd87f3d8985") == 0, path, sizeof million, 0); +} + +/* Selects the SHA-512 kernel: 0 the portable blocks, otherwise the CPU's + * kernel. Returns false when the CPU has none. */ +static bool select512(int kernel) { +#ifdef SCR_SHA_NI + scr_sha512_x86_state = 0; + int best = scr_sha512_x86(); + if (kernel > best - 1) return false; + scr_sha512_x86_state = kernel == 0 ? 1 : kernel + 1; + return true; +#elif defined(SCR_SHA512_ARM) + scr_sha512_arm_state = 0; + bool available = scr_sha512_arm(); + if (kernel > (available ? 1 : 0)) return false; + scr_sha512_arm_state = kernel == 0 ? 2 : 1; + return true; +#else + return kernel == 0; +#endif +} + +static const char *const kernel512_names[] = {"portable", "avx2-or-arm", "avx512"}; + static void vectors(const char *path) { unsigned char d[32]; char text[65]; @@ -108,8 +175,31 @@ int main(void) { vectors("ni"); } #endif + /* SHA-512 and SHA-384: every kernel against the portable blocks, and + * the known vectors through each. */ + int kernels512 = 0; + for (int k = 0; k < 3 && select512(k); k++) { + kernels512 = k + 1; + vectors512(kernel512_names[k]); + } + for (int k = 1; k < kernels512; k++) { + for (size_t len = 0; len <= 1100 + 1; len++) { + size_t n = len == 1101 ? BIG : len; + for (size_t c = 0; c < sizeof chunks / sizeof chunks[0]; c++) { + for (int sha384 = 0; sha384 < 2; sha384++) { + unsigned char a[64], b[64]; + select512(k); + sha512_of(data, n, chunks[c], sha384, a); + select512(0); + sha512_of(data, n, chunks[c], sha384, b); + check(memcmp(a, b, 64) == 0, kernel512_names[k], n, chunks[c]); + } + } + } + } free(data); fprintf(stderr, "sha-ni %s\n", ni ? "available" : "absent"); + fprintf(stderr, "sha-512 kernels %d\n", kernels512); fprintf(stderr, "%ld/%ld cases passed\n", total - failed, total); return failed != 0; } diff --git a/tests/corpus/5779-sha512-lengths.ts b/tests/corpus/5779-sha512-lengths.ts new file mode 100644 index 000000000..277408409 --- /dev/null +++ b/tests/corpus/5779-sha512-lengths.ts @@ -0,0 +1,35 @@ +// SHA-384 and SHA-512 over every length from 0 to 520 bytes, fed whole and +// in 1-, 7-, 100- and 128-byte updates, then over 1 MiB, and HMAC-SHA512 +// over a few lengths. Each length's digests are folded into one running +// SHA-512 so the output stays short. The CPU's SHA-512 kernel (vector or +// SHA-512 instructions) and the portable blocks give the same digests. +import { createHash, createHmac } from "node:crypto"; + +const data = Buffer.alloc(1 << 20); +let x = 0x9e3779b9; +for (let i = 0; i < data.length; i++) { + x ^= x << 13; + x ^= x >>> 17; + x ^= x << 5; + data[i] = x & 0xff; +} + +function digest(algorithm: string, input: Buffer, chunk: number): Buffer { + const hash = createHash(algorithm); + if (chunk === 0) hash.update(input); + else for (let at = 0; at < input.length; at += chunk) hash.update(input.subarray(at, at + chunk)); + return hash.digest(); +} + +for (const algorithm of ["sha384", "sha512"]) { + for (const chunk of [0, 1, 7, 100, 128]) { + const fold = createHash("sha512"); + for (let length = 0; length <= 520; length++) fold.update(digest(algorithm, data.subarray(0, length), chunk)); + console.log(algorithm, "chunk", chunk, fold.digest("hex").slice(0, 32)); + } + console.log(algorithm, "1 MiB", digest(algorithm, data, 0).toString("hex").slice(0, 32), digest(algorithm, data, 4096).toString("hex").slice(0, 32)); +} +console.log("abc", createHash("sha512").update("abc").digest("hex")); +for (const length of [0, 1, 111, 112, 127, 128, 129, 255, 256, 1000]) { + console.log("hmac", length, createHmac("sha512", "key").update(data.subarray(0, length)).digest("hex").slice(0, 32)); +} diff --git a/tests/harness/corpus-costs/5779-sha512-lengths.ts.json b/tests/harness/corpus-costs/5779-sha512-lengths.ts.json new file mode 100644 index 000000000..9e04a316c --- /dev/null +++ b/tests/harness/corpus-costs/5779-sha512-lengths.ts.json @@ -0,0 +1 @@ +{"plain":2610,"san":3540} From d02c85a6119ee2b64b61362a2884157a27f937e2 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Mon, 19 Oct 2026 21:23:53 -0700 Subject: [PATCH 20/35] Use classes that extend EventEmitter or a stream class as values Naming a class that extends EventEmitter, Readable, Writable, Duplex or Transform as a value was refused ("classes extending builtin bases as values"), and so was a class expression of that kind. The refusal also showed up as a deferred runtime fence at the class declaration, because the checked-dynamic tier reflects every class as a value. A user subclass always constructs through its own or synthesized constructor, whose super call runs the runtime initializer, so the class object's construct thunk serves it like any program class. EventTarget subclasses keep the refusal. Co-Authored-By: Claude Opus 5.5 --- .../src/frontend/lowering/lower-classes.ts | 21 ++----- .../6450-builtin-base-class-values.ts.json | 6 ++ ...51-builtin-base-class-expressions.cjs.json | 6 ++ .../corpus/6450-builtin-base-class-values.ts | 63 +++++++++++++++++++ .../6451-builtin-base-class-expressions.cjs | 33 ++++++++++ 5 files changed, 114 insertions(+), 15 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6450-builtin-base-class-values.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6451-builtin-base-class-expressions.cjs.json create mode 100644 tests/corpus/6450-builtin-base-class-values.ts create mode 100644 tests/corpus/6451-builtin-base-class-expressions.cjs diff --git a/packages/compiler/src/frontend/lowering/lower-classes.ts b/packages/compiler/src/frontend/lowering/lower-classes.ts index 8a5170a7a..81cf09e1f 100644 --- a/packages/compiler/src/frontend/lowering/lower-classes.ts +++ b/packages/compiler/src/frontend/lowering/lower-classes.ts @@ -2212,11 +2212,7 @@ export function collectClassShapeInner( } if (lowerer.collectingExprClasses.has(decl)) { for (let parent = base; parent; parent = parent.base) { - if ( - parent.builtinEmitter || - parent.builtinStream !== undefined || - parent.builtinEventTarget !== undefined - ) { + if (parent.builtinEventTarget !== undefined) { lowerer.unsupported( "SC1090", decl, @@ -5132,16 +5128,11 @@ export function classValueRef(lowerer: Lowerer, info: ClassInfo, blame: ts.Node) loc: locOf(blame), }; } - if ( - lowerer.inheritsBuiltinEmitterCtor(info) || - inheritsBuiltinStreamCtor(lowerer, info) || - eventTargetRootOf(info) !== null || - (() => { - for (let c = info.base; c; c = c.base) - if (c.builtinEmitter || c.builtinStream !== undefined) return true; - return false; - })() - ) { + // Classes extending EventEmitter or a stream class construct through + // their own (or synthesized) `%C.constructor`, whose super call runs the + // runtime initializer, so the class object's construct thunk serves them + // like any program class. EventTarget subclasses keep the refusal. + if (eventTargetRootOf(info) !== null) { lowerer.unsupported( "SC1090", blame, diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6450-builtin-base-class-values.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6450-builtin-base-class-values.ts.json new file mode 100644 index 000000000..bdd8fcbc8 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6450-builtin-base-class-values.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6450-builtin-base-class-values.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6451-builtin-base-class-expressions.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6451-builtin-base-class-expressions.cjs.json new file mode 100644 index 000000000..34cde5811 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6451-builtin-base-class-expressions.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6451-builtin-base-class-expressions.cjs" + ], + "diags": [] +} diff --git a/tests/corpus/6450-builtin-base-class-values.ts b/tests/corpus/6450-builtin-base-class-values.ts new file mode 100644 index 000000000..0dafa5d82 --- /dev/null +++ b/tests/corpus/6450-builtin-base-class-values.ts @@ -0,0 +1,63 @@ +// Classes extending Node builtin bases (Writable, Transform, EventEmitter) +// used as values: through a variable, a generic factory, an object table and +// an array, with instanceof against the class and its base. +import { EventEmitter } from "node:events"; +import { Readable, Transform, Writable } from "node:stream"; + +class Sink extends Writable { + readonly lines: string[] = []; + private readonly tag: string; + constructor(tag: string) { + super(); + this.tag = tag; + } + _write(chunk: Buffer, _enc: BufferEncoding, cb: (err?: Error | null) => void): void { + this.lines.push(this.tag + chunk.toString()); + cb(); + } +} + +class Upper extends Transform { + _transform(chunk: Buffer, _enc: BufferEncoding, cb: (err?: Error | null, data?: string) => void): void { + cb(null, chunk.toString().toUpperCase()); + } +} + +class Ticker extends EventEmitter { + count = 0; + tick(): void { + this.count++; + this.emit("tick", this.count); + } +} + +class Plain extends EventEmitter {} + +function build(ctor: new () => T): T { + return new ctor(); +} + +const SinkClass = Sink; +const s = new SinkClass("> "); +s.write("a"); +s.end("b", () => console.log("sink", JSON.stringify(s.lines), s instanceof Sink, s instanceof Writable)); + +const t = build(Ticker); +t.on("tick", (n: number) => console.log("tick", n)); +t.tick(); +t.tick(); +console.log("ticker", t instanceof Ticker, t instanceof EventEmitter, t.count); + +const p = build(Plain); +p.on("x", () => console.log("plain x")); +p.emit("x"); + +const registry = { upper: Upper }; +const u = new registry.upper(); +u.on("data", (d: Buffer) => console.log("upper", d.toString())); +u.end("hello"); + +const kinds = [Ticker, Ticker]; +console.log("names", kinds.map((k) => k.name).join(","), typeof Sink, Sink.name); +const r = Readable.from(["x"]); +console.log("readable", r instanceof Readable); diff --git a/tests/corpus/6451-builtin-base-class-expressions.cjs b/tests/corpus/6451-builtin-base-class-expressions.cjs new file mode 100644 index 000000000..64230a679 --- /dev/null +++ b/tests/corpus/6451-builtin-base-class-expressions.cjs @@ -0,0 +1,33 @@ +// Class expressions extending Node builtin bases (a bundler's output shape). +"use strict"; +const { Readable, Transform } = require("stream"); +const { EventEmitter } = require("events"); + +const Wrap = class extends Readable { + constructor(parts) { + super(); + this.parts = parts; + } + _read() { + const next = this.parts.shift(); + this.push(next === undefined ? null : next); + } +}; +const Bus = class extends EventEmitter {}; +const Lower = class extends Transform { + _transform(chunk, _enc, cb) { + cb(null, chunk.toString().toLowerCase()); + } +}; + +const r = new Wrap(["a", "b"]); +const chunks = []; +r.on("data", (c) => chunks.push(c.toString())); +r.on("end", () => console.log("wrap", chunks.join("+"), r instanceof Readable)); +const bus = new Bus(); +bus.on("ping", (n) => console.log("ping", n)); +bus.emit("ping", 1); +const l = new Lower(); +l.on("data", (d) => console.log("lower", d.toString())); +l.end("HeLLo"); +console.log(Wrap.name, Bus.name, typeof Lower); From ba23a111105c1d3dbd99caaab7d79fd0c7935ee4 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Mon, 19 Oct 2026 22:44:14 -0700 Subject: [PATCH 21/35] Format the npm case filter Co-Authored-By: Claude Opus 5.5 --- tests/harness/npm-cases.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/harness/npm-cases.ts b/tests/harness/npm-cases.ts index f77807d83..01718abf9 100644 --- a/tests/harness/npm-cases.ts +++ b/tests/harness/npm-cases.ts @@ -24,7 +24,9 @@ export function npmCases(fixturesRoot: string): NpmCase[] { // consumers exercise the static rewrite's surface (lexer-visible // names the shipped .d.ts never declares, the __toESM interop // family), so they stay out of the flagless island lane by design. - .filter((entry) => !/\/(246[5-9]|255[6-9]|256[0-7]|2571|258[0-4])-[^/]+\/main\.ts$/.test(entry)) + .filter( + (entry) => !/\/(246[5-9]|255[6-9]|256[0-7]|2571|258[0-4])-[^/]+\/main\.ts$/.test(entry), + ) .map((entry) => ({ name: entry.split("/").at(-2)!, entry, From 4b4f80581e3267a5c759a7c477d730d79311e52d Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Mon, 19 Oct 2026 22:44:14 -0700 Subject: [PATCH 22/35] Format the npm case filter Co-Authored-By: Claude Opus 5.5 --- tests/harness/npm-cases.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/harness/npm-cases.ts b/tests/harness/npm-cases.ts index 1d15ddc6b..ef46a59de 100644 --- a/tests/harness/npm-cases.ts +++ b/tests/harness/npm-cases.ts @@ -24,7 +24,9 @@ export function npmCases(fixturesRoot: string): NpmCase[] { // consumers exercise the static rewrite's surface (lexer-visible // names the shipped .d.ts never declares, the __toESM interop // family), so they stay out of the flagless island lane by design. - .filter((entry) => !/\/(246[5-9]|255[6-9]|256[0-7]|2571|258[0-2])-[^/]+\/main\.ts$/.test(entry)) + .filter( + (entry) => !/\/(246[5-9]|255[6-9]|256[0-7]|2571|258[0-2])-[^/]+\/main\.ts$/.test(entry), + ) .map((entry) => ({ name: entry.split("/").at(-2)!, entry, From 6197a45ab0796615bca979264168ce376be4a547 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 20 Oct 2026 01:48:22 -0500 Subject: [PATCH 23/35] Refuse arguments through class values of ctor-less stream subclasses A class without a constructor that extends a stream class or EventEmitter inherits the runtime constructor, which takes no arguments here. Direct construction with arguments already reported that, but construction through the class's value (`new registry.upper({ ... })`) completed the arguments against the empty inherited signature and dropped them. Both paths now share refuseInheritedBuiltinCtorArgs, so the class-value path reports at the first argument with the same message. --- .../src/frontend/lowering/lower-classes.ts | 48 ++++++++++++------- ...builtin-base-class-value-arguments.ts.json | 6 +++ .../builtin-base-class-value-arguments.ts | 14 ++++++ .../builtin-base-class-value-arguments.ts.txt | 17 +++++++ 4 files changed, 67 insertions(+), 18 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/builtin-base-class-value-arguments.ts.json create mode 100644 tests/diagnostics/builtin-base-class-value-arguments.ts create mode 100644 tests/harness/__snapshots__/builtin-base-class-value-arguments.ts.txt diff --git a/packages/compiler/src/frontend/lowering/lower-classes.ts b/packages/compiler/src/frontend/lowering/lower-classes.ts index 81cf09e1f..7f86de06c 100644 --- a/packages/compiler/src/frontend/lowering/lower-classes.ts +++ b/packages/compiler/src/frontend/lowering/lower-classes.ts @@ -8605,6 +8605,32 @@ function inheritsBuiltinStreamCtor(lowerer: Lowerer, info: ClassInfo): boolean { return false; } +/** `new C(args)` where C's constructor is the inherited runtime stream or + * EventEmitter constructor (a ctor-less chain into the base): that + * constructor takes zero arguments here (the options bag fences, like the + * super() form), so arguments report at the first one. Applies to direct + * construction and to construction through a class value of C. Stream + * subclasses come first: their chain roots at the emitter too, but the + * message should name the stream story. */ +function refuseInheritedBuiltinCtorArgs( + lowerer: Lowerer, + info: ClassInfo, + expr: ts.NewExpression, +): void { + const first = expr.arguments?.nodes[0]; + if (first === undefined) return; + if (inheritsBuiltinStreamCtor(lowerer, info)) { + lowerer.noLowering( + `new ${info.def.name.replace(/^%/, "")} with arguments through an inherited stream constructor`, + first, + "declare a constructor that passes an inline options object to super(...)", + ); + } + if (lowerer.inheritsBuiltinEmitterCtor(info)) { + lowerer.unsupported("SC1090", first, "EventEmitter constructor options ('captureRejections')"); + } +} + /** Preserve argument evaluation order and option presence through builtin * and inherited Error constructors. Checked-dynamic conversion retains * Error identity and distinguishes a missing cause from undefined. */ @@ -8729,24 +8755,7 @@ function lowerProgramClassNew( "construction through a function-local class's internal name", ); lowerer.noteEdge(`%${info.def.name}.constructor`); - // A ctor-less chain into an EventEmitter base inherits `new C()` — - // zero arguments (the options bag fences, like the super() form). - // Stream subclasses come first: their chain roots at the emitter - // too, but the message should name the stream story. - if (inheritsBuiltinStreamCtor(lowerer, info) && (expr.arguments?.nodes ?? []).length > 0) { - lowerer.noLowering( - `new ${info.def.name.replace(/^%/, "")} with arguments through an inherited stream constructor`, - expr.arguments!.nodes[0]!, - "declare a constructor that passes an inline options object to super(...)", - ); - } - if (lowerer.inheritsBuiltinEmitterCtor(info) && (expr.arguments?.nodes ?? []).length > 0) { - lowerer.unsupported( - "SC1090", - expr.arguments!.nodes[0]!, - "EventEmitter constructor options ('captureRejections')", - ); - } + refuseInheritedBuiltinCtorArgs(lowerer, info, expr); // A ctor-less chain into a builtin error base inherits both arguments; // each synthesized constructor forwards the same pair to its base. const args = lowerer.inheritsBuiltinErrorCtor(info) @@ -9994,6 +10003,9 @@ export function lowerNew(lowerer: Lowerer, expr: ts.NewExpression): IrExpr { `EventEmitter and stream class values constructed with options (construct with 'new ${info.def.name.slice(1)}({ ... })' directly)`, ); } + // A program class whose constructor is the inherited runtime one + // takes no arguments through its class value either. + refuseInheritedBuiltinCtorArgs(lowerer, info, expr); defineClassValueConstruct(lowerer, info); const args = lowerer.completeArgs( expr.arguments?.nodes ?? [], diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/builtin-base-class-value-arguments.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/builtin-base-class-value-arguments.ts.json new file mode 100644 index 000000000..622882c24 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/builtin-base-class-value-arguments.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/diagnostics/builtin-base-class-value-arguments.ts" + ], + "diags": [] +} diff --git a/tests/diagnostics/builtin-base-class-value-arguments.ts b/tests/diagnostics/builtin-base-class-value-arguments.ts new file mode 100644 index 000000000..db5be873b --- /dev/null +++ b/tests/diagnostics/builtin-base-class-value-arguments.ts @@ -0,0 +1,14 @@ +// A class without a constructor that extends a stream class inherits the +// runtime constructor, which takes no arguments here. Passing arguments +// through the class's value reports at the first argument, as direct +// construction does. +import { Transform } from "node:stream"; + +class Upper extends Transform {} +const registry = { upper: Upper }; +const upper = new registry.upper({ objectMode: true }); +upper.destroy(); + +const Kind = Upper; +const other = new Kind({ highWaterMark: 4 }); +other.destroy(); diff --git a/tests/harness/__snapshots__/builtin-base-class-value-arguments.ts.txt b/tests/harness/__snapshots__/builtin-base-class-value-arguments.ts.txt new file mode 100644 index 000000000..5e3af4f45 --- /dev/null +++ b/tests/harness/__snapshots__/builtin-base-class-value-arguments.ts.txt @@ -0,0 +1,17 @@ +builtin-base-class-value-arguments.ts:9:34 - error SC2020: 'new Upper with arguments through an inherited stream constructor' is part of the standard library types but has no scriptc lowering yet + + 8 | const registry = { upper: Upper }; + 9 | const upper = new registry.upper({ objectMode: true }); + | ^~~~~~~~~~~~~~~~~~~~ + 10 | upper.destroy(); + + hint: declare a constructor that passes an inline options object to super(...) + +builtin-base-class-value-arguments.ts:13:24 - error SC2020: 'new Upper with arguments through an inherited stream constructor' is part of the standard library types but has no scriptc lowering yet + + 12 | const Kind = Upper; + 13 | const other = new Kind({ highWaterMark: 4 }); + | ^~~~~~~~~~~~~~~~~~~~ + 14 | other.destroy(); + + hint: declare a constructor that passes an inline options object to super(...) \ No newline at end of file From 44f4e2e74c39f1ad1ad3fed7021a7547ed283183 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Mon, 19 Oct 2026 22:35:23 -0700 Subject: [PATCH 24/35] Refuse inline stream completion callbacks that name the error Node passes write and end callbacks an error argument (null on success), but the native callbacks receive none, so an inline callback that names the parameter read undefined. It now refuses where it is written. The stream-shims npm case compiles now that stream subclasses are values; its remaining scenarios are deferred refusals, so it is recorded as one. Co-Authored-By: Claude Opus 5.5 --- .../src/frontend/lowering/lower-stream.ts | 26 ++++++++++++++++--- .../stream-completion-callback-error.ts.json | 6 +++++ .../6451-builtin-base-class-expressions.cjs | 13 ++++++++++ .../stream-completion-callback-error.ts | 11 ++++++++ .../stream-completion-callback-error.ts.txt | 17 ++++++++++++ tests/harness/inheritance.test.ts | 20 ++++++-------- tests/harness/npm.test.ts | 2 +- 7 files changed, 78 insertions(+), 17 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/stream-completion-callback-error.ts.json create mode 100644 tests/diagnostics/stream-completion-callback-error.ts create mode 100644 tests/harness/__snapshots__/stream-completion-callback-error.ts.txt diff --git a/packages/compiler/src/frontend/lowering/lower-stream.ts b/packages/compiler/src/frontend/lowering/lower-stream.ts index 5769e1e0a..64d2e60e8 100644 --- a/packages/compiler/src/frontend/lowering/lower-stream.ts +++ b/packages/compiler/src/frontend/lowering/lower-stream.ts @@ -3654,8 +3654,10 @@ export function lowerStreamMethodCall( : lowerer.lowerExprExpecting(args[0]!, STRING); const fn = kind === "bytes" ? "writable.write" : "writable.writeStr"; if (cbArg !== undefined) { - const cb = lowerer.lowerExprExpecting(cbArg, funcOf([], VOID)); - if (cb.type.kind !== "func" || cb.type.params.length > 0) { + const cb = callbackDeclaresParameters(cbArg) + ? null + : lowerer.lowerExprExpecting(cbArg, funcOf([], VOID)); + if (cb === null || cb.type.kind !== "func" || cb.type.params.length > 0) { lowerer.noLowering( "write completion callbacks taking the error argument", cbArg, @@ -3712,8 +3714,10 @@ export function lowerStreamMethodCall( ); } if (cbArg !== undefined) { - const cb = lowerer.lowerExprExpecting(cbArg, funcOf([], VOID)); - if (cb.type.kind !== "func" || cb.type.params.length > 0) { + const cb = callbackDeclaresParameters(cbArg) + ? null + : lowerer.lowerExprExpecting(cbArg, funcOf([], VOID)); + if (cb === null || cb.type.kind !== "func" || cb.type.params.length > 0) { lowerer.noLowering( "end callbacks with parameters", cbArg, @@ -3978,3 +3982,17 @@ export function lowerStreamProperty( } return null; } + +/** Whether an inline completion callback declares a parameter. Node calls + * write and end callbacks with an error argument (null on success), but the + * native callback receives none, so the parameter would read undefined + * where Node passes null or the error. A function literal with parameters + * would otherwise adapt to the zero-parameter slot silently. */ +function callbackDeclaresParameters(node: ts.Expression): boolean { + let callback = node; + while (ts.isParenthesizedExpression(callback)) callback = callback.expression; + return ( + (ts.isArrowFunction(callback) || ts.isFunctionExpression(callback)) && + callback.parameters.nodes.length > 0 + ); +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/stream-completion-callback-error.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/stream-completion-callback-error.ts.json new file mode 100644 index 000000000..af31bb8cf --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/stream-completion-callback-error.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/diagnostics/stream-completion-callback-error.ts" + ], + "diags": [] +} diff --git a/tests/corpus/6451-builtin-base-class-expressions.cjs b/tests/corpus/6451-builtin-base-class-expressions.cjs index 64230a679..6d9241b97 100644 --- a/tests/corpus/6451-builtin-base-class-expressions.cjs +++ b/tests/corpus/6451-builtin-base-class-expressions.cjs @@ -31,3 +31,16 @@ const l = new Lower(); l.on("data", (d) => console.log("lower", d.toString())); l.end("HeLLo"); console.log(Wrap.name, Bus.name, typeof Lower); +{ + // A class declared in a block is collected like a class expression. + class Once extends Readable { + _read() { + this.push(null); + } + } + function onReadable() { + console.log("block", stream.read() === null); + } + const stream = new Once(); + stream.once("readable", onReadable); +} diff --git a/tests/diagnostics/stream-completion-callback-error.ts b/tests/diagnostics/stream-completion-callback-error.ts new file mode 100644 index 000000000..eb73b485d --- /dev/null +++ b/tests/diagnostics/stream-completion-callback-error.ts @@ -0,0 +1,11 @@ +import { Writable } from "node:stream"; + +// Node passes write and end callbacks an error argument (null on success); +// the native callbacks receive none, so a callback that names it refuses. +const sink = new Writable({ + write(_chunk, _encoding, done) { + done(); + }, +}); +sink.write("a", (err) => console.log("written", err)); +sink.end((err?: Error | null) => console.log("ended", err)); diff --git a/tests/harness/__snapshots__/stream-completion-callback-error.ts.txt b/tests/harness/__snapshots__/stream-completion-callback-error.ts.txt new file mode 100644 index 000000000..f3b876cc7 --- /dev/null +++ b/tests/harness/__snapshots__/stream-completion-callback-error.ts.txt @@ -0,0 +1,17 @@ +stream-completion-callback-error.ts:10:17 - error SC2020: 'write completion callbacks taking the error argument' is part of the standard library types but has no scriptc lowering yet + + 9 | }); + 10 | sink.write("a", (err) => console.log("written", err)); + | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + 11 | sink.end((err?: Error | null) => console.log("ended", err)); + + hint: () => void callbacks are supported (listen for 'error' to observe failures) + +stream-completion-callback-error.ts:11:10 - error SC2020: 'end callbacks with parameters' is part of the standard library types but has no scriptc lowering yet + + 10 | sink.write("a", (err) => console.log("written", err)); + 11 | sink.end((err?: Error | null) => console.log("ended", err)); + | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + 12 | + + hint: () => void callbacks are supported \ No newline at end of file diff --git a/tests/harness/inheritance.test.ts b/tests/harness/inheritance.test.ts index c0ce3d7d4..27a176764 100644 --- a/tests/harness/inheritance.test.ts +++ b/tests/harness/inheritance.test.ts @@ -204,14 +204,11 @@ r.on("data", (b: Buffer) => console.log(b.toString())); expect(llvm).toContain("call void @scr_stream_init_readable(ptr "); }); - test("a runtime-fenced nested stream class emits the refusal without class helpers", async () => { - // The phase-1 reachable bug: a class declared inside a block (a - // runtime fence in JS) whose instances are captured emitted capture- - // box RC adapters for the uncollected class — a C compile error. - // run()'s unregistered-class sweep now rewrites such type slots to - // the inert f64 placeholder before emission (no instance can exist; - // every use traps), so the capture box is a PLAIN-kind box and the - // class never reaches the emitter at all. + test("a block-scoped stream class lowers natively with its class helpers", async () => { + // A class declared inside a block is collected like a class expression. + // Its captured instances once needed the unregistered-class sweep (the + // class was fenced); a builtin stream base no longer fences, so the + // class reaches the emitter with its object type and stream init. const llvm = await emittedLlvm( "stream-nested-fence", `const { Readable } = require("stream"); @@ -226,11 +223,10 @@ r.on("data", (b: Buffer) => console.log(b.toString())); `, "js", ); - expect(llvm).not.toContain("sc_retain_R_v"); expect(llvm).not.toContain("uncollected class"); - expect(llvm).not.toContain("%sc_o_R"); - expect(llvm).toMatch(/%sc_l_stream_0 = alloca ptr/); - expect(llvm).toContain("call void @scr_throw_error_msg_code"); + expect(llvm).toContain("%sc_o_R"); + expect(llvm).toContain("call void @scr_stream_init_readable(ptr "); + expect(llvm).not.toContain("call void @scr_throw_error_msg_code"); }); test("a getter-only override of a pair fills the set slot with a thrower", async () => { diff --git a/tests/harness/npm.test.ts b/tests/harness/npm.test.ts index 54207f246..ac3210dee 100644 --- a/tests/harness/npm.test.ts +++ b/tests/harness/npm.test.ts @@ -104,7 +104,7 @@ const KNOWN: Record = { "package-imports": { build: "SC2013" }, "promise-bridge": { build: "SC1090" }, "self-name": { build: "SC2013" }, - "stream-shims": { build: "SC1090" }, + "stream-shims": { run: "SC1090" }, "util-shims": { run: "SC2020" }, "web-streams": { build: "SC2013" }, "workspace-copied": { build: "SC2013" }, From ea611bec028d394e80994e8cb4b9b4629681c999 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 20 Oct 2026 02:52:38 -0500 Subject: [PATCH 25/35] Convert object literals with accessors to unknown again #1490 also took records with accessor slots out of the dyn-convertible domain, which refused passing such a literal where `unknown` is expected. Native APIs read these values through their accessor slots: fileURLToPath, fileURLToPathBuffer and pathToFileURL call the `windows` getter of an options object, so tests/corpus/node24-file-url-options.ts stopped compiling. Records with accessor slots convert to checked-dynamic values again. JSON.stringify of their typed values stays refused, since the typed serializer cannot invoke the getter. --- packages/compiler/src/ir/ir.ts | 3 --- .../object-literal-accessor-values.ts | 14 ++-------- .../object-literal-accessor-values.ts.txt | 26 +++---------------- 3 files changed, 6 insertions(+), 37 deletions(-) diff --git a/packages/compiler/src/ir/ir.ts b/packages/compiler/src/ir/ir.ts index d6ff0fe72..de6bf558c 100644 --- a/packages/compiler/src/ir/ir.ts +++ b/packages/compiler/src/ir/ir.ts @@ -4267,9 +4267,6 @@ function canBoxDynCompositeAt( const shape = getRecord(t.shapeId); if (!shape) return false; if (recordTextCodecClass(shape) !== null) return false; - // Accessor slots hold the getter and setter closures, not property - // values (see accessorSlotProp). - if (shapeHasAccessorSlots(shape)) return false; // Recursive shapes answer coinductively, like isJsonSafeType. if (visiting.has(t.shapeId)) return true; visiting.add(t.shapeId); diff --git a/tests/diagnostics/object-literal-accessor-values.ts b/tests/diagnostics/object-literal-accessor-values.ts index 107ab1f69..e4c456a76 100644 --- a/tests/diagnostics/object-literal-accessor-values.ts +++ b/tests/diagnostics/object-literal-accessor-values.ts @@ -1,11 +1,6 @@ // Object literals with get/set accessors keep their closures in reserved -// slots, so their typed values have no JSON or checked-dynamic form: Node's -// JSON.stringify and dynamic reads invoke the getter, which neither a typed -// serializer nor a converted copy could do. - -function hold(value: unknown): unknown { - return value; -} +// slots, so their typed values have no JSON form: Node's JSON.stringify +// invokes the getter, which the typed serializer cannot do. const counter = { count: 1, @@ -17,11 +12,6 @@ const counter = { // JSON.stringify of the typed value would omit the getter's property. console.log(JSON.stringify(counter)); -// Converting the typed value to `unknown` or `any` would lose the getter. -hold(counter); -const viaAny: any = counter; -console.log(viaAny); - // A literal written into an `any` slot directly keeps its accessor. const direct: any = { get value(): number { diff --git a/tests/harness/__snapshots__/object-literal-accessor-values.ts.txt b/tests/harness/__snapshots__/object-literal-accessor-values.ts.txt index 89bc092aa..372849388 100644 --- a/tests/harness/__snapshots__/object-literal-accessor-values.ts.txt +++ b/tests/harness/__snapshots__/object-literal-accessor-values.ts.txt @@ -1,24 +1,6 @@ -object-literal-accessor-values.ts:18:28 - error SC1090: JSON.stringify of '{ get doubled(): number; count: number }' values (only number, string, boolean, records, arrays, unions of those, and 'unknown' stringify) are not supported yet +object-literal-accessor-values.ts:13:28 - error SC1090: JSON.stringify of '{ get doubled(): number; count: number }' values (only number, string, boolean, records, arrays, unions of those, and 'unknown' stringify) are not supported yet - 17 | // JSON.stringify of the typed value would omit the getter's property. - 18 | console.log(JSON.stringify(counter)); + 12 | // JSON.stringify of the typed value would omit the getter's property. + 13 | console.log(JSON.stringify(counter)); | ^~~~~~~ - 19 | - -object-literal-accessor-values.ts:21:6 - error SC1101: converting typed values to 'unknown' is not supported yet - - 20 | // Converting the typed value to `unknown` or `any` would lose the getter. - 21 | hold(counter); - | ^~~~~~~ - 22 | const viaAny: any = counter; - - hint: numbers, strings, booleans, JSON-safe records/arrays/unions (a deep copy), functions, program class instances, and supported native handles convert into 'unknown' slots; functions, exact-class round trips, and handles preserve identity, while this value's type has no dynamic representation yet - -object-literal-accessor-values.ts:22:21 - error SC1101: converting typed values to 'unknown' is not supported yet - - 21 | hold(counter); - 22 | const viaAny: any = counter; - | ^~~~~~~ - 23 | console.log(viaAny); - - hint: numbers, strings, booleans, JSON-safe records/arrays/unions (a deep copy), functions, program class instances, and supported native handles convert into 'unknown' slots; functions, exact-class round trips, and handles preserve identity, while this value's type has no dynamic representation yet \ No newline at end of file + 14 | \ No newline at end of file From c70d9d7e28d224fc00008696b835cdba213e3477 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Tue, 20 Oct 2026 00:53:19 -0700 Subject: [PATCH 26/35] Refuse returns a generic callback's inferred result would drop members of A callback passed to a generic call without a return annotation can return values the checker unifies into a narrower result type ('return items' beside 'return items.map((i) => ({ ...i, extra }))' infers Item[]). Converting the extended objects into that type copied them and dropped the added members, which Node keeps and a later wider view reads back. Such a return is now refused by name, pointing at the annotation that keeps the members. Co-Authored-By: Claude Opus 5.5 --- .../compiler/src/frontend/lowering/lowerer.ts | 72 +++++++++++++++++++ ...-generic-callback-extended-results.ts.json | 6 ++ .../inferred-callback-drops.ts.json | 6 ++ .../6480-generic-callback-extended-results.ts | 41 +++++++++++ tests/diagnostics/inferred-callback-drops.ts | 13 ++++ .../inferred-callback-drops.ts.txt | 6 ++ 6 files changed, 144 insertions(+) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6480-generic-callback-extended-results.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/inferred-callback-drops.ts.json create mode 100644 tests/corpus/6480-generic-callback-extended-results.ts create mode 100644 tests/diagnostics/inferred-callback-drops.ts create mode 100644 tests/harness/__snapshots__/inferred-callback-drops.ts.txt diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index 27011df24..3edf33d22 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -9264,6 +9264,7 @@ export class Lowerer { * lowering goes through here (via lowerExprExpecting) or calls this * directly when the expression was already lowered. */ coerceInto(node: ts.Node, expr: IrExpr, expected: IrType): IrExpr { + refuseInferredCallbackDrops(this, node, expr.type, expected); // A union destination can accept the binding's tagged storage directly. // The checker's single-arm flow type can be stale after a callback writes // the binding. Extracting that arm and wrapping it again would discard the @@ -12984,3 +12985,74 @@ export class Lowerer { return lowerStdlibMethodCall(this, call, access); } } + +/** A callback passed to a generic call without a return annotation, whose + * returns the checker unified into a narrower result type: `trace(name, + * async () => (cond ? items : items.map((i) => ({ ...i, sum }))))` infers + * `T` from the narrower `Item[]`, though at run time the extended objects + * keep `sum`. Converting such a return into the inferred type copies the + * records and drops the members the narrower type lacks, which Node keeps + * and any later wider view (`Promise`) reads back. Refuse + * that conversion by name rather than lose the members silently; an + * annotation on the callback's return type or on the call's type argument + * names a type that keeps them. */ +function refuseInferredCallbackDrops( + lowerer: Lowerer, + node: ts.Node, + actual: IrType, + expected: IrType, +): void { + if (typeEquals(actual, expected)) return; + const structured = (t: IrType): boolean => + t.kind === "record" || t.kind === "array" || t.kind === "promise"; + if (!structured(actual) || !structured(expected)) return; + let site: ts.Node = node; + while (ts.isParenthesizedExpression(site.parent) || ts.isAwaitExpression(site.parent)) + site = site.parent; + let fn: ts.Node | undefined; + if (ts.isReturnStatement(site.parent) && site.parent.expression === site) { + fn = site.parent.parent; + while (fn && !ts.isFunctionLike(fn)) fn = fn.parent; + } else if (ts.isArrowFunction(site.parent) && site.parent.body === site) { + fn = site.parent; + } + if (!fn || !(ts.isArrowFunction(fn) || ts.isFunctionExpression(fn)) || fn.type) return; + let arg: ts.Node = fn; + while (ts.isParenthesizedExpression(arg.parent)) arg = arg.parent; + const call = arg.parent; + if (!(ts.isCallExpression(call) || ts.isNewExpression(call))) return; + if (!call.arguments?.nodes.includes(arg as ts.Expression)) return; + const dropped = droppedMembers(lowerer, actual, expected, new Set()); + if (dropped.length === 0) return; + lowerer.unsupported( + "SC1090", + node, + `returning '${lowerer.fmt(actual)}' from a callback whose result the checker infers as the narrower '${lowerer.fmt(expected)}' (the conversion would drop ${dropped.map((d) => `'${d}'`).join(", ")}, which the object keeps at run time — annotate the callback's return type or the call's type argument with a type that declares them)`, + ); +} + +/** Member names a structural conversion from `from` into `to` drops: + * declared fields of a source record the target record lacks, through + * array elements and promise payloads. */ +function droppedMembers(lowerer: Lowerer, from: IrType, to: IrType, seen: Set): string[] { + if (typeEquals(from, to)) return []; + if (from.kind === "array" && to.kind === "array") + return droppedMembers(lowerer, from.elem, to.elem, seen); + if (from.kind === "promise" && to.kind === "promise") + return droppedMembers(lowerer, from.inner, to.inner, seen); + if (from.kind !== "record" || to.kind !== "record") return []; + const key = `${from.shapeId}>${to.shapeId}`; + if (seen.has(key)) return []; + seen.add(key); + const source = lowerer.shapes.get(from.shapeId); + const target = lowerer.shapes.get(to.shapeId); + if (!source || !target || source.tuple || target.tuple || target.indexValue) return []; + const out: string[] = []; + for (const f of source.fields) { + if (f.name.startsWith("%")) continue; + const t = target.fields.find((x) => x.name === f.name); + if (!t) out.push(f.name); + else out.push(...droppedMembers(lowerer, f.type, t.type, seen).map((n) => `${f.name}.${n}`)); + } + return out; +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6480-generic-callback-extended-results.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6480-generic-callback-extended-results.ts.json new file mode 100644 index 000000000..d3b5c41b7 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6480-generic-callback-extended-results.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6480-generic-callback-extended-results.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/inferred-callback-drops.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/inferred-callback-drops.ts.json new file mode 100644 index 000000000..a6b592336 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/inferred-callback-drops.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/diagnostics/inferred-callback-drops.ts" + ], + "diags": [] +} diff --git a/tests/corpus/6480-generic-callback-extended-results.ts b/tests/corpus/6480-generic-callback-extended-results.ts new file mode 100644 index 000000000..0d75a1f53 --- /dev/null +++ b/tests/corpus/6480-generic-callback-extended-results.ts @@ -0,0 +1,41 @@ +// A callback whose returns extend the elements it was given keeps the +// added members once its result type names them (an annotation on the +// callback or on the type argument), as Node keeps them on the objects. +type Plain = { name: string; n: number }; +type Item = Plain & { extra?: boolean }; +const plain: Plain[] = [ + { name: "a", n: 1 }, + { name: "b", n: 2 }, +]; + +function run(fn: () => T): T { + return fn(); +} +async function runAsync(fn: () => Promise): Promise { + return fn(); +} + +const annotated = run((): Item[] => { + if (plain.length === 0) return plain; + return plain.map((x) => ({ ...x, extra: x.n > 1 })); +}); +console.log(JSON.stringify(annotated)); + +const viaTypeArgument = run(() => { + if (plain.length > 5) return plain; + return plain.map((x) => ({ ...x, extra: true })); +}); +console.log(JSON.stringify(viaTypeArgument), viaTypeArgument[0]!.extra); + +runAsync(async (): Promise => { + if (plain.length === 0) return plain; + return plain.map((x) => ({ ...x, extra: false })); +}).then((items) => console.log(items)); + +// Callbacks whose results the checker types by their own returns keep +// every member already. +const mapped = plain.map((x) => (x.n > 1 ? x : { ...x, extra: true })); +const asItems: Item[] = mapped; +console.log(JSON.stringify(asItems)); +const summed = plain.reduce((acc, x) => ({ ...acc, total: acc.total + x.n }), { total: 0, label: "sum" }); +console.log(JSON.stringify(summed)); diff --git a/tests/diagnostics/inferred-callback-drops.ts b/tests/diagnostics/inferred-callback-drops.ts new file mode 100644 index 000000000..0d4808b6a --- /dev/null +++ b/tests/diagnostics/inferred-callback-drops.ts @@ -0,0 +1,13 @@ +type Plain = { name: string; n: number }; +type Item = Plain & { extra?: boolean }; +const plain: Plain[] = [{ name: "a", n: 1 }]; +function run(fn: () => T): T { + return fn(); +} +// The checker unifies both returns into Plain[]; the extended objects keep +// `extra` in Node, so converting them into Plain would drop it silently. +const items: Item[] = run(() => { + if (plain.length === 0) return plain; + return plain.map((x) => ({ ...x, extra: true })); +}); +console.log(items.length); diff --git a/tests/harness/__snapshots__/inferred-callback-drops.ts.txt b/tests/harness/__snapshots__/inferred-callback-drops.ts.txt new file mode 100644 index 000000000..daa62f484 --- /dev/null +++ b/tests/harness/__snapshots__/inferred-callback-drops.ts.txt @@ -0,0 +1,6 @@ +inferred-callback-drops.ts:11:10 - error SC1090: returning '{ extra: boolean; n: number; name: string }[]' from a callback whose result the checker infers as the narrower '{ n: number; name: string }[]' (the conversion would drop 'extra', which the object keeps at run time — annotate the callback's return type or the call's type argument with a type that declares them) is not supported yet + + 10 | if (plain.length === 0) return plain; + 11 | return plain.map((x) => ({ ...x, extra: true })); + | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + 12 | }); \ No newline at end of file From 66d2010388cb572fc5617c322555e2e8120f7eb5 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 20 Oct 2026 03:08:09 -0500 Subject: [PATCH 27/35] Remove lazy loaders together and keep their receiver rule Lazy loaders that share a var statement each removed their declarators on their own, and declarators of one statement removed separately leave overlapping edits, of which applyEdits keeps the first. A later loader in the statement (one using __toESM, say) stayed behind and the interop pass refused the package. Every removed declarator of the pass (the interop requires, the loaders and the unused export tables) now goes into one set, removed once. A call through a loader (`load().fn()`) respelled the callee as the export's live read, which drops the namespace receiver. Loader calls now take the rule namespace calls already use: the callee must be a top-level function that never reads `this`; other calls keep the package refused. --- .../frontend/npm-static-bundled-esm.test.ts | 32 +++++++++++++++ .../src/frontend/npm-static-bundled-esm.ts | 39 +++++++++++-------- 2 files changed, 55 insertions(+), 16 deletions(-) diff --git a/packages/compiler/src/frontend/npm-static-bundled-esm.test.ts b/packages/compiler/src/frontend/npm-static-bundled-esm.test.ts index 1e2ba24a3..98decbb41 100644 --- a/packages/compiler/src/frontend/npm-static-bundled-esm.test.ts +++ b/packages/compiler/src/frontend/npm-static-bundled-esm.test.ts @@ -152,6 +152,38 @@ test("minified initializers sharing a var statement lower the same way", () => { ); }); +test("memoized lazy loaders sharing a var statement lower together", () => { + const lowered = expectSameBehavior(`${helpers}var a_exports = {}; +__export(a_exports, { v: () => v }); +var v, init_a = __esm({ "a.ts"() { console.log("a init"); v = 1; } }); +var b_exports = {}; +__export(b_exports, { default: () => w }); +var w, init_b = __esm({ "b.ts"() { console.log("b init"); w = 2; } }); +var nsA, loadA = () => nsA ??= (init_a(), __toCommonJS(a_exports)), nsB, loadB = () => nsB ??= __toESM((init_b(), __toCommonJS(b_exports))); +console.log(loadA().v, loadB().default, loadA().v); +`); + expect(lowered).not.toContain("loadA"); + expect(lowered).not.toContain("loadB"); +}); + +test("calls through a memoized lazy loader keep the receiver rule", () => { + const source = (use: string): string => `${helpers}var w_exports = {}; +__export(w_exports, { who: () => who, plain: () => plain }); +function who() { return this.__esModule; } +function plain(x) { return "plain " + x; } +var init_w = __esm({ "w.ts"() {} }); +var ns, load = () => ns ??= (init_w(), __toCommonJS(w_exports)); +console.log(${use}); +`; + expect(lower(source("load().who()"))).toEqual({ + degrade: expect.stringContaining( + "calls only of exported functions that do not read 'this'", + ) as string, + }); + const lowered = expectSameBehavior(source("load().plain(1), (load().plain)(2)")); + expect(lowered).not.toContain("load()"); +}); + test.each([ ["is asynchronous", `var init_a = __esm({ async "a.ts"() { await 0; } });\ninit_a();`], [ diff --git a/packages/compiler/src/frontend/npm-static-bundled-esm.ts b/packages/compiler/src/frontend/npm-static-bundled-esm.ts index 6e9c4f315..9c77128e1 100644 --- a/packages/compiler/src/frontend/npm-static-bundled-esm.ts +++ b/packages/compiler/src/frontend/npm-static-bundled-esm.ts @@ -449,7 +449,6 @@ export function lowerBundledEsmInits( } } } - edits.push(...removeDeclarators(sf, deadDeclarators)); /* Namespaces of lazily initialized modules read through members. */ let mainTarget: string | null = null; @@ -500,6 +499,17 @@ export function lowerBundledEsmInits( }; const getterText = (value: ts.Expression): string => applyEdits(source, value.getStart(sf), value.getEnd(), edits); + /** A call through the namespace (`ns.fn()`, `L().fn()`) passes the object + * the read came from as `this`; the lowered object differs from Node's + * module object (`__esModule`, property attributes), so it must stay + * unobservable: the callee is a top-level function that never reads + * `this`. */ + const receiverHidden = (values: ReadonlyMap, name: string): boolean => { + const value = values.get(name); + if (value === undefined || !ts.isIdentifier(value) || !stable(value.text)) return false; + const decl = top.get(value.text)?.node; + return decl !== undefined && ts.isFunctionDeclaration(decl) && !readsThis(decl); + }; /* Memoized lazy loaders of those namespaces, `L = () => M ??= NS` (a * bundle plugin's lazy re-exports): every `L().name` becomes the @@ -564,7 +574,7 @@ export function lowerBundledEsmInits( const table = tables.get(target); if (table === undefined) continue; const loaderWhy = { - degrade: `its memoized lazy loader '${name}' returns the namespace of a bundled ES module as a value, a module object the static rewrite cannot represent (only member reads of its result compile)`, + degrade: `its memoized lazy loader '${name}' returns the namespace of a bundled ES module as a value, a module object the static rewrite cannot represent (only member reads of its result compile, and calls only of exported functions that do not read 'this')`, }; const memo = memoize.left.text; const memoDecl = top.get(memo)?.node; @@ -612,6 +622,10 @@ export function lowerBundledEsmInits( const member = (access as ts.PropertyAccessExpression).name.text; const value = entries.get(member); if (value === undefined || shadowed(access, table.entries)) return loaderWhy; + // The rewrite reads the export without the namespace object, so a + // call through it (`L().fn()`) loses its receiver. + if (calledWithReceiver(access as ts.Expression) && !receiverHidden(entries, member)) + return loaderWhy; const init = initCall === null ? "" : `${initCall.getText(sf)}, `; edits.push({ start: access.getStart(sf), @@ -619,7 +633,7 @@ export function lowerBundledEsmInits( text: `(${init}${getterText(value)})`, }); } - edits.push(...removeDeclarators(sf, new Set([d, memoDecl]))); + deadDeclarators.add(d).add(memoDecl); loaderSpans.push({ start: d.getStart(sf), end: d.getEnd() }); lowered.add(target); } @@ -664,24 +678,13 @@ export function lowerBundledEsmInits( else break; } const values = new Map(table.entries); - // A call through the namespace (`ns.fn()`) passes the object the read - // came from as `this`; the lowered object differs from Node's module - // object (`__esModule`, property attributes), so it must stay - // unobservable: the callee is a top-level function that never reads - // `this`. - const receiverHidden = (name: string): boolean => { - const value = values.get(name); - if (value === undefined || !ts.isIdentifier(value) || !stable(value.text)) return false; - const decl = top.get(value.text)?.node; - return decl !== undefined && ts.isFunctionDeclaration(decl) && !readsThis(decl); - }; const memberRead = (access: ts.Node | undefined, of: ts.Node): boolean => access !== undefined && ts.isPropertyAccessExpression(access) && access.expression === of && values.has(access.name.text) && plainRead(access) && - (!calledWithReceiver(access) || receiverHidden(access.name.text)); + (!calledWithReceiver(access) || receiverHidden(values, access.name.text)); const holder = ns.parent; // `__reExport(main, ns, module.exports)`: a star re-export of the // module into the bundle's own exports (`export * from` in its entry). @@ -792,8 +795,12 @@ export function lowerBundledEsmInits( ) return { degrade: ESM_SHAPE_DEGRADE }; edits.push({ start: table.stmt.getStart(sf), end: table.stmt.getEnd(), text: "" }); - edits.push(...removeDeclarators(sf, new Set([decl]))); + deadDeclarators.add(decl); } + // Every removed declarator in one pass: declarators of one statement + // removed separately would leave overlapping edits, and applyEdits keeps + // only the first of them. + edits.push(...removeDeclarators(sf, deadDeclarators)); /* The run-once closures. */ for (const init of inits.values()) { From 603c6720f9f7dd8e155bbf3e3bbfd62d848da059 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 20 Oct 2026 03:13:40 -0500 Subject: [PATCH 28/35] Keep function export rewrites from capturing names or keys The pass respells `exports.x` and `module.exports.x` through the exported function's name. A parameter, local, nested function or catch binding of that name between a respelled read and the top level would capture it, so the pass now applies only when the file binds the name once, at its top-level declaration. `{ exports }` after the rebinding respelled to `{ SemVer }`, which renames the property; the shorthand now becomes `{ exports: SemVer }`. --- .../npm-static-cjs-exports-alias.test.ts | 81 +++++++++++++++++++ .../frontend/npm-static-cjs-exports-alias.ts | 20 ++++- 2 files changed, 97 insertions(+), 4 deletions(-) create mode 100644 packages/compiler/src/frontend/npm-static-cjs-exports-alias.test.ts diff --git a/packages/compiler/src/frontend/npm-static-cjs-exports-alias.test.ts b/packages/compiler/src/frontend/npm-static-cjs-exports-alias.test.ts new file mode 100644 index 000000000..c3704ea29 --- /dev/null +++ b/packages/compiler/src/frontend/npm-static-cjs-exports-alias.test.ts @@ -0,0 +1,81 @@ +import { spawnSync } from "node:child_process"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, expect, test } from "vitest"; +import { nodeFrontendServices } from "./services-node.js"; + +const dirs: string[] = []; +afterAll(() => { + for (const dir of dirs) rmSync(dir, { recursive: true, force: true }); +}); + +function rewrite(source: string): string | null { + return nodeFrontendServices().cjsFunctionExports(source, "/pkg/index.js"); +} + +function run(file: string) { + const result = spawnSync(process.execPath, [file], { encoding: "utf8" }); + return { status: result.status, stdout: result.stdout, stderr: result.stderr }; +} + +/** The rewritten module prints what the original prints. */ +function expectSameBehavior(source: string): string { + const result = rewrite(source); + expect(typeof result).toBe("string"); + const dir = mkdtempSync(join(tmpdir(), "scriptc-cjs-function-exports-")); + dirs.push(dir); + const original = join(dir, "original.cjs"); + const rewritten = join(dir, "rewritten.cjs"); + writeFileSync(original, source); + writeFileSync(rewritten, result as string); + const oracle = run(original); + expect(oracle.status).toBe(0); + expect(run(rewritten)).toEqual(oracle); + return result as string; +} + +const report = ` +console.log(typeof module.exports, module.exports === Version, Object.keys(Version).join(",")); +`; + +test("members of a function export object read and write the function", () => { + const rewritten = expectSameBehavior(`exports = module.exports = Version; +exports.parse = parse; +module.exports.major = (v) => parse(v)[0]; +function Version(v) { return { v }; } +function parse(v) { return v.split(".").map(Number); } +console.log(exports.major("2.1.0"), module.exports.parse("1.2").length); +${report}`); + expect(rewritten).toContain("Version.parse = parse"); + expect(rewritten).not.toContain("exports = module.exports"); +}); + +test.each([ + ["a parameter", "function inner(Version) { return exports.parse; }"], + ["a local", "function inner() { const Version = {}; return exports.parse; }"], + ["a nested function", "function inner() { function Version() {} return exports.parse; }"], + [ + "a catch binding", + "function inner() { try { throw 0; } catch (Version) { return exports.parse; } }", + ], +])("a nested binding of the function's name (%s) keeps the module as it is", (_, inner) => { + expect( + rewrite(`exports = module.exports = Version; +exports.parse = parse; +function Version() {} +function parse() {} +${inner} +`), + ).toBeNull(); +}); + +test("an `exports` shorthand property keeps its key", () => { + const rewritten = expectSameBehavior(`exports = module.exports = Version; +exports.tag = "v"; +function Version() {} +const holder = { exports }; +console.log(Object.keys(holder).join(","), holder.exports === Version, holder.exports.tag); +${report}`); + expect(rewritten).toContain("{ exports: Version }"); +}); diff --git a/packages/compiler/src/frontend/npm-static-cjs-exports-alias.ts b/packages/compiler/src/frontend/npm-static-cjs-exports-alias.ts index c88c9949e..dfe45b862 100644 --- a/packages/compiler/src/frontend/npm-static-cjs-exports-alias.ts +++ b/packages/compiler/src/frontend/npm-static-cjs-exports-alias.ts @@ -15,12 +15,14 @@ * runs and nothing replaces the export object again: the statement is a * top-level statement preceded only by inert declarations, it is the * file's only write of `module.exports` and `exports`, the function is a - * top-level declaration nothing reassigns, and `exports`/`module` are the - * CommonJS globals (never declared in the file). Other shapes answer + * top-level declaration nothing reassigns and the file's only binding of + * its name, and `exports`/`module` are the CommonJS globals (never + * declared in the file). Other shapes answer * null and keep their existing treatment. */ import * as ts from "./ts7/syntax.js"; import { applyEdits, + countDeclarations, declaredNames, isWritten, references, @@ -105,7 +107,10 @@ export function normalizeCjsFunctionExports(sf: ts.SourceFile): string | null { const fnDecls = stmts.filter( (st) => ts.isFunctionDeclaration(st) && st.name?.text === fnName && st.body !== undefined, ); - if (fnDecls.length !== 1) return null; + // The respelled reads name the function from wherever they sit, so no + // other binding of its name (a parameter, a local, a nested function) + // may exist anywhere in the file. + if (fnDecls.length !== 1 || countDeclarations(sf, fnName) !== 1) return null; const edits: Edit[] = []; let unsafe = false; @@ -131,7 +136,14 @@ export function normalizeCjsFunctionExports(sf: ts.SourceFile): string | null { unsafe = true; continue; } - edits.push({ start: r.getStart(sf), end: r.getEnd(), text: fnName }); + // `{ exports }` keeps its key: the shorthand spells out the value. + const shorthand = + r.parent !== undefined && ts.isShorthandPropertyAssignment(r.parent) && r.parent.name === r; + edits.push({ + start: r.getStart(sf), + end: r.getEnd(), + text: shorthand ? `exports: ${fnName}` : fnName, + }); continue; } if (r.text === "module") { From e1e45f12b9a86803c51c83e52a685080295aa61c Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 20 Oct 2026 03:55:43 -0500 Subject: [PATCH 29/35] Give converted object literals their accessors An object literal with get/set accessors converted to `any` or `unknown` listed its reserved closure slots (`%get:x`) as data keys, so a dynamic read of the accessor answered undefined without calling the getter, and Object.keys and JSON.stringify listed the slot instead of the property. The checked-dynamic copy now defines each accessor as one enumerable, configurable property whose getter and setter are the slot closures, in declaration order: the record shape records its accessor key order (`accessorOrder`). Live views of such records build that copy, and checking a copy back into the record reads the closures from the accessor's getter and setter. Reads, writes, Object.keys, Object.entries, JSON.stringify and property descriptors answer as in Node. --- packages/compiler/src/backend/llvm/dyn.ts | 63 +++++++++++++++++-- .../src/backend/llvm/expr-stream-bridges.ts | 2 + packages/compiler/src/frontend/type-mapper.ts | 43 ++++++++----- .../src/ir/function-references.test.ts | 1 + packages/compiler/src/ir/ir.ts | 4 ++ .../object-literal-accessors-dyn.ts.json | 6 ++ packages/runtime/src/scr_dyn.h | 5 ++ packages/runtime/src/scr_json.c | 25 ++++++++ tests/corpus/object-literal-accessors-dyn.ts | 31 +++++++++ .../object-literal-accessors-dyn.ts.json | 1 + 10 files changed, 162 insertions(+), 19 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/object-literal-accessors-dyn.ts.json create mode 100644 tests/corpus/object-literal-accessors-dyn.ts create mode 100644 tests/harness/corpus-costs/object-literal-accessors-dyn.ts.json diff --git a/packages/compiler/src/backend/llvm/dyn.ts b/packages/compiler/src/backend/llvm/dyn.ts index 86b4f5be8..7e414daee 100644 --- a/packages/compiler/src/backend/llvm/dyn.ts +++ b/packages/compiler/src/backend/llvm/dyn.ts @@ -34,6 +34,7 @@ import { InternalCompilerError } from "../../errors.js"; * ScrDynPath { parent, key, index } — the %ScrDynPath type. */ import type { IrRecordShape, IrType, IrUnionDef } from "../../ir/ir.js"; import { + accessorSlotProp, DYN_HANDLE_KINDS, isDirentShape, isDynTypedRefType, @@ -613,6 +614,18 @@ export class LlDyn { * tracking does not count the read's reference. Returns the value and * the matching release. */ private memberRead(B: BlockBuilder, key: string, t: IrType): { value: string; release: string } { + // An accessor slot reads the getter or setter of the accessor property + // the checked-dynamic copy holds for it (see accessorSlotProp). + const slot = accessorSlotProp(key); + if (slot !== null) { + this.host.declare(`declare ptr @scr_dyn_obj_accessor_fn(ptr, ptr, ${this.S}, i1 zeroext)`); + this.host.declare(`declare void @scr_dyn_release(ptr)`); + const fn = B.tmp(); + B.line( + `${fn} = call ptr @scr_dyn_obj_accessor_fn(ptr %d, ptr ${this.host.cstr(slot.prop)}, ${this.S} ${Buffer.byteLength(slot.prop, "utf8")}, i1 ${slot.kind === "set" ? "true" : "false"})`, + ); + return { value: fn, release: "scr_dyn_release" }; + } const held = this.isRecursiveShape(t) || this.holdsRecursiveShape(t); const release = held ? "scr_dyn_release_held" : "scr_dyn_release"; this.host.declare(`declare void @${release}(ptr)`); @@ -2687,13 +2700,53 @@ export class LlDyn { // Keys insert in DECLARED order (JS insertion order); internal // '%'-fields follow so a record→dyn→record round trip keeps them. const byName = new Map(shape.fields.map((f) => [f.name, f])); - const order = shape.declaredOrder ?? shape.fields.map((f) => f.name); + // An accessor property is one key whose getter and setter are the + // slot closures (see accessorSlotProp), in accessorOrder's place. + const order = shape.accessorOrder ?? shape.declaredOrder ?? shape.fields.map((f) => f.name); const inOrder = new Set(order); - const dynFields = [ - ...order.map((n) => byName.get(n)).filter((f) => f !== undefined), - ...shape.fields.filter((f) => !inOrder.has(f.name)), + const accessors = new Set( + shape.accessorOrder === undefined + ? [] + : shape.fields.flatMap((f) => { + const slot = accessorSlotProp(f.name); + return slot === null ? [] : [slot.prop]; + }), + ); + type DynField = { name: string; type: IrType } | { accessor: string }; + const dynFields: DynField[] = [ + ...order.flatMap((n) => { + if (accessors.has(n)) return [{ accessor: n } as DynField]; + const f = byName.get(n); + return f === undefined ? [] : [f as DynField]; + }), + ...shape.fields.filter( + (f) => + !inOrder.has(f.name) && !(accessors.size > 0 && accessorSlotProp(f.name) !== null), + ), ]; - for (const f of dynFields) { + for (const entry of dynFields) { + if ("accessor" in entry) { + host.declare( + `declare void @scr_dyn_obj_set_accessor(ptr, ptr, ${host.sizeType}, ptr, ptr)`, + ); + const boxed = (kind: "get" | "set"): string => { + const slot = byName.get(`%${kind}:${entry.accessor}`); + if (slot === undefined) return "null"; + const fv = loadFieldOf(slot.name, slot.type); + const conv = B.tmp(); + B.line( + `${conv} = call ptr @${this.toDynHelper(slot.type)}(${this.valTy(slot.type)} ${fv})`, + ); + return conv; + }; + const getter = boxed("get"); + const setter = boxed("set"); + B.line( + `call void @scr_dyn_obj_set_accessor(ptr ${d}, ptr ${host.cstr(entry.accessor)}, ${host.sizeType} ${Buffer.byteLength(entry.accessor, "utf8")}, ptr ${getter}, ptr ${setter}) ; ${llvmCommentText(entry.accessor)}`, + ); + continue; + } + const f = entry; const klen = Buffer.byteLength(f.name, "utf8"); const fv = loadFieldOf(f.name, f.type); // An ABSENT optional field contributes no key. diff --git a/packages/compiler/src/backend/llvm/expr-stream-bridges.ts b/packages/compiler/src/backend/llvm/expr-stream-bridges.ts index 134c74b92..1841053df 100644 --- a/packages/compiler/src/backend/llvm/expr-stream-bridges.ts +++ b/packages/compiler/src/backend/llvm/expr-stream-bridges.ts @@ -15,6 +15,7 @@ import { isDirentShape, isDynTypedRefType, isRefCounted, + shapeHasAccessorSlots, typeKey, } from "../../ir/ir.js"; import { mangleFunction, mangleGlobal, mangleRecordStruct } from "../mangle.js"; @@ -965,6 +966,7 @@ export function streamTypedRefMaterializeAdapter( * Declared-field records and tuples cover the live stream values. */ const ordinary = shape.indexValue || + shapeHasAccessorSlots(shape) || shape.fields.some((field) => field.name === "handleEvent" && field.type.kind === "func"); if (ordinary) { const out = B.tmp(); diff --git a/packages/compiler/src/frontend/type-mapper.ts b/packages/compiler/src/frontend/type-mapper.ts index 94506f034..b095ecbb8 100644 --- a/packages/compiler/src/frontend/type-mapper.ts +++ b/packages/compiler/src/frontend/type-mapper.ts @@ -5895,6 +5895,7 @@ interface RecordShapeParts { fields: { name: string; type: IrType }[]; indexValue?: IrType; declaredOrder?: string[]; + accessorOrder?: string[]; } function mapRecordType(widened: ts.Type, ctx: TypeMapperCtx): IrType | null { @@ -5931,15 +5932,14 @@ function mapRecordType(widened: ts.Type, ctx: TypeMapperCtx): IrType | null { // (generic type parameters, mixin instantiations) cannot intern by // checker-type identity — fence recursive generic-open shapes. if (contextResolutions !== sensitivityAtEntry) return null; - return { - kind: "record", - shapeId: shapes.finalizeRecursive( - widened, - inner.fields, - inner.indexValue, - inner.declaredOrder, - ), - }; + const recursive = shapes.finalizeRecursive( + widened, + inner.fields, + inner.indexValue, + inner.declaredOrder, + ); + adoptAccessorOrder(shapes.get(recursive), inner.accessorOrder); + return { kind: "record", shapeId: recursive }; } // A record with no typed payload contract is a view of an existing // object, including across parameters and returns. Building a native @@ -5954,15 +5954,20 @@ function mapRecordType(widened: ts.Type, ctx: TypeMapperCtx): IrType | null { return DYN; const refined = refinedConstituentShape(widened, inner, ctx); if (refined !== null) return { kind: "record", shapeId: refined }; - return { - kind: "record", - shapeId: shapes.intern(inner.fields, false, inner.indexValue, inner.declaredOrder), - }; + const shapeId = shapes.intern(inner.fields, false, inner.indexValue, inner.declaredOrder); + adoptAccessorOrder(shapes.get(shapeId), inner.accessorOrder); + return { kind: "record", shapeId }; } finally { shapes.inProgress.delete(widened); } } +/** The first-seen accessor key order of a shape with accessor slots. */ +function adoptAccessorOrder(shape: IrRecordShape | undefined, order: string[] | undefined): void { + if (shape !== undefined && order !== undefined && shape.accessorOrder === undefined) + shape.accessorOrder = order; +} + /** A refinement intersection (`Expr & { kind: "mul" }`, which the checker * reduces to `Mul & { kind: "mul" }`) keeps the shape of the constituent it * refines when the refinement changes no field's representation. Recursive @@ -6322,8 +6327,18 @@ function mapRecordTypeInner( const declaredOrder = esOwnKeyOrder( fields.filter((f) => accessorSlotProp(f.name) === null).map((f) => f.name), ); + // With accessors: every own key in that order, an accessor property + // named once for its get/set pair (the checked-dynamic copy lists it). + const accessorOrder = fields.some((f) => accessorSlotProp(f.name) !== null) + ? esOwnKeyOrder([...new Set(fields.map((f) => accessorSlotProp(f.name)?.prop ?? f.name))]) + : undefined; fields.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)); - return { fields, ...(indexValue ? { indexValue } : {}), declaredOrder }; + return { + fields, + ...(indexValue ? { indexValue } : {}), + declaredOrder, + ...(accessorOrder ? { accessorOrder } : {}), + }; } } diff --git a/packages/compiler/src/ir/function-references.test.ts b/packages/compiler/src/ir/function-references.test.ts index c2d461e09..2c5a45dd5 100644 --- a/packages/compiler/src/ir/function-references.test.ts +++ b/packages/compiler/src/ir/function-references.test.ts @@ -170,6 +170,7 @@ const MODULE_FIELDS: Record>, Role> = { "records[].id": "other", "records[].fields[].name": "other", "records[].declaredOrder[]": "other", + "records[].accessorOrder[]": "other", "unions[].id": "other", "unions[].discriminant.field": "other", "unions[].discriminant.cases[].values[]": "other", diff --git a/packages/compiler/src/ir/ir.ts b/packages/compiler/src/ir/ir.ts index de6bf558c..34e70f3f0 100644 --- a/packages/compiler/src/ir/ir.ts +++ b/packages/compiler/src/ir/ir.ts @@ -1621,6 +1621,10 @@ export interface IrRecordShape { * fields carry that declaredOrder OMITS are internal '%'-fields (Dirent's * %dtype) — hidden from every key-order surface, JSON included. */ declaredOrder?: string[]; + /** Shapes with accessor slots: every own key in declaration order, with + * an accessor property named once (see accessorSlotProp). The + * checked-dynamic copy lists its keys in this order. */ + accessorOrder?: string[]; } /** fs.Dirent rows: the hidden %dtype entry kind (libuv's UV_DIRENT diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/object-literal-accessors-dyn.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/object-literal-accessors-dyn.ts.json new file mode 100644 index 000000000..ffe0c75d3 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/object-literal-accessors-dyn.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/object-literal-accessors-dyn.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_dyn.h b/packages/runtime/src/scr_dyn.h index 6fdbbb408..ff6763cda 100644 --- a/packages/runtime/src/scr_dyn.h +++ b/packages/runtime/src/scr_dyn.h @@ -1076,6 +1076,11 @@ extern SCR_TL size_t scr_uncloneable_count; ScrDyn *scr_dyn_from_error(const ScrError *e); void scr_error_register_dyn(const ScrVt *vt, ScrDyn *(*box)(void *)); ScrDyn *scr_error_dyn_fields(const ScrError *e); +/* An accessor property of a fresh object: enumerable and configurable, + * with the given getter and setter (either may be NULL). Moves both. */ +void scr_dyn_obj_set_accessor(ScrDyn *obj, const char *key, size_t key_len, ScrDyn *getter, ScrDyn *setter); +/* The getter or setter of an own accessor property, +1 (undefined when none). */ +ScrDyn *scr_dyn_obj_accessor_fn(const ScrDyn *obj, const char *key, size_t key_len, bool setter); void scr_error_commit_dyn(ScrError *e, const ScrDyn *view); /* The reverse extraction, riding the same identity cache: a dyn error * that came from a runtime ScrError answers THAT instance (+1; diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index c42e66734..131514d4b 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -4356,6 +4356,31 @@ void scr_dyn_obj_set(ScrDyn *obj, const char *key, size_t key_len, ScrDyn *value scr_dyn_obj_put(obj, key, key_len, value); } +/* The getter (or, with `setter`, the setter) of the own accessor property + * `key`: +1, or the undefined value when the property is not an accessor + * or has none. */ +ScrDyn *scr_dyn_obj_accessor_fn(const ScrDyn *obj, const char *key, size_t key_len, bool setter) { + const ScrDynEntry *entry = obj->kind == SCR_DYN_OBJ ? scr_dyn_find_entry(obj, key, key_len) : NULL; + ScrDyn *fn = entry && entry->accessor ? (setter ? entry->setter : entry->getter) : NULL; + return scr_dyn_retain(fn ? fn : scr_dyn_undefined()); +} + +/* An object literal's get/set accessor property on a fresh copy: + * enumerable and configurable, with the given getter and setter functions + * (either NULL when the literal declares none). Moves both. */ +void scr_dyn_obj_set_accessor(ScrDyn *obj, const char *key, size_t key_len, ScrDyn *getter, ScrDyn *setter) { + scr_dyn_obj_put(obj, key, key_len, scr_dyn_retain(scr_dyn_undefined())); + ScrDynEntry *entry = scr_dyn_find_entry(obj, key, key_len); + if (!entry) { + scr_dyn_release(getter); + scr_dyn_release(setter); + return; + } + entry->accessor = true; + entry->getter = getter; + entry->setter = setter; +} + /* A class instance's view takes its native field values whatever the * attributes copied into it (a frozen instance's view is built the same way * as any other). Consumes value. */ diff --git a/tests/corpus/object-literal-accessors-dyn.ts b/tests/corpus/object-literal-accessors-dyn.ts new file mode 100644 index 000000000..3b7dc95e4 --- /dev/null +++ b/tests/corpus/object-literal-accessors-dyn.ts @@ -0,0 +1,31 @@ +// Object literals with get/set accessors held as `any` or `unknown` keep +// their accessors, as in Node: a read calls the getter, a write calls the +// setter, Object.keys, Object.entries and JSON.stringify list the accessor +// in declaration order and call the getter, and the property descriptor +// has a get and set function. Writes to data properties reach the +// original object. + +function dyn(value: unknown): any { + return value; +} + +let count = 0; +const lit = { + a: 1, + get g(): number { count++; return 42; }, + set g(v: number) { console.log("set", v); }, + b: 2, + get only(): string { return "read-only"; }, +}; +const d = dyn(lit); +console.log("read", d.g, d.only, count); +console.log("keys", Object.keys(d).join(","), JSON.stringify(d), count); +console.log("entries", Object.entries(d).map((e) => e.join("=")).join(";")); +console.log("in", "g" in d, Object.hasOwn(d, "g")); +d.g = 7; +d.a = 5; +console.log("write", lit.a, d.a); +const desc = Object.getOwnPropertyDescriptor(d, "g")!; +console.log("descriptor", typeof desc.get, typeof desc.set, desc.enumerable, desc.configurable); +const u: unknown = lit; +console.log("unknown", (u as any).g, JSON.stringify(u)); diff --git a/tests/harness/corpus-costs/object-literal-accessors-dyn.ts.json b/tests/harness/corpus-costs/object-literal-accessors-dyn.ts.json new file mode 100644 index 000000000..9643a761b --- /dev/null +++ b/tests/harness/corpus-costs/object-literal-accessors-dyn.ts.json @@ -0,0 +1 @@ +{ "plain": 2903, "san": 3777 } From 1ccda45f7dcef77db7727a6a929ecf71ceea03d4 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 20 Oct 2026 03:59:25 -0500 Subject: [PATCH 30/35] Decode an AST list's position only when it is read Every read of an AST list field built an AstNodeList whose pos and end were decoded from the wire first, though most readers only use `nodes`. The list now keeps the wire file and the list's index, and pos and end decode when they are read. The allocation per read stays the same size, and a decoded file still holds no list objects. --- packages/compiler/src/frontend/ts7/ast-node.ts | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/packages/compiler/src/frontend/ts7/ast-node.ts b/packages/compiler/src/frontend/ts7/ast-node.ts index 92ab5e4b8..3760fff8d 100644 --- a/packages/compiler/src/frontend/ts7/ast-node.ts +++ b/packages/compiler/src/frontend/ts7/ast-node.ts @@ -19,13 +19,22 @@ import { SyntaxKind, NodeFlags } from "./enums.js"; /** A list of child nodes with the list's own position, the shape of tsc-ts's * NodeList. The file keeps the nodes array, not this object: each read of a * list field returns a new AstNodeList over the same array, so `nodes` keeps - * its identity and a decoded file holds no list objects. */ + * its identity and a decoded file holds no list objects. The position is + * decoded from the wire when it is read; most readers only use `nodes`. */ export class AstNodeList { constructor( - readonly pos: number, - readonly end: number, + private readonly wire: AstWireFile, + private readonly index: number, readonly nodes: readonly T[], ) {} + + get pos(): number { + return this.wire.pos(this.index) >>> 0; + } + + get end(): number { + return this.wire.end(this.index) >>> 0; + } } /** One immutable AST response owns one object per node and one array per @@ -83,7 +92,7 @@ export class AstFile { this.lists[index] = nodes; this.materialized?.(); } - return new AstNodeList(this.wire.pos(index) >>> 0, this.wire.end(index) >>> 0, nodes); + return new AstNodeList(this.wire, index, nodes); } namedChild(node: AstNode, name: string): number { From 1092f69ea9d4abad2894d95f630b1a55aa89fda4 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Tue, 20 Oct 2026 05:22:55 -0700 Subject: [PATCH 31/35] Split a chained Set call the self-hosted compiler cannot lower `deadDeclarators.add(d).add(memoDecl)` stopped the self-hosting build with "chained Set method calls ... has no scriptc lowering yet". Co-Authored-By: Claude Opus 5.5 --- packages/compiler/src/frontend/npm-static-bundled-esm.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/compiler/src/frontend/npm-static-bundled-esm.ts b/packages/compiler/src/frontend/npm-static-bundled-esm.ts index 9c77128e1..d9fa88fc4 100644 --- a/packages/compiler/src/frontend/npm-static-bundled-esm.ts +++ b/packages/compiler/src/frontend/npm-static-bundled-esm.ts @@ -633,7 +633,8 @@ export function lowerBundledEsmInits( text: `(${init}${getterText(value)})`, }); } - deadDeclarators.add(d).add(memoDecl); + deadDeclarators.add(d); + deadDeclarators.add(memoDecl); loaderSpans.push({ start: d.getStart(sf), end: d.getEnd() }); lowered.add(target); } From c475aabc2a1c8a8ce75a2738de0088c7218b1ebe Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 20 Oct 2026 08:41:04 -0500 Subject: [PATCH 32/35] Refuse arguments to inherited stream constructors from untyped code Construction from checked-dynamic code (`new F(options)` with F an untyped value holding a ctor-less subclass of a stream class or EventEmitter) dispatched to the class's constructor and completed the arguments against the inherited signature, which takes none here, so the options were dropped. The checked construction dispatch now reports those arguments at run time with the message typed construction reports at compile time, located at the class: a fixed argument count refuses at the call, and a spread call refuses when its argument list is not empty. --- .../frontend/lowering/class-construction.ts | 48 ++++++++++++++++++- .../src/frontend/lowering/lower-classes.ts | 28 +++++++++++ tests/harness/errors.test.ts | 31 ++++++++++++ 3 files changed, 105 insertions(+), 2 deletions(-) diff --git a/packages/compiler/src/frontend/lowering/class-construction.ts b/packages/compiler/src/frontend/lowering/class-construction.ts index 7f4a003c9..30f504b4a 100644 --- a/packages/compiler/src/frontend/lowering/class-construction.ts +++ b/packages/compiler/src/frontend/lowering/class-construction.ts @@ -18,6 +18,7 @@ import type { Lowerer } from "./lowerer.js"; import type { ParamShape } from "./call-signatures.js"; import { classPrototypeData } from "./class-prototypes.js"; import { classInstanceOf } from "./class-dynamic-dispatch.js"; +import { inheritedBuiltinCtorFence } from "./lower-classes.js"; const PREFIX = "%class.construct:"; const PACKED = `${PREFIX}packed`; @@ -380,6 +381,29 @@ export class ClassConstructionDispatch { changed = true; continue; } + const type: IrType = { kind: "classval", className: name }; + const callee = varRef("p.0", DYN, loc); + // A constructor inherited from a runtime stream or EventEmitter + // takes no arguments here: arguments report at run time, as they do + // at compile time for typed construction. + const fence = inheritedBuiltinCtorFence(lowerer, info, loc); + if (fence !== null && fn.name !== PACKED && fn.params.length > 1) { + fn.body.unshift({ + kind: "if", + cond: { + kind: "libCall", + fn: "dyn.classIs", + args: [callee, { kind: "strLit", value: typeKey(type), type: STRING, loc }], + type: BOOL, + loc, + }, + then: [fence], + else_: null, + loc, + }); + changed = true; + continue; + } const args = constructorArguments( lowerer, info.ctorParams, @@ -388,8 +412,6 @@ export class ClassConstructionDispatch { fn.name === PACKED ? varRef("p.1", DYN, loc) : undefined, ); if (!args) continue; - const type: IrType = { kind: "classval", className: name }; - const callee = varRef("p.0", DYN, loc); const value = constructedValue(lowerer, info, { kind: "newValue", callee: { kind: "dynCheck", value: callee, type, loc }, @@ -400,6 +422,28 @@ export class ClassConstructionDispatch { const body: IrStmt[] = [ { kind: "return", value: lowerer.coerceToExpected(value, DYN), loc }, ]; + // A spread call's argument count is known only at run time. + if (fence !== null && fn.name === PACKED) + body.unshift({ + kind: "if", + cond: { + kind: "bin", + op: ">", + left: { + kind: "libCall", + fn: "dyn.arrLen", + args: [varRef("p.1", DYN, loc)], + type: F64, + loc, + }, + right: { kind: "numLit", value: 0, type: F64, loc }, + type: BOOL, + loc, + }, + then: [fence], + else_: null, + loc, + }); fn.body.unshift({ kind: "if", cond: { diff --git a/packages/compiler/src/frontend/lowering/lower-classes.ts b/packages/compiler/src/frontend/lowering/lower-classes.ts index 714588a57..cd0c6b8bd 100644 --- a/packages/compiler/src/frontend/lowering/lower-classes.ts +++ b/packages/compiler/src/frontend/lowering/lower-classes.ts @@ -8725,6 +8725,34 @@ function inheritsBuiltinStreamCtor(lowerer: Lowerer, info: ClassInfo): boolean { return false; } +/** The run-time form of refuseInheritedBuiltinCtorArgs, for construction + * from checked-dynamic code (`new F(options)` with F an untyped value + * holding C), whose argument count is known only to the caller: a + * runtimeFence located at C, or null when C's constructor takes its + * arguments. */ +export function inheritedBuiltinCtorFence( + lowerer: Lowerer, + info: ClassInfo, + loc: SrcLoc, +): IrStmt | null { + let feature: string | null = null; + if (inheritsBuiltinStreamCtor(lowerer, info)) + feature = `new ${info.def.name.replace(/^%/, "")} with arguments through an inherited stream constructor (declare a constructor that passes an inline options object to super(...))`; + else if (lowerer.inheritsBuiltinEmitterCtor(info)) + feature = "EventEmitter constructor options ('captureRejections')"; + if (feature === null) return null; + // Located at the class, whose constructor takes no arguments. + const at = info.decl !== null ? locOf(info.decl) : loc; + const diag = unsupportedDiag("SC1090", at, feature); + lowerer.runtimeFences.push(diag); + const line = + info.decl !== null + ? ts.getLineAndCharacterOfPosition(info.decl.getSourceFile(), at.start).line + 1 + : null; + const where = line !== null ? ` [${diag.code} at ${at.file}:${line}]` : ""; + return { kind: "runtimeFence", code: diag.code, message: `${diag.message}${where}`, loc }; +} + /** `new C(args)` where C's constructor is the inherited runtime stream or * EventEmitter constructor (a ctor-less chain into the base): that * constructor takes zero arguments here (the options bag fences, like the diff --git a/tests/harness/errors.test.ts b/tests/harness/errors.test.ts index 1f3d9729b..f08ed45d7 100644 --- a/tests/harness/errors.test.ts +++ b/tests/harness/errors.test.ts @@ -359,6 +359,37 @@ const arr = tuple.map(() => new Date(0));`, expect(r).toEqual({ stdout: reference.stdout, stderr: reference.stderr, exitCode: 0 }); }); + test.each([ + ["an argument", "make(Upper, { objectMode: true });"], + ["a spread argument", "spread(Upper, [{ objectMode: true }]);"], + ])( + "%s through an untyped factory to an inherited stream constructor is a named deferred fence (JS lane)", + async (_, call) => { + // The constructor a ctor-less stream subclass inherits takes no + // arguments here; typed construction refuses them at compile time, + // and construction from untyped code refuses them at run time + // instead of constructing without them. + const r = await compileAndRun( + `inherited-stream-ctor-${call.startsWith("make") ? "args" : "spread"}`, + `'use strict'; +const { Transform } = require('node:stream'); +class Upper extends Transform {} +function make(F, options) { return new F(options); } +function bare(F) { return new F(); } +function spread(F, args) { return new F(...args); } +console.log('bare', bare(Upper) instanceof Upper, spread(Upper, []) instanceof Upper); +${call} +`, + "cjs", + ); + expect(r.exitCode).toBe(1); + expect(r.stdout).toBe("bare true true\n"); + expect(r.stderr).toMatch( + /^Uncaught Error: new Upper with arguments through an inherited stream constructor .* is not supported yet \[SC1090 at .*inherited-stream-ctor-\w+\.cjs:3\]\n$/, + ); + }, + ); + test("extending a property-assigned class ABOVE its assignment is a named deferred fence (JS lane)", async () => { // The property-assigned class-expression family (corpus 2032/2033 // pins what lowers): `exports.O = class extends exports.I {}` with From 1ef2004c9beb0049762f4724d51eda6479c5cdd5 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 20 Oct 2026 08:46:56 -0500 Subject: [PATCH 33/35] Compare only the 48 bytes SHA-384 writes in the cross-kernel check --- packages/runtime/test/test_sha_kernels.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/runtime/test/test_sha_kernels.c b/packages/runtime/test/test_sha_kernels.c index f9f9b9df7..da7d79235 100644 --- a/packages/runtime/test/test_sha_kernels.c +++ b/packages/runtime/test/test_sha_kernels.c @@ -192,7 +192,7 @@ int main(void) { sha512_of(data, n, chunks[c], sha384, a); select512(0); sha512_of(data, n, chunks[c], sha384, b); - check(memcmp(a, b, 64) == 0, kernel512_names[k], n, chunks[c]); + check(memcmp(a, b, sha384 ? 48 : 64) == 0, kernel512_names[k], n, chunks[c]); } } } From 5e0ad5bbaa005dd3431394b6e33f92289d146a10 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 20 Oct 2026 08:51:47 -0500 Subject: [PATCH 34/35] Compare checker panics without their text in the checker identity check A statement whose lowering meets a checker panic keeps the panic's text in its SC0004 fence. tsgo and tsc-ts reach the same panic for `s.match(re) || []` (6187-js-match-or-empty-array.js) and describe it in their own words, so the check now replaces the text after `panic:` before it compares the IR and the diagnostics. --- scripts/checker-identity.mjs | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/scripts/checker-identity.mjs b/scripts/checker-identity.mjs index 35d01cb93..a710d5946 100644 --- a/scripts/checker-identity.mjs +++ b/scripts/checker-identity.mjs @@ -1,6 +1,7 @@ #!/usr/bin/env node // Compiles every corpus program to LLVM IR with each TypeScript 7 API server (tsc-ts, the default, and tsgo with -// SCRIPTC_CHECKER=tsgo) and fails unless both give the same IR and the same diagnostics. +// SCRIPTC_CHECKER=tsgo) and fails unless both give the same IR and the same diagnostics. The text of a checker panic +// is the checker's own (Go's runtime error for tsgo, tsc-ts's for tsc-ts), so it is compared as `panic: …`. // // Usage: node scripts/checker-identity.mjs [--jobs ] [--optimization release|dev] [--filter ] // @@ -73,11 +74,11 @@ async function main() { const [a, b] = checkers.map((checker) => results.get(checker).get(file)); const name = file.slice(corpusDir.length + 1); if (a === undefined || b === undefined) differences.push(`${name}: not compiled`); - else if (a.ok !== b.ok || a.diagnostics !== b.diagnostics) + else if (a.ok !== b.ok || withoutPanicText(a.diagnostics) !== withoutPanicText(b.diagnostics)) differences.push( `${name}: tsgo ${a.ok ? "compiles" : a.diagnostics}; tsc-ts ${b.ok ? "compiles" : b.diagnostics}`, ); - else if (a.ok && !readFileSync(a.llvm).equals(readFileSync(b.llvm))) + else if (a.ok && irWithoutPanicText(a.llvm) !== irWithoutPanicText(b.llvm)) differences.push(`${name}: LLVM IR differs`); } if (differences.length > 0) { @@ -96,6 +97,31 @@ async function main() { } } +/** A diagnostic list with each SC0004 checker panic's text replaced by `…`. */ +function withoutPanicText(diagnostics) { + return diagnostics?.replace(/(SC0004: [^;]*?: panic: )[^;]*/g, "$1…"); +} + +/** A module's LLVM IR with each SC0004 checker panic's text replaced by `…`. A fence statement keeps the + * diagnostic's message in a string constant named for a hash of its contents, and passes its length. */ +function irWithoutPanicText(llvm) { + let ir = readFileSync(llvm, "utf8"); + const constant = + /^(@sc_cs_\d+) = internal constant \[\d+ x i8\] c"(the TypeScript checker crashed answering a query here [^"]*?: panic: )[^"]*?( \[SC0004 at [^"]*)"$/gm; + const names = []; + ir = ir.replace(constant, (_, name, head, tail) => { + names.push(name); + return `@checker_panic_${names.length} = internal constant c"${head}…${tail}"`; + }); + names.forEach((name, index) => { + ir = ir.replace( + new RegExp(`${name}\\b(, i64 \\d+)?`, "g"), + (_, length) => `@checker_panic_${index + 1}${length === undefined ? "" : ", i64 …"}`, + ); + }); + return ir; +} + async function worker([outDir, optimization, ...files]) { const { compile } = await import( pathToFileURL(join(repoRoot, "packages/compiler/dist/index.js")).href From d850e14d505dd8c17ef584014564b5b112e89642 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 20 Oct 2026 08:58:50 -0500 Subject: [PATCH 35/35] Refuse dropped members through conditional and satisfies returns The refusal of callback returns that the inferred result type would cut down found the return by walking up through parentheses and await only, so a return inside a conditional's branch or a `satisfies` expression converted silently and lost the extended members. It now walks through every parent whose value can be the expression's own: parentheses, await, satisfies, non-null assertions, conditional branches, the operands of ??, || and &&, and the right operand of a comma. --- .../compiler/src/frontend/lowering/lowerer.ts | 31 +++++++++++++++++-- ...nferred-callback-drops-conditional.ts.json | 6 ++++ .../inferred-callback-drops-satisfies.ts.json | 6 ++++ .../inferred-callback-drops-conditional.ts | 12 +++++++ .../inferred-callback-drops-satisfies.ts | 13 ++++++++ ...inferred-callback-drops-conditional.ts.txt | 6 ++++ .../inferred-callback-drops-satisfies.ts.txt | 6 ++++ 7 files changed, 78 insertions(+), 2 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/inferred-callback-drops-conditional.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/inferred-callback-drops-satisfies.ts.json create mode 100644 tests/diagnostics/inferred-callback-drops-conditional.ts create mode 100644 tests/diagnostics/inferred-callback-drops-satisfies.ts create mode 100644 tests/harness/__snapshots__/inferred-callback-drops-conditional.ts.txt create mode 100644 tests/harness/__snapshots__/inferred-callback-drops-satisfies.ts.txt diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index 3edf33d22..4798f49c5 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -13007,8 +13007,7 @@ function refuseInferredCallbackDrops( t.kind === "record" || t.kind === "array" || t.kind === "promise"; if (!structured(actual) || !structured(expected)) return; let site: ts.Node = node; - while (ts.isParenthesizedExpression(site.parent) || ts.isAwaitExpression(site.parent)) - site = site.parent; + while (yieldsOperand(site)) site = site.parent!; let fn: ts.Node | undefined; if (ts.isReturnStatement(site.parent) && site.parent.expression === site) { fn = site.parent.parent; @@ -13031,6 +13030,34 @@ function refuseInferredCallbackDrops( ); } +/** True when the value of `node`'s parent can be `node`'s own value: + * parentheses, `await`, `satisfies`, a non-null assertion, a conditional's + * branch, either operand of `??`, `||` and `&&`, and the right operand of a + * comma. */ +function yieldsOperand(node: ts.Node): boolean { + const p = node.parent; + if (p === undefined) return false; + if ( + ts.isParenthesizedExpression(p) || + ts.isAwaitExpression(p) || + ts.isSatisfiesExpression(p) || + ts.isNonNullExpression(p) + ) + return true; + if (ts.isConditionalExpression(p)) return p.whenTrue === node || p.whenFalse === node; + if (ts.isBinaryExpression(p)) { + const op = p.operatorToken.kind; + if ( + op === ts.SyntaxKind.QuestionQuestionToken || + op === ts.SyntaxKind.BarBarToken || + op === ts.SyntaxKind.AmpersandAmpersandToken + ) + return true; + return op === ts.SyntaxKind.CommaToken && p.right === node; + } + return false; +} + /** Member names a structural conversion from `from` into `to` drops: * declared fields of a source record the target record lacks, through * array elements and promise payloads. */ diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/inferred-callback-drops-conditional.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/inferred-callback-drops-conditional.ts.json new file mode 100644 index 000000000..2014c8620 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/inferred-callback-drops-conditional.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/diagnostics/inferred-callback-drops-conditional.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/inferred-callback-drops-satisfies.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/inferred-callback-drops-satisfies.ts.json new file mode 100644 index 000000000..44c5e8bbc --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/inferred-callback-drops-satisfies.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/diagnostics/inferred-callback-drops-satisfies.ts" + ], + "diags": [] +} diff --git a/tests/diagnostics/inferred-callback-drops-conditional.ts b/tests/diagnostics/inferred-callback-drops-conditional.ts new file mode 100644 index 000000000..f124beed9 --- /dev/null +++ b/tests/diagnostics/inferred-callback-drops-conditional.ts @@ -0,0 +1,12 @@ +type Plain = { name: string; n: number }; +type Item = Plain & { extra?: boolean }; +const plain: Plain[] = [{ name: "a", n: 1 }]; +function run(fn: () => T): T { + return fn(); +} +// The checker unifies both branches into Plain[]; the extended objects keep +// `extra` in Node, so converting them into Plain would drop it silently. +const items: Item[] = run(() => + plain.length === 0 ? plain : plain.map((x) => ({ ...x, extra: true })), +); +console.log(items.length); diff --git a/tests/diagnostics/inferred-callback-drops-satisfies.ts b/tests/diagnostics/inferred-callback-drops-satisfies.ts new file mode 100644 index 000000000..ad58237b8 --- /dev/null +++ b/tests/diagnostics/inferred-callback-drops-satisfies.ts @@ -0,0 +1,13 @@ +type Plain = { name: string; n: number }; +type Item = Plain & { extra?: boolean }; +const plain: Plain[] = [{ name: "a", n: 1 }]; +function run(fn: () => T): T { + return fn(); +} +// `satisfies` checks the return against Plain[] without changing it; the +// extended objects keep `extra` in Node. +const items: Item[] = run(() => { + if (plain.length === 0) return plain; + return plain.map((x) => ({ ...x, extra: true })) satisfies Plain[]; +}); +console.log(items.length); diff --git a/tests/harness/__snapshots__/inferred-callback-drops-conditional.ts.txt b/tests/harness/__snapshots__/inferred-callback-drops-conditional.ts.txt new file mode 100644 index 000000000..a14f19317 --- /dev/null +++ b/tests/harness/__snapshots__/inferred-callback-drops-conditional.ts.txt @@ -0,0 +1,6 @@ +inferred-callback-drops-conditional.ts:10:32 - error SC1090: returning '{ extra: boolean; n: number; name: string }[]' from a callback whose result the checker infers as the narrower '{ n: number; name: string }[]' (the conversion would drop 'extra', which the object keeps at run time — annotate the callback's return type or the call's type argument with a type that declares them) is not supported yet + + 9 | const items: Item[] = run(() => + 10 | plain.length === 0 ? plain : plain.map((x) => ({ ...x, extra: true })), + | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + 11 | ); \ No newline at end of file diff --git a/tests/harness/__snapshots__/inferred-callback-drops-satisfies.ts.txt b/tests/harness/__snapshots__/inferred-callback-drops-satisfies.ts.txt new file mode 100644 index 000000000..ea1e37549 --- /dev/null +++ b/tests/harness/__snapshots__/inferred-callback-drops-satisfies.ts.txt @@ -0,0 +1,6 @@ +inferred-callback-drops-satisfies.ts:11:10 - error SC1090: returning '{ extra: boolean; n: number; name: string }[]' from a callback whose result the checker infers as the narrower '{ n: number; name: string }[]' (the conversion would drop 'extra', which the object keeps at run time — annotate the callback's return type or the call's type argument with a type that declares them) is not supported yet + + 10 | if (plain.length === 0) return plain; + 11 | return plain.map((x) => ({ ...x, extra: true })) satisfies Plain[]; + | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + 12 | }); \ No newline at end of file