From ee9f0d64eb37293cbb32c739709871e8c53e43f7 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Wed, 30 Sep 2026 02:17:56 -0500 Subject: [PATCH] feat: ship the self-hosted native scriptc CLI (#585) * feat: ship the self-hosted native scriptc CLI - Run the shared compiler and CLI natively, including library builds and compile-time evaluation. - Install native platform packages and ship relocatable standalone distributions. - Preserve program semantics and reuse validated build artifacts for fast rebuilds. * fix: complete native CLI integration and validation - Preserve library diagnostics and run recursive compiler work on the main stack. - Restore packaged helper permissions and exercise native npm and Wasm library builds. - Recover interrupted Sandbox status probes within the original command deadline. --- .github/workflows/ci.yml | 17 +- .github/workflows/release.yml | 54 +- .gitignore | 1 + README.md | 2 +- RELEASING.md | 10 +- docs/src/app/platforms/page.mdx | 4 +- docs/src/app/quickstart/page.mdx | 8 +- package.json | 2 +- packages/cli-darwin-arm64/package.json | 29 + packages/cli-darwin-x64/package.json | 29 + packages/cli-linux-arm64-gnu/package.json | 32 + packages/cli-linux-arm64-musl/package.json | 32 + packages/cli-linux-x64-gnu/package.json | 32 + packages/cli-linux-x64-musl/package.json | 32 + packages/cli-win32-x64-msvc/package.json | 29 + packages/cli/README.md | 2 +- packages/cli/package.json | 32 +- packages/cli/scripts/install-native.mjs | 113 + packages/cli/scripts/prepare-native.mjs | 18 + packages/cli/src/command.ts | 1 + packages/cli/src/host.ts | 1 + packages/cli/src/main.ts | 357 +--- packages/cli/src/output-options.ts | 92 +- packages/cli/src/paths.ts | 104 +- packages/cli/src/usage.ts | 87 +- packages/cli/src/wasi-runner.ts | 39 +- packages/cli/test/native-install.test.ts | 131 ++ packages/cli/test/wasi-runtime-pack.test.ts | 10 +- .../ambient/scriptc-node-fallback.d.ts | 5 +- packages/compiler/native/comptime.c | 189 ++ packages/compiler/native/host.c | 46 + packages/compiler/native/host.ffi.json | 6 + packages/compiler/package.json | 8 +- packages/compiler/src/backend/build-cache.ts | 27 +- packages/compiler/src/backend/cache-root.ts | 24 + packages/compiler/src/backend/link-trace.ts | 22 + packages/compiler/src/backend/llvm/emitter.ts | 52 +- .../compiler/src/backend/llvm/expr-async.ts | 2 +- .../src/backend/native-link-info-core.ts | 155 ++ .../compiler/src/backend/native-link-info.ts | 143 +- .../compiler/src/backend/native-toolchain.ts | 217 +- packages/compiler/src/backend/native-tools.ts | 37 +- .../compiler/src/backend/object-localize.ts | 16 +- .../src/backend/runtime-pack-native.ts | 36 +- .../src/backend/target-diagnostics.ts | 117 + .../compiler/src/backend/target-platform.ts | 80 + packages/compiler/src/backend/targets.ts | 31 +- .../src/backend/toolchain-environment.ts | 110 + .../compiler/src/backend/vendor-archives.ts | 11 +- .../compiler/src/backend/vendor-inputs.ts | 9 + packages/compiler/src/cli/command.ts | 325 +++ packages/compiler/src/cli/host.ts | 20 + packages/compiler/src/cli/output-options.ts | 91 + packages/compiler/src/cli/paths.ts | 61 + packages/compiler/src/cli/usage.ts | 86 + packages/compiler/src/cli/wasi-paths.ts | 45 + packages/compiler/src/cli/wasi-runner.ts | 39 + packages/compiler/src/compile-types.ts | 157 ++ packages/compiler/src/coverage/report.ts | 14 +- .../compiler/src/coverage/surface-manifest.ts | 2 +- packages/compiler/src/executable/prepare.ts | 79 + packages/compiler/src/frontend/analysis.ts | 90 + packages/compiler/src/frontend/dts-paths.ts | 19 +- .../src/frontend/input-tracker.test.ts | 24 + .../compiler/src/frontend/input-tracker.ts | 30 +- .../lowering/array-indexed-mutation.ts | 17 +- .../src/frontend/lowering/lower-builtins.ts | 58 +- .../src/frontend/lowering/lower-calls.ts | 3 +- .../src/frontend/lowering/lower-containers.ts | 4 +- .../src/frontend/lowering/lower-exprs.ts | 2 +- .../compiler/src/frontend/lowering/lowerer.ts | 50 +- .../lowering/object-enumeration-receiver.ts | 34 + .../lowering/sanitize-class-types.test.ts | 61 + .../frontend/lowering/sanitize-class-types.ts | 68 + .../compiler/src/frontend/pipeline-native.ts | 8 +- packages/compiler/src/frontend/pipeline.ts | 8 + .../compiler/src/frontend/provenance-core.ts | 452 ++++ packages/compiler/src/frontend/provenance.ts | 454 +--- packages/compiler/src/frontend/type-mapper.ts | 30 +- packages/compiler/src/index.ts | 1086 +--------- packages/compiler/src/ir/ir.ts | 2 +- .../compiler/src/library/cache-primitives.ts | 2 +- packages/compiler/src/library/fence-eval.ts | 8 +- packages/compiler/src/library/int-infer.ts | 19 +- .../compiler/src/library/library-profile.ts | 9 +- packages/compiler/src/library/prepare.ts | 665 ++++++ .../compiler/src/library/sidecar-validate.ts | 16 +- packages/compiler/src/library/sidecar.ts | 16 +- .../compiler/src/native/arguments.test.ts | 34 - packages/compiler/src/native/arguments.ts | 79 - packages/compiler/src/native/cache.test.ts | 69 + packages/compiler/src/native/cache.ts | 92 + packages/compiler/src/native/cli.ts | 100 + .../{driver.test.ts => compiler.test.ts} | 18 +- packages/compiler/src/native/compiler.ts | 286 +++ packages/compiler/src/native/comptime.test.ts | 68 + packages/compiler/src/native/comptime.ts | 64 + packages/compiler/src/native/driver.ts | 142 -- .../src/native/executable-cache.test.ts | 159 ++ .../compiler/src/native/executable-cache.ts | 210 ++ .../compiler/src/native/file-identity.test.ts | 38 + packages/compiler/src/native/file-identity.ts | 13 + packages/compiler/src/native/library.ts | 102 + packages/compiler/src/native/main.ts | 15 - packages/compiler/src/native/prepare.test.ts | 71 + packages/compiler/src/native/prepare.ts | 178 ++ packages/compiler/src/native/sanitizer.ts | 128 ++ packages/compiler/src/native/task.test.ts | 18 + packages/compiler/src/native/task.ts | 11 + packages/compiler/src/native/toolchain.ts | 77 +- .../test/ts7/baselines/order-parity.json | 1882 +++++++++-------- packages/runtime/src/scr_lib.c | 37 +- pnpm-lock.yaml | 71 +- scripts/bench-builds.mjs | 11 +- scripts/build-native-cli.mts | 67 + scripts/build-native-compiler.mts | 54 - scripts/native-cli-assets.mts | 73 + scripts/package-native-cli.mjs | 57 + scripts/sandbox-command.mjs | 21 + scripts/sandbox-test.mjs | 32 +- scripts/sync-versions.mjs | 3 + scripts/verify-native-cli.mjs | 50 + tests/corpus/array-fill-unit-values.ts | 10 + .../array-optional-number-write-index.ts | 12 + tests/corpus/closure-nullable-union-return.ts | 65 + tests/corpus/crypto-hash-optional-input.ts | 17 + tests/corpus/fs-remove-readonly.ts | 18 + tests/corpus/fs-write-string-bytes-union.ts | 26 + .../corpus/object-enumeration-array-reads.ts | 9 + tests/corpus/record-optional-json-presence.ts | 32 + tests/corpus/util-parseargs-wrapped/main.ts | 26 + .../util-parseargs-wrapped/tsconfig.json | 3 + tests/fixtures/self-hosting/class-types.ts | 26 + tests/fixtures/self-hosting/native-cache.ts | 24 + tests/harness/library-dispatch.test.ts | 2 +- tests/harness/sandbox-command.test.ts | 43 +- .../harness/self-hosting-class-types.test.ts | 30 + .../harness/self-hosting-native-cache.test.ts | 41 + .../self-hosting-native-driver.test.ts | 148 +- .../self-hosting-native-toolchain.test.ts | 4 +- .../self-hosting-serialization.test.ts | 1 + tests/harness/wasm-library.test.ts | 34 +- vitest.config.ts | 5 + 143 files changed, 7608 insertions(+), 4095 deletions(-) create mode 100644 packages/cli-darwin-arm64/package.json create mode 100644 packages/cli-darwin-x64/package.json create mode 100644 packages/cli-linux-arm64-gnu/package.json create mode 100644 packages/cli-linux-arm64-musl/package.json create mode 100644 packages/cli-linux-x64-gnu/package.json create mode 100644 packages/cli-linux-x64-musl/package.json create mode 100644 packages/cli-win32-x64-msvc/package.json create mode 100644 packages/cli/scripts/install-native.mjs create mode 100644 packages/cli/scripts/prepare-native.mjs create mode 100644 packages/cli/src/command.ts create mode 100644 packages/cli/src/host.ts create mode 100644 packages/cli/test/native-install.test.ts create mode 100644 packages/compiler/native/comptime.c create mode 100644 packages/compiler/native/host.c create mode 100644 packages/compiler/native/host.ffi.json create mode 100644 packages/compiler/src/backend/cache-root.ts create mode 100644 packages/compiler/src/backend/link-trace.ts create mode 100644 packages/compiler/src/backend/native-link-info-core.ts create mode 100644 packages/compiler/src/backend/target-diagnostics.ts create mode 100644 packages/compiler/src/backend/target-platform.ts create mode 100644 packages/compiler/src/backend/toolchain-environment.ts create mode 100644 packages/compiler/src/backend/vendor-inputs.ts create mode 100644 packages/compiler/src/cli/command.ts create mode 100644 packages/compiler/src/cli/host.ts create mode 100644 packages/compiler/src/cli/output-options.ts create mode 100644 packages/compiler/src/cli/paths.ts create mode 100644 packages/compiler/src/cli/usage.ts create mode 100644 packages/compiler/src/cli/wasi-paths.ts create mode 100644 packages/compiler/src/cli/wasi-runner.ts create mode 100644 packages/compiler/src/compile-types.ts create mode 100644 packages/compiler/src/executable/prepare.ts create mode 100644 packages/compiler/src/frontend/analysis.ts create mode 100644 packages/compiler/src/frontend/lowering/object-enumeration-receiver.ts create mode 100644 packages/compiler/src/frontend/lowering/sanitize-class-types.test.ts create mode 100644 packages/compiler/src/frontend/lowering/sanitize-class-types.ts create mode 100644 packages/compiler/src/frontend/provenance-core.ts create mode 100644 packages/compiler/src/library/prepare.ts delete mode 100644 packages/compiler/src/native/arguments.test.ts delete mode 100644 packages/compiler/src/native/arguments.ts create mode 100644 packages/compiler/src/native/cache.test.ts create mode 100644 packages/compiler/src/native/cache.ts create mode 100644 packages/compiler/src/native/cli.ts rename packages/compiler/src/native/{driver.test.ts => compiler.test.ts} (83%) create mode 100644 packages/compiler/src/native/compiler.ts create mode 100644 packages/compiler/src/native/comptime.test.ts create mode 100644 packages/compiler/src/native/comptime.ts delete mode 100644 packages/compiler/src/native/driver.ts create mode 100644 packages/compiler/src/native/executable-cache.test.ts create mode 100644 packages/compiler/src/native/executable-cache.ts create mode 100644 packages/compiler/src/native/file-identity.test.ts create mode 100644 packages/compiler/src/native/file-identity.ts create mode 100644 packages/compiler/src/native/library.ts delete mode 100644 packages/compiler/src/native/main.ts create mode 100644 packages/compiler/src/native/prepare.test.ts create mode 100644 packages/compiler/src/native/prepare.ts create mode 100644 packages/compiler/src/native/sanitizer.ts create mode 100644 packages/compiler/src/native/task.test.ts create mode 100644 packages/compiler/src/native/task.ts create mode 100644 scripts/build-native-cli.mts delete mode 100644 scripts/build-native-compiler.mts create mode 100644 scripts/native-cli-assets.mts create mode 100644 scripts/package-native-cli.mjs create mode 100644 scripts/verify-native-cli.mjs create mode 100644 tests/corpus/array-fill-unit-values.ts create mode 100644 tests/corpus/array-optional-number-write-index.ts create mode 100644 tests/corpus/closure-nullable-union-return.ts create mode 100644 tests/corpus/crypto-hash-optional-input.ts create mode 100644 tests/corpus/fs-remove-readonly.ts create mode 100644 tests/corpus/fs-write-string-bytes-union.ts create mode 100644 tests/corpus/object-enumeration-array-reads.ts create mode 100644 tests/corpus/record-optional-json-presence.ts create mode 100644 tests/corpus/util-parseargs-wrapped/main.ts create mode 100644 tests/corpus/util-parseargs-wrapped/tsconfig.json create mode 100644 tests/fixtures/self-hosting/class-types.ts create mode 100644 tests/fixtures/self-hosting/native-cache.ts create mode 100644 tests/harness/self-hosting-class-types.test.ts create mode 100644 tests/harness/self-hosting-native-cache.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ab35020b..a1146d22 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -178,6 +178,8 @@ jobs: with: version: 0.16.0 - run: CC=zig AR=zig pnpm --filter @scriptc/runtime-linux-x64-gnu build:native + - if: matrix.compiler == 'native' + run: pnpm --filter @scriptc/runtime-wasm32-wasi build:native - run: pnpm build - run: pnpm test ${{ matrix.test }} @@ -299,7 +301,7 @@ jobs: test "$(/tmp/hello)" = "hello world" ' - name: WASI helper object/runtime-pack smoke - run: pnpm test packages/cli/test/wasi-runtime-pack.test.ts + run: pnpm test packages/cli/test/wasi-runtime-pack.test.ts tests/harness/wasm-library.test.ts llvm_artifacts_macos_arm64: name: test (macOS arm64 LLVM artifacts, no clang, ${{ matrix.shard }}/5) @@ -355,22 +357,29 @@ jobs: node scripts/verify-llvm-package.mjs "$RUNNER_TEMP/$(basename "$TARBALL")" - name: Packed npm installation smoke if: matrix.shard == 1 + env: + # Release optimization is covered by the compiler bootstrap jobs. + # This lane verifies packaging and installation with a faster seed. + SCRIPTC_NATIVE_OPTIMIZATION: dev run: | + pnpm --filter @scriptc/cli-darwin-arm64 build:native pnpm --dir packages/runtime pack --pack-destination "$RUNNER_TEMP" --silent pnpm --dir packages/runtime-darwin-arm64 pack --pack-destination "$RUNNER_TEMP" --silent pnpm --dir packages/compiler pack --pack-destination "$RUNNER_TEMP" --silent + pnpm --dir packages/cli-darwin-arm64 pack --pack-destination "$RUNNER_TEMP" --silent pnpm --dir packages/cli pack --pack-destination "$RUNNER_TEMP" --silent PREFIX="$RUNNER_TEMP/installed-scriptc" - npm install --prefix "$PREFIX" --ignore-scripts \ + npm install --prefix "$PREFIX" --no-audit --no-fund \ "$RUNNER_TEMP/scriptc-runtime-$(node -p "require('./packages/runtime/package.json').version").tgz" \ "$RUNNER_TEMP/scriptc-runtime-darwin-arm64-$(node -p "require('./packages/runtime-darwin-arm64/package.json').version").tgz" \ "$RUNNER_TEMP/scriptc-llvm-darwin-arm64-$(node -p "require('./packages/llvm-darwin-arm64/package.json').version").tgz" \ "$RUNNER_TEMP/scriptc-compiler-$(node -p "require('./packages/compiler/package.json').version").tgz" \ + "$RUNNER_TEMP/scriptc-cli-darwin-arm64-$(node -p "require('./packages/cli-darwin-arm64/package.json').version").tgz" \ "$RUNNER_TEMP/scriptc-$(node -p "require('./packages/cli/package.json').version").tgz" - "$PREFIX/node_modules/.bin/scriptc" build tests/corpus/001-hello.ts \ + PATH="" "$PREFIX/node_modules/.bin/scriptc" build tests/corpus/001-hello.ts \ --emit=obj -o "$RUNNER_TEMP/installed.o" file "$RUNNER_TEMP/installed.o" | grep 'Mach-O 64-bit object arm64' - "$PREFIX/node_modules/.bin/scriptc" build tests/corpus/001-hello.ts \ + PATH="" "$PREFIX/node_modules/.bin/scriptc" build tests/corpus/001-hello.ts \ --print=native-link-info -o "$RUNNER_TEMP/installed-link.o" \ > "$RUNNER_TEMP/installed-link.json" node examples/native-object/link.mjs cc \ diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 89f18cc6..9a50514e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -61,37 +61,48 @@ jobs: runner: macos-15 helper: llvm-darwin-arm64 runtime: runtime-darwin-arm64 + cli: cli-darwin-arm64 - platform: darwin-x64 runner: macos-15-intel helper: llvm-darwin-x64 runtime: runtime-darwin-x64 + cli: cli-darwin-x64 - platform: linux-x64 runner: ubuntu-24.04 helper: llvm-linux-x64-gnu runtime: runtime-linux-x64-gnu + cli: cli-linux-x64-gnu llvm_asset: LLVM-22.1.8-Linux-X64 use_zig: true - platform: linux-arm64 runner: ubuntu-24.04-arm helper: llvm-linux-arm64-gnu runtime: runtime-linux-arm64-gnu + cli: cli-linux-arm64-gnu llvm_asset: LLVM-22.1.8-Linux-ARM64 use_zig: true - platform: windows-x64 runner: windows-2022 helper: llvm-win32-x64-msvc runtime: runtime-win32-x64-msvc + cli: cli-win32-x64-msvc use_zig: true - platform: linux-x64-musl runner: ubuntu-24.04 helper: llvm-linux-x64-musl runtime: runtime-linux-x64-musl + cli: cli-linux-x64-musl + seed_helper: llvm-linux-x64-gnu + cli_target: x86_64-linux-musl llvm_asset: LLVM-22.1.8-Linux-X64 use_zig: true - platform: linux-arm64-musl runner: ubuntu-24.04-arm helper: llvm-linux-arm64-musl runtime: runtime-linux-arm64-musl + cli: cli-linux-arm64-musl + seed_helper: llvm-linux-arm64-gnu + cli_target: aarch64-linux-musl llvm_asset: LLVM-22.1.8-Linux-ARM64 use_zig: true - platform: wasm32-wasi @@ -171,12 +182,25 @@ jobs: run: | pnpm --filter @scriptc/${{ matrix.helper }} build:native pnpm --filter @scriptc/${{ matrix.runtime }} build:native - - uses: actions/upload-artifact@v4 + - name: Build host helper for cross-libc bootstrap + if: matrix.seed_helper != '' + run: pnpm --filter @scriptc/${{ matrix.seed_helper }} build:native + - name: Build native CLI distribution + if: matrix.cli != '' + env: + SCRIPTC_TARGET: ${{ matrix.cli_target }} + run: | + pnpm --filter @scriptc/compiler --filter scriptc build + pnpm --filter @scriptc/${{ matrix.cli }} build:native + node scripts/verify-native-cli.mjs packages/${{ matrix.cli }} --run + - name: Upload native packages + uses: actions/upload-artifact@v4 with: name: native-${{ matrix.platform }} path: | packages/${{ matrix.helper }} packages/${{ matrix.runtime }} + ${{ matrix.cli != '' && format('packages/{0}/dist', matrix.cli) || '' }} retention-days: 1 build-mobile-runtime-packs: @@ -266,12 +290,13 @@ jobs: # upload-artifact does not preserve executable bits. Restore the # helper mode before pnpm runs its prepack checks. find packages -type f -path '*/bin/scriptc-llvm-codegen' -exec chmod 755 {} + + find packages -type f \( -path '*/dist/bin/scriptc' -o -path '*/dist/lib/typescript/lib/tsc' -o -path '*/dist/lib/scriptc-comptime' \) -exec chmod 755 {} + pnpm -r build - name: Check version sync run: | VERSION="${{ needs.check-release.outputs.version }}" - for pkg in packages/runtime packages/runtime-darwin-arm64 packages/llvm-darwin-arm64 packages/runtime-linux-x64-gnu packages/llvm-linux-x64-gnu packages/runtime-linux-arm64-gnu packages/llvm-linux-arm64-gnu packages/runtime-linux-x64-musl packages/llvm-linux-x64-musl packages/runtime-linux-arm64-musl packages/llvm-linux-arm64-musl packages/runtime-wasm32-wasi packages/runtime-ios-arm64 packages/runtime-ios-simulator-arm64 packages/runtime-android-arm64 packages/runtime-win32-x64-msvc packages/llvm-win32-x64-msvc packages/compiler packages/cli; do + for pkg in packages/runtime packages/runtime-* packages/llvm-* packages/compiler packages/cli packages/cli-*; do V=$(node -p "require('./$pkg/package.json').version") if [ "$V" != "$VERSION" ]; then echo "Version mismatch: $pkg is $V, expected $VERSION" @@ -356,6 +381,8 @@ jobs: publish_dir packages/runtime publish_dir packages/runtime-darwin-arm64 publish_dir packages/llvm-darwin-arm64 "$HELPER_TARBALL" + publish_dir packages/runtime-darwin-x64 + publish_dir packages/llvm-darwin-x64 publish_dir packages/runtime-linux-x64-gnu publish_dir packages/llvm-linux-x64-gnu publish_dir packages/runtime-linux-arm64-gnu @@ -371,19 +398,13 @@ jobs: publish_dir packages/runtime-win32-x64-msvc publish_dir packages/llvm-win32-x64-msvc publish_dir packages/compiler + for pkg in packages/cli-*; do publish_dir "$pkg"; done publish_dir packages/cli env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # The GitHub release is a tag, notes, and one asset: the surface - # manifest (packages/compiler/surface-manifest.json — the machine- - # readable listing of the surface the static tier compiles at this - # version, regenerated here and verified against the committed file). - # The platform helper ships through its npm package rather than as a GitHub - # release asset, so this job runs AFTER a successful npm publish and never - # gates it. The body is the CHANGELOG.md block between the - # release:start/release:end markers, which RELEASING.md keeps on the - # latest entry only. + # Publish standalone native distributions and the surface manifest after + # npm publishing succeeds. The release body comes from the marked changelog. github-release: name: Create GitHub Release needs: [check-release, publish] @@ -427,6 +448,16 @@ jobs: fi echo "Extracted release notes for $VERSION ($LINES lines)" + - name: Download native distributions and runtime packs + uses: actions/download-artifact@v4 + with: + pattern: native-* + path: packages + merge-multiple: true + + - name: Package standalone compilers + run: node scripts/package-native-cli.mjs packages /tmp/scriptc-release-assets + - name: Create GitHub Release run: | VERSION="${{ needs.check-release.outputs.version }}" @@ -445,5 +476,6 @@ jobs: # Attach the surface manifest (idempotent: --clobber makes # re-runs replace the asset instead of failing). gh release upload "$TAG" packages/compiler/surface-manifest.json --clobber + gh release upload "$TAG" /tmp/scriptc-release-assets/* --clobber env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 880a0b9f..b70ddf10 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ output/ /packages/runtime-darwin-arm64/runtime-pack.json /packages/runtime-darwin-arm64/.runtime-pack-* /packages/llvm-*/bin/ +/packages/cli/bin/ /packages/runtime-*/artifacts/ /packages/runtime-*/runtime-pack.json /packages/runtime-*/.runtime-pack-* diff --git a/README.md b/README.md index 2c8d849e..f92388f0 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ scriptc is experimental and targets macOS, Linux, Windows, and WebAssembly via W ## Installation -The compiler requires Node.js 24 or newer. `--emit=ir|llvm` needs only Node. `--emit=asm|obj` uses the matching optional platform helper installed with scriptc on supported macOS, Linux, and Windows hosts (and for WASI), but needs no compiler, archiver, linker, or SDK. Ordinary LLVM executable builds need a platform linker driver and SDK/sysroot, but use the bundled helper plus precompiled runtime pack rather than compiling generated or runtime C. Set `SCRIPTC_LINKER` to choose that driver. Runtime development with `--sanitize` additionally needs a C compiler. The executables it produces do not require Node. +The installed compiler runs natively on supported macOS, Linux, and Windows hosts. Compilation, compile-time evaluation, and native execution do not require Node. `--emit=ir|llvm|asm|obj` uses the bundled TypeScript checker and LLVM helper without an external compiler, archiver, linker, or SDK. Executable builds additionally need a platform linker driver and SDK/sysroot; precompiled runtime packs supply the C runtime. Set `SCRIPTC_LINKER` to choose that driver. Runtime development with `--sanitize` additionally needs a C compiler. Node.js 24 or newer is needed for npm installation, development from a source checkout, the JavaScript compiler API, and `scriptc run` of WASI modules. ```console $ npm install -g scriptc diff --git a/RELEASING.md b/RELEASING.md index ae9cd4cf..cd86db4b 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -14,15 +14,7 @@ To prepare a release: CI (`.github/workflows/release.yml`) compares the version in `packages/cli/package.json` to what `scriptc` has on npm. If it differs, it builds platform packages on matching macOS, Linux, and Windows runners, verifies the version spine, and publishes the runtime, helper, compiler, and CLI packages in dependency order. After the publish succeeds, a separate job creates the git tag `v` and the GitHub release with the marked changelog entry as its body, and attaches `surface-manifest.json` — the machine-readable listing of the surface the static tier compiles at that version (stable per-entry ids, so two releases diff mechanically; see `packages/compiler/src/coverage/surface-manifest.ts` for the schema). The job regenerates the manifest from the tree and fails on any byte difference from the committed file before attaching, so the asset is always the manifest of the code being released. The same file ships inside the `@scriptc/compiler` package as `@scriptc/compiler/surface-manifest.json`. -The release job builds and strips each pinned LLVM helper on its matching host, -then builds the matching precompiled runtime pack before publishing the -constrained platform packages ahead of `@scriptc/compiler`. Ordinary LLVM-tier -executables use the helper for the program object and the platform pack for -runtime objects; the user's toolchain performs only the final platform link. -Explicit C builds, LLVM refusals, and `--sanitize` retain the external C -toolchain path. npm postinstall skips local runtime-cache compilation when the -platform pack is available. The GitHub release remains a tag, release notes, and the -manifest asset; the npm publish never waits on the GitHub release. +The release job builds each pinned LLVM helper, precompiled runtime pack, and native CLI on its matching host. The native CLI bundles its TypeScript checker, LLVM helper, runtime pack, declarations, and comptime evaluator. npm postinstall installs the matching native executable; Node is needed for npm installation but not native compilation or execution. The user's toolchain performs the final platform link. Sanitized builds also use it to instrument program LLVM and compile the C runtime. Platform packages publish ahead of `@scriptc/compiler` and `scriptc`. The GitHub release attaches standalone compiler archives, their SHA-256 checksums, and the surface manifest after npm publication succeeds. Publishing uses npm trusted publishing (OIDC) — there is no npm token secret. Each published package must have a GitHub Actions trusted publisher for diff --git a/docs/src/app/platforms/page.mdx b/docs/src/app/platforms/page.mdx index 716a4498..7431895c 100644 --- a/docs/src/app/platforms/page.mdx +++ b/docs/src/app/platforms/page.mdx @@ -2,7 +2,7 @@ ## Owned LLVM targets -Source artifacts selected with --emit=ir|llvm need only Node. On supported hosts, --emit=asm|obj uses the version-matched platform helper installed with scriptc and does not invoke a compiler, archiver, linker, or SDK. Supported assembly/object targets are macOS arm64/x64 (macOS 14.0 artifacts; helper needs macOS 15+), Linux x64/arm64 glibc, Windows x64 MSVC, Linux x64/arm64 musl, and WASI Preview 1. Ordinary LLVM executables additionally need the platform linker and SDK/sysroot; the helper emits the program object and the packaged runtime supplies objects/archives, so that driver compiles neither program nor runtime C. Runtime development with --sanitize requires an external LLVM toolchain and a C compiler for instrumentation. Object output retains undefined runtime references and is not a library archive. +The compiler runs natively and bundles its TypeScript checker and LLVM helper. Outputs selected with --emit=ir|llvm|asm|obj need no Node installation or external compiler, archiver, linker, or SDK. Supported assembly/object targets are macOS arm64/x64 (macOS 14.0 artifacts; helper needs macOS 15+), Linux x64/arm64 glibc, Windows x64 MSVC, Linux x64/arm64 musl, and WASI Preview 1. Executables additionally need the platform linker and SDK/sysroot; the helper emits the program object and the packaged runtime supplies objects/archives. Runtime development with --sanitize requires an external LLVM toolchain and a C compiler for instrumentation. Object output retains undefined runtime references and is not a library archive. ## Cross-compilation via zig @@ -57,7 +57,7 @@ The full library-mode feature set applies: profile-declared exports and ABI entr ### WebAssembly (WASI Preview 1) -Set SCRIPTC_TARGET=wasm32-wasi to produce a standalone .wasm module. Without -o, scriptc build hello.ts writes .scriptc/hello.wasm. scriptc run hosts the module with Node's WASI implementation, inherits stdio and environment, preopens the current working directory as /, and maps the host platform's temporary directory to the guest's /tmp. A built module can run in another WASI Preview 1 host instead. +Set SCRIPTC_TARGET=wasm32-wasi to produce a standalone .wasm module. Without -o, scriptc build hello.ts writes .scriptc/hello.wasm. Compilation does not require Node. scriptc run requires Node.js 24 or newer on PATH to host the module with Node's WASI implementation, inherits stdio and environment, preopens the current working directory as /, and maps the host platform's temporary directory to the guest's /tmp. A built module can run in another WASI Preview 1 host instead. WASI is a production LLVM target with the same language tiers as the native targets. Its 32-bit LLVM ABI supports collections, closures, exceptions, classes, checked dynamic values, async/await, promises, synchronous and asynchronous generators, timers, stdin/readline events, process-exit listeners, filesystem callbacks and promises, and the --dynamic QuickJS island. diff --git a/docs/src/app/quickstart/page.mdx b/docs/src/app/quickstart/page.mdx index b69c6a28..2e2fc92d 100644 --- a/docs/src/app/quickstart/page.mdx +++ b/docs/src/app/quickstart/page.mdx @@ -4,9 +4,9 @@ Install the CLI from npm and compile your first binary in a couple of minutes. ## Prerequisites -- **macOS arm64** is the primary platform ([Linux and Windows](/platforms) are cross-compilation targets). -- **Node ≥ 24** — to run the compiler. The binaries it produces need no Node at all. -- **clang** — preinstalled with the Xcode Command Line Tools; required for executable builds, but not for --emit=ir|c|llvm|asm|obj. Assembly/object output uses the matching helper installed with scriptc and requires macOS 15+. +- **macOS 15+ arm64/x64, Linux arm64/x64, or Windows x64** — see [platform support](/platforms) for target details. +- **Node ≥ 24 and npm** — for npm installation. The installed compiler and its native output run without Node. Standalone distributions are also available from [GitHub Releases](https://github.com/vercel-labs/scriptc/releases). +- **A platform linker and SDK** — for executable builds. On macOS, install the Xcode Command Line Tools. --emit=ir|llvm|asm|obj uses bundled tools and needs no external linker or SDK. ## Install @@ -14,6 +14,8 @@ Install the CLI from npm and compile your first binary in a couple of minutes. $ npm install -g scriptc ``` +Keep optional dependencies and installation scripts enabled so npm can install the native command for your platform. + To work from a clone instead (`pnpm install && pnpm build` in the repo, then `pnpm scriptc` from the repo directory), see the [repository](https://github.com/vercel-labs/scriptc). ## Your first binary diff --git a/package.json b/package.json index d0f57f78..91c1845c 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "scripts": { "build": "pnpm -r --filter \"./packages/*\" run build", "build:fresh": "rm -rf packages/compiler/dist packages/cli/dist node_modules/.cache/scriptc-tsc && pnpm build", - "build:native-compiler": "node --max-old-space-size=8192 --import tsx scripts/build-native-compiler.mts", + "build:native-compiler": "node --max-old-space-size=8192 --import tsx scripts/build-native-cli.mts", "bench:builds": "node scripts/bench-builds.mjs", "test": "vitest run", "test:ts7": "node scripts/test-ts7.mjs", diff --git a/packages/cli-darwin-arm64/package.json b/packages/cli-darwin-arm64/package.json new file mode 100644 index 00000000..3a7afaca --- /dev/null +++ b/packages/cli-darwin-arm64/package.json @@ -0,0 +1,29 @@ +{ + "name": "@scriptc/cli-darwin-arm64", + "version": "0.1.7", + "description": "Native scriptc compiler for darwin-arm64", + "license": "Apache-2.0", + "homepage": "https://scriptc.dev", + "repository": { + "type": "git", + "url": "git+https://github.com/vercel-labs/scriptc.git", + "directory": "packages/cli-darwin-arm64" + }, + "os": [ + "darwin" + ], + "cpu": [ + "arm64" + ], + "files": [ + "dist" + ], + "scripts": { + "build": "node -e \"\"", + "build:native": "node --max-old-space-size=8192 --import tsx ../../scripts/build-native-cli.mts dist", + "prepack": "node ../../scripts/verify-native-cli.mjs ." + }, + "publishConfig": { + "access": "public" + } +} diff --git a/packages/cli-darwin-x64/package.json b/packages/cli-darwin-x64/package.json new file mode 100644 index 00000000..4091cb9d --- /dev/null +++ b/packages/cli-darwin-x64/package.json @@ -0,0 +1,29 @@ +{ + "name": "@scriptc/cli-darwin-x64", + "version": "0.1.7", + "description": "Native scriptc compiler for darwin-x64", + "license": "Apache-2.0", + "homepage": "https://scriptc.dev", + "repository": { + "type": "git", + "url": "git+https://github.com/vercel-labs/scriptc.git", + "directory": "packages/cli-darwin-x64" + }, + "os": [ + "darwin" + ], + "cpu": [ + "x64" + ], + "files": [ + "dist" + ], + "scripts": { + "build": "node -e \"\"", + "build:native": "node --max-old-space-size=8192 --import tsx ../../scripts/build-native-cli.mts dist", + "prepack": "node ../../scripts/verify-native-cli.mjs ." + }, + "publishConfig": { + "access": "public" + } +} diff --git a/packages/cli-linux-arm64-gnu/package.json b/packages/cli-linux-arm64-gnu/package.json new file mode 100644 index 00000000..4910e637 --- /dev/null +++ b/packages/cli-linux-arm64-gnu/package.json @@ -0,0 +1,32 @@ +{ + "name": "@scriptc/cli-linux-arm64-gnu", + "version": "0.1.7", + "description": "Native scriptc compiler for linux-arm64-gnu", + "license": "Apache-2.0", + "homepage": "https://scriptc.dev", + "repository": { + "type": "git", + "url": "git+https://github.com/vercel-labs/scriptc.git", + "directory": "packages/cli-linux-arm64-gnu" + }, + "os": [ + "linux" + ], + "cpu": [ + "arm64" + ], + "files": [ + "dist" + ], + "scripts": { + "build": "node -e \"\"", + "build:native": "node --max-old-space-size=8192 --import tsx ../../scripts/build-native-cli.mts dist", + "prepack": "node ../../scripts/verify-native-cli.mjs ." + }, + "publishConfig": { + "access": "public" + }, + "libc": [ + "glibc" + ] +} diff --git a/packages/cli-linux-arm64-musl/package.json b/packages/cli-linux-arm64-musl/package.json new file mode 100644 index 00000000..d068d734 --- /dev/null +++ b/packages/cli-linux-arm64-musl/package.json @@ -0,0 +1,32 @@ +{ + "name": "@scriptc/cli-linux-arm64-musl", + "version": "0.1.7", + "description": "Native scriptc compiler for linux-arm64-musl", + "license": "Apache-2.0", + "homepage": "https://scriptc.dev", + "repository": { + "type": "git", + "url": "git+https://github.com/vercel-labs/scriptc.git", + "directory": "packages/cli-linux-arm64-musl" + }, + "os": [ + "linux" + ], + "cpu": [ + "arm64" + ], + "files": [ + "dist" + ], + "scripts": { + "build": "node -e \"\"", + "build:native": "node --max-old-space-size=8192 --import tsx ../../scripts/build-native-cli.mts dist", + "prepack": "node ../../scripts/verify-native-cli.mjs ." + }, + "publishConfig": { + "access": "public" + }, + "libc": [ + "musl" + ] +} diff --git a/packages/cli-linux-x64-gnu/package.json b/packages/cli-linux-x64-gnu/package.json new file mode 100644 index 00000000..26b08f88 --- /dev/null +++ b/packages/cli-linux-x64-gnu/package.json @@ -0,0 +1,32 @@ +{ + "name": "@scriptc/cli-linux-x64-gnu", + "version": "0.1.7", + "description": "Native scriptc compiler for linux-x64-gnu", + "license": "Apache-2.0", + "homepage": "https://scriptc.dev", + "repository": { + "type": "git", + "url": "git+https://github.com/vercel-labs/scriptc.git", + "directory": "packages/cli-linux-x64-gnu" + }, + "os": [ + "linux" + ], + "cpu": [ + "x64" + ], + "files": [ + "dist" + ], + "scripts": { + "build": "node -e \"\"", + "build:native": "node --max-old-space-size=8192 --import tsx ../../scripts/build-native-cli.mts dist", + "prepack": "node ../../scripts/verify-native-cli.mjs ." + }, + "publishConfig": { + "access": "public" + }, + "libc": [ + "glibc" + ] +} diff --git a/packages/cli-linux-x64-musl/package.json b/packages/cli-linux-x64-musl/package.json new file mode 100644 index 00000000..89861b1b --- /dev/null +++ b/packages/cli-linux-x64-musl/package.json @@ -0,0 +1,32 @@ +{ + "name": "@scriptc/cli-linux-x64-musl", + "version": "0.1.7", + "description": "Native scriptc compiler for linux-x64-musl", + "license": "Apache-2.0", + "homepage": "https://scriptc.dev", + "repository": { + "type": "git", + "url": "git+https://github.com/vercel-labs/scriptc.git", + "directory": "packages/cli-linux-x64-musl" + }, + "os": [ + "linux" + ], + "cpu": [ + "x64" + ], + "files": [ + "dist" + ], + "scripts": { + "build": "node -e \"\"", + "build:native": "node --max-old-space-size=8192 --import tsx ../../scripts/build-native-cli.mts dist", + "prepack": "node ../../scripts/verify-native-cli.mjs ." + }, + "publishConfig": { + "access": "public" + }, + "libc": [ + "musl" + ] +} diff --git a/packages/cli-win32-x64-msvc/package.json b/packages/cli-win32-x64-msvc/package.json new file mode 100644 index 00000000..4eeb0f5a --- /dev/null +++ b/packages/cli-win32-x64-msvc/package.json @@ -0,0 +1,29 @@ +{ + "name": "@scriptc/cli-win32-x64-msvc", + "version": "0.1.7", + "description": "Native scriptc compiler for win32-x64-msvc", + "license": "Apache-2.0", + "homepage": "https://scriptc.dev", + "repository": { + "type": "git", + "url": "git+https://github.com/vercel-labs/scriptc.git", + "directory": "packages/cli-win32-x64-msvc" + }, + "os": [ + "win32" + ], + "cpu": [ + "x64" + ], + "files": [ + "dist" + ], + "scripts": { + "build": "node -e \"\"", + "build:native": "node --max-old-space-size=8192 --import tsx ../../scripts/build-native-cli.mts dist", + "prepack": "node ../../scripts/verify-native-cli.mjs ." + }, + "publishConfig": { + "access": "public" + } +} diff --git a/packages/cli/README.md b/packages/cli/README.md index 526cbcac..f9e498b9 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -22,7 +22,7 @@ $ scriptc build fib.ts -o fib && ./fib $ npm install -g scriptc ``` -Requires Node.js 24. Executable builds require a platform linker driver and SDK/sysroot. On supported macOS, Linux, and Windows hosts, LLVM executables use the matching optional helper and precompiled runtime pack, so the driver only links; select that driver with `SCRIPTC_LINKER`. Runtime development with `--sanitize` additionally needs a C compiler. `--emit=ir|llvm` requires only Node, while `--emit=asm|obj` requires neither an external compiler nor a linker. +The installed compiler runs natively on supported macOS, Linux, and Windows hosts. Compilation, compile-time evaluation, and native execution do not require Node. `--emit=ir|llvm|asm|obj` uses the bundled TypeScript checker and LLVM helper without an external compiler, archiver, linker, or SDK. Executable builds additionally need a platform linker driver and SDK/sysroot; precompiled runtime packs supply the C runtime. Set `SCRIPTC_LINKER` to choose that driver. Runtime development with `--sanitize` additionally needs a C compiler. Node.js 24 or newer is needed for npm installation and `scriptc run` of WASI modules. Builds use a bounded persistent cache. Unchanged source can reuse the validated frontend result and LLVM program objects. Library identity getters occupy a separate LLVM module, so an identity change can reuse the large program object. Runtime objects come from the installed pack. Executable cache entries verify their native dependencies; FFI builds relink against current external inputs. Set `SCRIPTC_NO_CACHE=1` to bypass the cache or `SCRIPTC_CACHE_DIR` to select its location. An existing POSIX override must already be private. diff --git a/packages/cli/package.json b/packages/cli/package.json index a3e23250..9307bd9b 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,7 +1,7 @@ { "name": "scriptc", "version": "0.1.7", - "description": "Compile ordinary TypeScript and JavaScript to small, fast native executables — no Node, no V8, no JavaScript engine in the binary", + "description": "Compile ordinary TypeScript and JavaScript to small, fast native executables \u2014 no Node, no V8, no JavaScript engine in the binary", "license": "Apache-2.0", "homepage": "https://scriptc.dev", "repository": { @@ -11,19 +11,41 @@ }, "type": "module", "bin": { - "scriptc": "dist/bootstrap.js" + "scriptc": "bin/scriptc.exe" }, "files": [ - "dist", + "bin", "scripts" ], "engines": { "node": ">=24" }, "scripts": { - "build": "node ../../node_modules/typescript/bin/tsc -p tsconfig.json" + "build": "node ../../node_modules/typescript/bin/tsc -p tsconfig.json", + "prepack": "node scripts/prepare-native.mjs", + "postinstall": "node scripts/install-native.mjs" }, - "dependencies": { + "devDependencies": { "@scriptc/compiler": "workspace:*" + }, + "optionalDependencies": { + "@scriptc/cli-darwin-arm64": "workspace:*", + "@scriptc/cli-darwin-x64": "workspace:*", + "@scriptc/cli-linux-arm64-gnu": "workspace:*", + "@scriptc/cli-linux-arm64-musl": "workspace:*", + "@scriptc/cli-linux-x64-gnu": "workspace:*", + "@scriptc/cli-linux-x64-musl": "workspace:*", + "@scriptc/cli-win32-x64-msvc": "workspace:*", + "@scriptc/runtime-darwin-arm64": "workspace:*", + "@scriptc/runtime-darwin-x64": "workspace:*", + "@scriptc/runtime-linux-x64-gnu": "workspace:*", + "@scriptc/runtime-linux-arm64-gnu": "workspace:*", + "@scriptc/runtime-linux-x64-musl": "workspace:*", + "@scriptc/runtime-linux-arm64-musl": "workspace:*", + "@scriptc/runtime-win32-x64-msvc": "workspace:*", + "@scriptc/runtime-wasm32-wasi": "workspace:*", + "@scriptc/runtime-ios-arm64": "workspace:*", + "@scriptc/runtime-ios-simulator-arm64": "workspace:*", + "@scriptc/runtime-android-arm64": "workspace:*" } } diff --git a/packages/cli/scripts/install-native.mjs b/packages/cli/scripts/install-native.mjs new file mode 100644 index 00000000..79a4c17f --- /dev/null +++ b/packages/cli/scripts/install-native.mjs @@ -0,0 +1,113 @@ +import { constants, copyFileSync, chmodSync, existsSync, mkdirSync, readFileSync, realpathSync, renameSync, rmSync, writeFileSync } from "node:fs"; +import { createRequire } from "node:module"; +import { dirname, isAbsolute, join, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +export function nativeCliPackage(platform, architecture, libc) { + const host = `${platform}-${architecture}`; + if (host === "darwin-arm64" || host === "darwin-x64") return `@scriptc/cli-${host}`; + if (host === "win32-x64") return "@scriptc/cli-win32-x64-msvc"; + if (host === "linux-x64" || host === "linux-arm64") { + if (libc !== "glibc" && libc !== "musl") throw new Error("could not identify the Linux C library"); + return `@scriptc/cli-${host}-${libc === "musl" ? "musl" : "gnu"}`; + } + throw new Error(`scriptc has no native command for ${host}`); +} + +function hostLibc() { + if (process.platform !== "linux") return null; + const report = process.report.getReport(); + if (report.header.glibcVersionRuntime) return "glibc"; + if (report.sharedObjects.some((path) => /(?:^|\/)ld-musl-|libc\.musl-/.test(path))) return "musl"; + // A statically linked Node has no loaded libc in its process report. + // Only select musl when its loader is present for this architecture. + const arch = process.arch === "arm64" ? "aarch64" : "x86_64"; + return existsSync(`/lib/ld-musl-${arch}.so.1`) ? "musl" : null; +} + +export function relocateToolchain(manifest, sourceDirectory, destinationDirectory) { + const relocated = { ...manifest }; + const pathFrom = (value) => relative(destinationDirectory, resolve(sourceDirectory, value)); + for (const name of ["ts7", "llvm_package", "runtime_pack", "runtime_sources", "declarations", "comptime", "wasi_node_runner"]) { + if (typeof relocated[name] === "string") relocated[name] = pathFrom(relocated[name]); + } + for (const name of ["linker", "dsymutil", "archiver", "relocatable_linker"]) { + const value = relocated[name]; + if (typeof value === "string" && !isAbsolute(value) && /[/\\]/.test(value)) relocated[name] = pathFrom(value); + } + if (relocated.runtime_packs) relocated.runtime_packs = relocated.runtime_packs.map((pack) => ({ ...pack, path: pathFrom(pack.path) })); + return relocated; +} + +export function installNativeCli(directory, packageName = nativeCliPackage(process.platform, process.arch, hostLibc())) { + directory = realpathSync(directory); + const require = createRequire(join(directory, "package.json")); + const packageManifest = JSON.parse(readFileSync(join(directory, "package.json"), "utf8")); + const version = packageManifest.version; + let platformManifest; + try { platformManifest = require.resolve(`${packageName}/package.json`); } + catch { throw new Error(`scriptc requires ${packageName}@${version}; reinstall with optional dependencies enabled`); } + const identity = JSON.parse(readFileSync(platformManifest, "utf8")); + if (identity.name !== packageName || identity.version !== version) throw new Error(`scriptc requires ${packageName}@${version}, found ${identity.version}`); + const sourceDirectory = join(dirname(platformManifest), "dist", "bin"); + const source = join(sourceDirectory, process.platform === "win32" ? "scriptc.exe" : "scriptc"); + const original = JSON.parse(readFileSync(source + ".json", "utf8")); + if (original.schema !== "scriptc.native-toolchain.v1" || original.compiler_version !== version) throw new Error("native compiler toolchain version does not match this installation"); + const bin = join(directory, "bin"); + mkdirSync(bin, { recursive: true }); + const manifest = relocateToolchain(original, sourceDirectory, bin); + const packs = new Map((manifest.runtime_packs ?? []).map((pack) => [pack.target, pack])); + for (const suffix of ["darwin-arm64", "darwin-x64", "linux-x64-gnu", "linux-arm64-gnu", "linux-x64-musl", "linux-arm64-musl", + "win32-x64-msvc", "wasm32-wasi", "ios-arm64", "ios-simulator-arm64", "android-arm64"]) { + const name = `@scriptc/runtime-${suffix}`; + if (packageManifest.optionalDependencies?.[name] === undefined) continue; + let path; + try { path = require.resolve(`${name}/package.json`); } + catch { continue; } + const packIdentity = JSON.parse(readFileSync(path, "utf8")); + if (packIdentity.name !== name || packIdentity.version !== version) throw new Error(`scriptc requires ${name}@${version}`); + const pack = JSON.parse(readFileSync(join(dirname(path), "runtime-pack.json"), "utf8")); + if (pack.schema !== "scriptc.runtime-pack.v1" || pack.package !== name || pack.version !== version) { + throw new Error(`invalid runtime pack installed for ${name}@${version}`); + } + if (!packs.has(pack.target.name)) packs.set(pack.target.name, { target: pack.target.name, path: relative(bin, dirname(path)) }); + } + manifest.runtime_packs = [...packs.values()]; + // npm normalizes modes for payloads outside package bin entries. These + // tools are launched directly by the compiler after installation. + if (process.platform !== "win32") { + for (const path of [manifest.ts7, manifest.comptime, join(manifest.llvm_package, "bin/scriptc-llvm-codegen")]) { + chmodSync(resolve(bin, path), 0o755); + } + } + // The common filename lets npm's Windows shim invoke a native executable. + // POSIX bin links also execute this file directly, without a JS launcher. + const destination = join(bin, "scriptc.exe"); + const staged = destination + `.install-${process.pid}`; + try { + copyFileSync(source, staged, constants.COPYFILE_FICLONE); + chmodSync(staged, 0o755); + writeFileSync(staged + ".json", JSON.stringify(manifest, null, 2) + "\n"); + renameSync(staged + ".json", destination + ".json"); + renameSync(staged, destination); + } finally { + rmSync(staged, { force: true }); + rmSync(staged + ".json", { force: true }); + } + return destination; +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + try { + const directory = resolve(dirname(fileURLToPath(import.meta.url)), ".."); + const manifest = JSON.parse(readFileSync(join(directory, "package.json"), "utf8")); + // A source checkout builds the native distribution separately. pnpm pack + // replaces workspace ranges with release versions before publication. + const workspace = Object.values(manifest.optionalDependencies ?? {}).some((value) => String(value).startsWith("workspace:")); + if (!workspace) installNativeCli(directory); + } + catch (error) { + process.stderr.write(`scriptc: ${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 1; + } +} diff --git a/packages/cli/scripts/prepare-native.mjs b/packages/cli/scripts/prepare-native.mjs new file mode 100644 index 00000000..0af4064f --- /dev/null +++ b/packages/cli/scripts/prepare-native.mjs @@ -0,0 +1,18 @@ +import { mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +export function prepareNativeCommand(directory) { + const bin = join(directory, "bin"); + mkdirSync(bin, { recursive: true }); + // npm creates bin links before postinstall. The payload must exist in the + // tarball, and no shebang may pin the Windows shim to an interpreter. + // Installation replaces this placeholder with the platform executable. + writeFileSync(join(bin, "scriptc.exe"), + "echo 'scriptc: native installation is incomplete; enable install scripts and reinstall scriptc' >&2\nexit 1\n", { mode: 0o755 }); + rmSync(join(bin, "scriptc.exe.json"), { force: true }); +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + prepareNativeCommand(resolve(dirname(fileURLToPath(import.meta.url)), "..")); +} diff --git a/packages/cli/src/command.ts b/packages/cli/src/command.ts new file mode 100644 index 00000000..1dea65f1 --- /dev/null +++ b/packages/cli/src/command.ts @@ -0,0 +1 @@ +export * from "@scriptc/compiler/cli/command"; diff --git a/packages/cli/src/host.ts b/packages/cli/src/host.ts new file mode 100644 index 00000000..3e612755 --- /dev/null +++ b/packages/cli/src/host.ts @@ -0,0 +1 @@ +export * from "@scriptc/compiler/cli/host"; diff --git a/packages/cli/src/main.ts b/packages/cli/src/main.ts index 35ec1cb3..ad77e5e0 100644 --- a/packages/cli/src/main.ts +++ b/packages/cli/src/main.ts @@ -1,12 +1,9 @@ import { spawn } from "node:child_process"; -import { existsSync, readFileSync, rmSync, statSync } from "node:fs"; -import { dirname, join, resolve } from "node:path"; +import { existsSync, readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; -import { parseArgs } from "node:util"; -import { analyze, compile, compileLibrary, isExactExternalTypeSpecifier, renderDiagnostics, renderCoverage, resolveProvenanceSources, setProvenanceSources, sourceTargetPlatform, warmNativeCaches, type NativeCacheWarmProfile } from "@scriptc/compiler"; -import { resolveOutputOptions } from "./output-options.js"; -import { selectOutputPaths } from "./paths.js"; -import { CLI_OPTIONS, USAGE } from "./usage.js"; +import { analyze, compile, compileLibrary, resolveProvenanceSources, sourceTargetPlatform, warmNativeCaches } from "@scriptc/compiler"; +import { runCli } from "./command.js"; /** The version of the installed package. Read from the manifest rather than * baked in by the build, so a stamped release and a source checkout answer @@ -19,334 +16,20 @@ function version(): string { return manifest.version ?? "unknown"; } -/* The exit discipline: NEVER process.exit() after writing output. stdout/ - * stderr to a PIPE are async streams — process.exit() drops whatever libuv - * hasn't flushed yet, which truncates large diagnostic renders at the pipe - * buffer (observed: 64KB cut mid-code-frame). Every path sets - * process.exitCode and returns instead; Node exits naturally once the - * streams drain. */ -class CliExit extends Error { - constructor(readonly code: number) { - super(`exit ${code}`); - } -} - -function fail(msg: string): never { - process.stderr.write(msg + "\n"); - throw new CliExit(1); -} - -/** parseArgs, with its throw turned into the CLI's own one-line error. - * Unparseable arguments are a USER error — an unknown flag or a missing - * value used to reach the top level as an uncaught ERR_PARSE_ARGS_* and - * print a Node stack trace over the user's terminal. */ -function parseCli(): ReturnType> { - try { - return parseArgs({ options: CLI_OPTIONS, allowPositionals: true, allowNegative: true }); - } catch (err) { - // parseArgs appends a paragraph about `--` and positionals to the - // unknown-option message; the first sentence is the part that names - // what was wrong, and USAGE below already covers what was meant. - const raw = err instanceof Error ? err.message : String(err); - const msg = raw.split("\n")[0]!.split(". ")[0]!; - fail(`scriptc: ${msg}\n\n${USAGE}`); - } -} - -async function main(): Promise { - const { values, positionals } = parseCli(); - const externalTypeArgs = values["external-types"] ?? []; - - if (values.version) { - process.stdout.write(`${version()}\n`); - return 0; - } - - if (values.help || positionals.length === 0) { - process.stdout.write(USAGE); - return values.help ? 0 : 1; - } - - const [command, inputArg] = positionals; - if (command === "cache") { - if (inputArg !== "warm") fail(`unknown cache command "${inputArg ?? ""}" (supported: warm)\n\n${USAGE}`); - if (values.lib || values.dynamic || values.backend !== undefined || values.emit !== undefined || values.print !== undefined || values.ffi !== undefined || values.profile !== undefined || values.strip || values["windows-subsystem"] !== undefined || (values["npm-static"] ?? []).length > 0 || values["provenance-sources"] || externalTypeArgs.length > 0 || values.out !== undefined || values["emit-ir"] || !values["keep-llvm"]) { - fail(`scriptc cache warm takes only native optimization/sanitizer options and profile names\n\n${USAGE}`); - } - const optimization = values.optimization; - if (optimization !== undefined && optimization !== "release" && optimization !== "dev") { - fail(`unknown optimization "${optimization}" (supported: release, dev)\n\n${USAGE}`); - } - const profileArgs = positionals.slice(2); - const knownProfiles = new Set(["runtime", "tls", "dynamic"]); - for (const profile of profileArgs) { - if (!knownProfiles.has(profile as NativeCacheWarmProfile)) { - fail(`unknown cache warm profile "${profile}" (supported: runtime, tls, dynamic)`); - } - } - let result; - try { - result = await warmNativeCaches({ - ...(optimization === undefined ? {} : { optimization }), - sanitize: values.sanitize, - ...(profileArgs.length === 0 - ? {} - : { profiles: profileArgs as NativeCacheWarmProfile[] }), - }); - } catch (error) { - fail(`scriptc: ${error instanceof Error ? error.message : String(error)}`); - } - process.stdout.write(`${result.cacheRoot}\n`); - for (const profile of result.profiles) { - process.stdout.write(`${profile.profile}\t${Math.round(profile.elapsedMs)}ms\n`); - } - return 0; - } - if (command !== "build" && command !== "run" && command !== "coverage") { - fail(`unknown command "${command}"\n\n${USAGE}`); - } - if (values.lib) { - // LIBRARY mode: the profile names the entry module and pins the - // emission; the executable lane's mode flags have no meaning here - // (library artifacts are static-tier only, and there is no fallback - // concept — bare npm specifiers are static-or-refuse: the npm-static - // eligibility bar runs automatically, eligible packages compile into - // the graph, ineligible ones refuse with SC4013). - if (command !== "build") fail(`--lib is a build mode (scriptc build --lib --profile )\n\n${USAGE}`); - const profileArg = values.profile; - if (!profileArg) fail(`scriptc build --lib needs --profile \n\n${USAGE}`); - if (inputArg) { - fail("scriptc build --lib takes no input positional: the profile names the entry module"); - } - if (values.dynamic || values.backend !== undefined || values.emit !== undefined || values.print !== undefined || values.optimization !== undefined || values.strip || values.ffi !== undefined || values["windows-subsystem"] !== undefined || (values["npm-static"] ?? []).length > 0 || externalTypeArgs.length > 0) { - fail( - "scriptc build --lib takes no --dynamic/--backend/--emit/--print/--optimization/--strip/--windows-subsystem/--npm-static/--ffi/--external-types: the profile pins the emission and optimization, npm imports are judged automatically, outbound FFI belongs to executable builds, and external type mappings belong to coverage", - ); - } - const profilePath = resolve(profileArg); - const libOutDir = values.out ? dirname(resolve(values.out)) : join(dirname(profilePath), ".scriptc"); - const result = await compileLibrary({ - profilePath, - outDir: libOutDir, - ...(values.out ? { outPath: resolve(values.out) } : {}), - emitIr: values["emit-ir"], - sanitize: values.sanitize, +process.exitCode = await runCli(process.argv.slice(2), { + version, analyze: async (entry, options) => analyze(entry, options), compile, compileLibrary, resolveProvenanceSources, sourceTargetPlatform, warmNativeCaches, + run: (binary) => new Promise((resolveExit) => { + let child; + if (sourceTargetPlatform() === "wasi") { + const builtRunner = fileURLToPath(new URL("./wasi-runner.js", import.meta.url)); + const runner = existsSync(builtRunner) ? builtRunner : fileURLToPath(new URL("./wasi-runner.ts", import.meta.url)); + child = spawn(process.execPath, [...process.execArgv, "--no-warnings", runner, binary], { stdio: "inherit" }); + } else child = spawn(binary, [], { stdio: "inherit" }); + child.on("exit", (code, signal) => { + if (signal) { + process.stderr.write(`scriptc: program killed by ${signal}\n`); + resolveExit(1); + } else resolveExit(code ?? 0); }); - if (!result.ok) { - const color = process.stderr.isTTY ?? false; - process.stderr.write(renderDiagnostics(result.diagnostics, result.sourceTexts, { color }) + "\n"); - const n = result.diagnostics.length; - process.stderr.write(`\n${n} error${n === 1 ? "" : "s"}.\n`); - return 1; - } - if (!values["keep-llvm"]) rmSync(result.llvmPath, { force: true }); - process.stdout.write(`${result.archivePath}\n`); - // The contract sidecar rides the same invocation when the profile - // declares one — name it so the embedder's tooling knows where to look. - if (result.sidecarPath !== undefined) process.stdout.write(`${result.sidecarPath}\n`); - return 0; - } - if (values["emit-ir"] && (command === "build" || command === "run")) { - process.stderr.write("scriptc: warning: --emit-ir is deprecated; use --emit=ir for IR as the primary output\n"); - } - if (!inputArg) fail(`missing input file\n\n${USAGE}`); - const input = resolve(inputArg); - if (command === "coverage" && values.emit !== undefined) { - fail(`--emit is a build/run option\n\n${USAGE}`); - } - if (command === "coverage" && values.strip) { - fail(`--strip is a build/run option\n\n${USAGE}`); - } - if (values.print !== undefined && values.print !== "native-link-info") { - fail(`unknown print kind "${values.print}" (supported: native-link-info)\n\n${USAGE}`); - } - const printNativeLinkInfo = values.print === "native-link-info"; - if (printNativeLinkInfo && command !== "build") { - fail(`--print=native-link-info is a build option\n\n${USAGE}`); - } - if (printNativeLinkInfo && values.emit !== undefined && values.emit !== "obj") { - fail(`--print=native-link-info requires --emit=obj\n\n${USAGE}`); - } - if (externalTypeArgs.length > 0 && command !== "coverage") { - fail(`--external-types is a coverage-only option\n\n${USAGE}`); - } - const externalTypes: Record = Object.create(null) as Record; - for (const mapping of externalTypeArgs) { - const equals = mapping.indexOf("="); - if (equals <= 0 || equals === mapping.length - 1) { - fail(`invalid --external-types mapping ${JSON.stringify(mapping)} (expected )`); - } - const specifier = mapping.slice(0, equals).trim(); - const declarationArg = mapping.slice(equals + 1).trim(); - if (!isExactExternalTypeSpecifier(specifier)) { - fail(`invalid --external-types specifier ${JSON.stringify(specifier)} (expected an exact bare package specifier)`); - } - if (!/\.d\.(?:ts|mts|cts)$/.test(declarationArg)) { - fail(`invalid --external-types declaration ${JSON.stringify(declarationArg)} (expected a .d.ts, .d.mts, or .d.cts file)`); - } - if (externalTypes[specifier] !== undefined) { - fail(`duplicate --external-types mapping for ${JSON.stringify(specifier)}`); - } - const declarationPath = resolve(declarationArg); - try { - if (!statSync(declarationPath).isFile()) throw new Error("not a file"); - } catch { - fail(`--external-types declaration does not name a readable file: ${declarationPath}`); - } - externalTypes[specifier] = declarationPath; - } - const ffiProfilePath = values.ffi !== undefined ? resolve(values.ffi) : undefined; - if (values.backend !== undefined && values.backend !== "llvm") { - fail(`unknown backend "${values.backend}" (supported: llvm)\n\n${USAGE}`); - } - const optimization = values.optimization; - if (optimization !== undefined && optimization !== "release" && optimization !== "dev") { - fail(`unknown optimization "${optimization}" (supported: release, dev)\n\n${USAGE}`); - } - const windowsSubsystem = values["windows-subsystem"]; - if (windowsSubsystem !== undefined && windowsSubsystem !== "console" && windowsSubsystem !== "gui") { - fail(`unknown Windows subsystem "${windowsSubsystem}" (supported: console, gui)\n\n${USAGE}`); - } - if (windowsSubsystem !== undefined && command === "coverage") { - fail(`--windows-subsystem is only supported for executable builds\n\n${USAGE}`); - } - const output = command === "coverage" - ? null - : resolveOutputOptions(command, { - ...(values.emit === undefined && !printNativeLinkInfo - ? {} - : { emit: values.emit ?? "obj" }), - emitIr: values["emit-ir"], - ...(values.backend === undefined ? {} : { backend: values.backend }), - keepLlvm: values["keep-llvm"], - sanitize: values.sanitize, - ...(values.optimization === undefined ? {} : { optimization: values.optimization }), - strip: values.strip, - ...(windowsSubsystem === undefined ? {} : { windowsSubsystem }), - ...(values.ffi === undefined ? {} : { ffi: values.ffi }), - }); - if (output !== null && !output.ok) fail(`${output.message}\n\n${USAGE}`); - const backend = output?.ok ? output.backend : undefined; - - // --npm-static: repeatable and comma-splittable; the literal "auto" - // switches to eligibility-based detection (mixing "auto" with names - // is rejected — the shapes answer different questions). - const npmStaticRaw = (values["npm-static"] ?? []).flatMap((v) => v.split(",")).map((v) => v.trim()).filter((v) => v !== ""); - let npmStatic: string[] | "auto" | undefined; - if (npmStaticRaw.includes("auto")) { - if (npmStaticRaw.length > 1) fail(`--npm-static auto cannot be combined with package names\n\n${USAGE}`); - npmStatic = "auto"; - } else if (npmStaticRaw.length > 0) { - npmStatic = npmStaticRaw; - } - - // --provenance-sources resolves BEFORE the program loads (tsgo needs the - // source "paths" at creation): attestations and source trees fetch (or - // ride the content-addressed cache / the offline manifest), the registry - // installs, and every fallback prints as a note — never a failure. - const provenance = values["provenance-sources"] ? await resolveProvenanceSources(input) : null; - if (provenance !== null) { - setProvenanceSources(provenance); - for (const pkg of provenance.packages) { - process.stderr.write( - `provenance: ${pkg.name}@${pkg.version} ← ${pkg.repo.replace(/^git\+/, "")} @ ${pkg.commit.slice(0, 12)} (source compiles statically)\n`, - ); - } - for (const note of provenance.notes) process.stderr.write(`provenance: ${note}\n`); - } - - if (command === "coverage") { - const { coverage, sourceTexts } = analyze(input, { - dynamic: values.dynamic, - ...(npmStatic !== undefined ? { npmStatic } : {}), - ...(ffiProfilePath !== undefined ? { ffiProfilePath } : {}), - ...(Object.keys(externalTypes).length > 0 ? { externalTypes } : {}), - }); - const color = process.stdout.isTTY ?? false; - process.stdout.write(renderCoverage(coverage, { color, sourceTexts }) + "\n"); - return coverage.preflightFailed ? 1 : 0; - } - - if (output === null || !output.ok) throw new Error("internal output-option state"); - if (windowsSubsystem !== undefined && sourceTargetPlatform() !== "win32") { - fail(`--windows-subsystem requires a Windows executable target\n\n${USAGE}`); - } - const { outDir, outPath } = selectOutputPaths(input, output.cliOutputKind, values.out); - - let nativeLinkInfo: object | undefined; - const build = async (): Promise => { - const result = await compile(input, { - outPath, - outDir, - outputKind: output.outputKind, - emitIr: output.emitIr, - sanitize: values.sanitize, - dynamic: values.dynamic, - ...(backend !== undefined ? { backend } : {}), - ...(optimization !== undefined ? { optimization } : {}), - ...(values.strip ? { strip: true } : {}), - ...(windowsSubsystem !== undefined ? { windowsSubsystem } : {}), - ...(npmStatic !== undefined ? { npmStatic } : {}), - ...(ffiProfilePath !== undefined ? { ffiProfilePath } : {}), - ...(printNativeLinkInfo ? { nativeLinkInfo: true } : {}), - }); - if (!result.ok) { - const color = process.stderr.isTTY ?? false; - process.stderr.write(renderDiagnostics(result.diagnostics, result.sourceTexts, { color }) + "\n"); - const n = result.diagnostics.length; - process.stderr.write(`\n${n} error${n === 1 ? "" : "s"}.\n`); - throw new CliExit(1); - } - if (result.artifact.kind === "exe") { - if (!values["keep-llvm"]) rmSync(result.artifact.translationUnitPath, { force: true }); - } else if (result.artifact.kind === "obj") { - nativeLinkInfo = result.artifact.nativeLinkInfo; - } - return result.artifact.path; - }; - - const binary = await build(); - - if (command === "run") { - return new Promise((resolveExit) => { - let child; - if (sourceTargetPlatform() === "wasi") { - const builtRunner = fileURLToPath(new URL("./wasi-runner.js", import.meta.url)); - const runner = existsSync(builtRunner) - ? builtRunner - : fileURLToPath(new URL("./wasi-runner.ts", import.meta.url)); - child = spawn( - process.execPath, - [...process.execArgv, "--no-warnings", runner, binary], - { stdio: "inherit" }, - ); - } else { - child = spawn(binary, [], { stdio: "inherit" }); - } - child.on("exit", (code, signal) => { - if (signal) { - process.stderr.write(`scriptc: program killed by ${signal}\n`); - resolveExit(1); - } else { - resolveExit(code ?? 0); - } - }); - }); - } - if (printNativeLinkInfo) { - if (nativeLinkInfo === undefined) throw new Error("internal native-link-info state"); - // Keep stdout pure JSON for tooling; the ordinary artifact path is in - // program.object inside the document. - process.stdout.write(`${JSON.stringify(nativeLinkInfo, null, 2)}\n`); - } else { - process.stdout.write(`${binary}\n`); - } - return 0; -} - -try { - process.exitCode = await main(); -} catch (err) { - if (err instanceof CliExit) process.exitCode = err.code; - else throw err; -} + }), +}); diff --git a/packages/cli/src/output-options.ts b/packages/cli/src/output-options.ts index df5fd24e..f9da6296 100644 --- a/packages/cli/src/output-options.ts +++ b/packages/cli/src/output-options.ts @@ -1,91 +1 @@ -import type { CompileOutputKind } from "@scriptc/compiler"; -import type { CliOutputKind } from "./paths.js"; - -export interface OutputOptionValues { - emit?: string; - emitIr: boolean; - backend?: string; - keepLlvm: boolean; - sanitize: boolean; - optimization?: string; - strip?: boolean; - windowsSubsystem?: string; - ffi?: string; -} - -export type OutputOptionResolution = - | { - ok: true; - outputKind: CompileOutputKind; - cliOutputKind: CliOutputKind; - backend?: "llvm"; - emitIr: boolean; - deprecateEmitIr: boolean; - } - | { ok: false; message: string }; - -const SOURCE_KINDS = new Set(["ir", "llvm"]); -const NATIVE_ARTIFACT_KINDS = new Set(["asm", "obj"]); - -/** Pure compatibility/validation matrix for build/run output selection. */ -export function resolveOutputOptions( - command: "build" | "run", - values: OutputOptionValues, -): OutputOptionResolution { - if (values.backend !== undefined && values.backend !== "llvm") { - return { ok: false, message: `unknown backend "${values.backend}" (supported: llvm)` }; - } - const backend = values.backend as "llvm" | undefined; - const rawEmit = values.emit; - if ( - rawEmit !== undefined && rawEmit !== "ir" && rawEmit !== "llvm" && - rawEmit !== "asm" && rawEmit !== "obj" && rawEmit !== "exe" - ) { - return { - ok: false, - message: `unknown emit kind "${rawEmit}" (supported: ir, llvm, asm, obj, exe)`, - }; - } - const emit = (rawEmit ?? "exe") as CliOutputKind; - if (command === "run" && emit !== "exe") { - return { ok: false, message: `scriptc run requires --emit=exe` }; - } - if (values.emitIr && rawEmit !== undefined && emit !== "ir" && emit !== "exe") { - return { ok: false, message: `--emit-ir cannot be combined with --emit=${emit}; use --emit=ir` }; - } - if (values.emitIr && emit === "ir") { - return { ok: false, message: `--emit-ir and --emit=ir select the same output; use --emit=ir` }; - } - if (values.windowsSubsystem !== undefined && emit !== "exe") { - return { ok: false, message: `--windows-subsystem is only supported with --emit=exe` }; - } - if (values.strip && emit !== "exe") { - return { ok: false, message: `--strip is only supported with --emit=exe` }; - } - if (emit === "ir" && backend !== undefined) { - return { ok: false, message: `--emit=ir cannot be combined with --backend; IR is emitted before backend selection` }; - } - if (SOURCE_KINDS.has(emit)) { - if (!values.keepLlvm) { - return { ok: false, message: `--no-keep-llvm is only meaningful with --emit=exe` }; - } - if (values.sanitize) { - return { ok: false, message: `--sanitize is only meaningful with --emit=exe` }; - } - if (values.optimization !== undefined) { - return { ok: false, message: `--optimization is only meaningful with --emit=exe` }; - } - } - if (NATIVE_ARTIFACT_KINDS.has(emit) && !values.keepLlvm) { - return { ok: false, message: `--no-keep-llvm is only meaningful with --emit=exe` }; - } - const outputKind = emit as CompileOutputKind; - return { - ok: true, - outputKind, - cliOutputKind: emit, - ...(emit === "ir" && backend === undefined ? {} : { backend: "llvm" as const }), - emitIr: values.emitIr && emit === "exe", - deprecateEmitIr: values.emitIr, - }; -} +export * from "@scriptc/compiler/cli/output-options"; diff --git a/packages/cli/src/paths.ts b/packages/cli/src/paths.ts index a803e2a3..0995a357 100644 --- a/packages/cli/src/paths.ts +++ b/packages/cli/src/paths.ts @@ -1,103 +1 @@ -import { tmpdir } from "node:os"; -import { basename, dirname, join, resolve } from "node:path"; -import { buildTargetPlatform, sourceTargetPlatform, wasiGuestPath } from "@scriptc/compiler"; - -export type CliOutputKind = "ir" | "llvm" | "asm" | "obj" | "exe"; - -const POSIX_SUFFIXES: Record = { - ir: ".ir.json", - llvm: ".ll", - asm: ".s", - obj: ".o", - exe: "", -}; - -const WINDOWS_SUFFIXES: Record = { - ...POSIX_SUFFIXES, - asm: ".asm", - obj: ".obj", - exe: ".exe", -}; - -export function defaultOutputName( - stem: string, - kind: CliOutputKind, - platform?: string, -): string { - const selectedPlatform = platform ?? (kind === "exe" ? sourceTargetPlatform() : process.platform); - if (kind === "exe" && selectedPlatform === "wasi") return `${stem}.wasm`; - return `${stem}${(selectedPlatform === "win32" ? WINDOWS_SUFFIXES : POSIX_SUFFIXES)[kind]}`; -} - -export interface OutputPaths { - outDir: string; - outPath: string; -} - -/** One authority for explicit and default primary artifact paths. */ -export function selectOutputPaths( - input: string, - kind: CliOutputKind, - explicitOut?: string, - platform?: string, -): OutputPaths { - const absoluteInput = resolve(input); - const outDir = explicitOut === undefined - ? join(dirname(absoluteInput), ".scriptc") - : dirname(resolve(explicitOut)); - const stem = basename(absoluteInput).replace(/\.(ts|mts|cts|js|mjs|cjs|c|ll)$/, ""); - return { - outDir, - outPath: explicitOut === undefined - ? join(outDir, defaultOutputName(stem, kind, platform)) - : resolve(explicitOut), - }; -} - -/** Default executable filename for the build target. Explicit --out paths - * stay exact; only scriptc's generated default needs the Windows PE suffix. */ -export function defaultExecutableName(stem: string, platform: string = buildTargetPlatform()): string { - return defaultOutputName(stem, "exe", platform); -} - -/** Host paths exposed by `scriptc run` to a WASI Preview 1 module. Guest - * `/tmp` maps to the host's real platform temp directory instead of assuming - * the POSIX spelling exists (notably false on Windows). */ -export function wasiPreopens( - cwd: string = process.cwd(), - hostTmp: string = tmpdir(), -): Record { - return { "/": cwd, "/tmp": hostTmp }; -} - -/** Environment inherited by a WASI module. Host-absolute directory values - * must not claim paths outside the guest namespace: `/` is the module's - * capability root/home/cwd and `/tmp` is its writable temporary directory. */ -export function wasiEnvironment( - env: NodeJS.ProcessEnv = process.env, - cwd: string = process.cwd(), - hostTmp: string = tmpdir(), -): Record { - const guest = Object.fromEntries( - Object.entries(env).filter((entry): entry is [string, string] => entry[1] !== undefined), - ); - - guest["PWD"] = "/"; - guest["HOME"] = "/"; - guest["TMPDIR"] = "/tmp"; - if (guest["USERPROFILE"] !== undefined) guest["USERPROFILE"] = "/"; - if (guest["TMP"] !== undefined) guest["TMP"] = "/tmp"; - if (guest["TEMP"] !== undefined) guest["TEMP"] = "/tmp"; - - // These optional shell/package-manager paths retain their meaning only - // when they fall under a capability the runner actually exposes. - for (const key of ["OLDPWD", "INIT_CWD"] as const) { - const value = guest[key]; - if (value === undefined) continue; - const mapped = wasiGuestPath(value, cwd, hostTmp); - if (mapped === null) delete guest[key]; - else guest[key] = mapped; - } - - return guest; -} +export * from "@scriptc/compiler/cli/paths"; diff --git a/packages/cli/src/usage.ts b/packages/cli/src/usage.ts index 062e2653..dadabbc5 100644 --- a/packages/cli/src/usage.ts +++ b/packages/cli/src/usage.ts @@ -1,86 +1 @@ -export const USAGE = `scriptc — TypeScript/JavaScript to native and WebAssembly executables (experimental) - -Usage: - scriptc build [options] compile to an executable or source artifact - scriptc run [options] compile and run - scriptc coverage how much compiles statically, and why not - scriptc coverage --dynamic what a --dynamic build compiles, and what still blocks it - scriptc coverage --external-types - type-resolve an embedder-provided module for analysis - scriptc build --lib --profile library mode: compile the profile's entry - module to a linkable static archive - (.lib.a), or a Wasm reactor - (.wasm) on wasm32-wasi, - exporting the profile symbols; a profile - with a sidecar section also gets the - contract sidecar JSON beside the archive - scriptc cache warm [runtime|tls|dynamic…] prebuild expensive native cache families - for the current compiler/SDK/target - -Options: - -o, --out primary output path (default: .scriptc/) - --emit primary output: ir, llvm, asm, obj, or exe - (default: exe). asm/obj use the matching platform helper - --print print machine-readable metadata instead of the output path - (native-link-info implies --emit=obj and never links) - --backend code generator (llvm) - --optimization - native optimization posture (default: release/-O2). dev - uses -O0, source breakpoints, and cached LLVM object shards; - macOS executable builds also produce an adjacent .dSYM - --strip remove symbol/debug payload from the linked executable - for smaller builds (opt in; --emit=exe only) - --windows-subsystem - Windows executable subsystem (default: console). gui - prevents Windows from opening a console window - --keep-llvm keep generated LLVM IR beside the executable (default) - --no-keep-llvm delete generated LLVM IR after compiling - --emit-ir also write IR beside an executable or library archive; - deprecated for executables: use --emit=ir for primary IR - --sanitize build with ASan + runtime RC audit - --dynamic embed the dynamic engine (adds ~620KB; static stays the default) - --ffi bind signature-only TypeScript declarations to native - C symbols and link the manifest's archives/libraries - --npm-static - compile the named npm packages' shipped JS statically as - program modules (repeatable; "auto" opts in every eligible - direct import: own .d.ts, unminified JS, no build-transform - markers). A package preflight refuses falls back to the - island (--dynamic) with a coverage-report note — opt-in, - experimental - --provenance-sources - EXPERIMENTAL: compile npm dependencies from their - provenance-attested SOURCE (fetched at the attested - commit) as static program modules; packages without a - usable attestation keep the island path (a note, never - a failure) - --external-types - coverage only: map an exact bare module specifier to a - local declaration file. The declaration supplies types - for analysis; the host module remains an explicit - external-boundary blocker (repeatable) - -h, --help show this help - -v, --version print the version -`; - -export const CLI_OPTIONS = { - out: { type: "string", short: "o" }, - emit: { type: "string" }, - print: { type: "string" }, - backend: { type: "string" }, - optimization: { type: "string" }, - strip: { type: "boolean", default: false }, - "windows-subsystem": { type: "string" }, - "keep-llvm": { type: "boolean", default: true }, - "emit-ir": { type: "boolean", default: false }, - sanitize: { type: "boolean", default: false }, - dynamic: { type: "boolean", default: false }, - ffi: { type: "string" }, - "npm-static": { type: "string", multiple: true }, - "provenance-sources": { type: "boolean", default: false }, - "external-types": { type: "string", multiple: true }, - lib: { type: "boolean", default: false }, - profile: { type: "string" }, - help: { type: "boolean", short: "h", default: false }, - version: { type: "boolean", short: "v", default: false }, -} as const; +export * from "@scriptc/compiler/cli/usage"; diff --git a/packages/cli/src/wasi-runner.ts b/packages/cli/src/wasi-runner.ts index 932100c4..3a885260 100644 --- a/packages/cli/src/wasi-runner.ts +++ b/packages/cli/src/wasi-runner.ts @@ -1,39 +1,2 @@ #!/usr/bin/env node -/* The subprocess host for `scriptc run` on wasm32-wasi. Keeping the WASI - * instance outside the CLI process preserves native run's exit isolation: - * process.exit(), traps, and signals cannot take the compiler process down. - */ -import { readFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { WASI } from "node:wasi"; -import { wasiGuestPath } from "@scriptc/compiler"; -import { wasiEnvironment, wasiPreopens } from "./paths.js"; - -type WasiInstance = Parameters[0]; -declare const WebAssembly: { - instantiate( - bytes: Uint8Array, - imports: ReturnType, - ): Promise<{ instance: WasiInstance }>; -}; - -const binary = process.argv[2]; -if (binary === undefined) throw new Error("scriptc WASI runner needs a module path"); -const cwd = process.cwd(); -const hostTmp = tmpdir(); - -const wasi = new WASI({ - version: "preview1", - args: [wasiGuestPath(binary, cwd, hostTmp) ?? binary], - env: wasiEnvironment(process.env, cwd, hostTmp), - // A native scriptc executable inherits access to the caller's filesystem. - // WASI is capability-based, so expose the caller's working tree as `/` - // and the platform's actual temporary directory as guest `/tmp`. - preopens: wasiPreopens(cwd, hostTmp), - returnOnExit: true, -}); -const instantiated = await WebAssembly.instantiate( - await readFile(binary), - wasi.getImportObject(), -); -process.exitCode = wasi.start(instantiated.instance); +import "@scriptc/compiler/cli/wasi-runner"; diff --git a/packages/cli/test/native-install.test.ts b/packages/cli/test/native-install.test.ts new file mode 100644 index 00000000..60982102 --- /dev/null +++ b/packages/cli/test/native-install.test.ts @@ -0,0 +1,131 @@ +import { execFileSync } from "node:child_process"; +import { copyFileSync, mkdtempSync, mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { afterEach, expect, test } from "vitest"; +import { installNativeCli, nativeCliPackage } from "../scripts/install-native.mjs"; +import { prepareNativeCommand } from "../scripts/prepare-native.mjs"; + +const roots: string[] = []; +afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); + +function fixture() { + const root = mkdtempSync(join(process.platform === "win32" ? tmpdir() : "/tmp", "scriptc-install-")); + roots.push(root); + const directory = join(root, "installation"); + const packageName = "@scriptc/cli-darwin-arm64"; + const platform = join(directory, "node_modules", packageName); + const bin = join(platform, "dist/bin"); + mkdirSync(bin, { recursive: true }); + writeFileSync(join(directory, "package.json"), JSON.stringify({ name: "scriptc", version: "1.2.3" })); + writeFileSync(join(platform, "package.json"), JSON.stringify({ name: packageName, version: "1.2.3" })); + const binary = join(bin, process.platform === "win32" ? "scriptc.exe" : "scriptc"); + writeFileSync(binary, "native compiler payload"); + const toolchain = { + schema: "scriptc.native-toolchain.v1", compiler_version: "1.2.3", target: "macos-arm64", + ts7: "../lib/typescript/lib/tsc", llvm_package: "../lib/llvm", runtime_pack: "../lib/runtime", + runtime_packs: [{ target: "macos-arm64", path: "../lib/runtime" }], + linker: "clang", linker_args: [], dsymutil: "dsymutil", comptime: "../lib/comptime", + wasi_node_runner: "../lib/wasi/cli/wasi-runner.js", + }; + writeFileSync(binary + ".json", JSON.stringify(toolchain)); + const helpers = [toolchain.ts7, toolchain.comptime, join(toolchain.llvm_package, "bin/scriptc-llvm-codegen")] + .map((path) => resolve(bin, path)); + for (const path of helpers) { + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, "native helper payload", { mode: 0o644 }); + } + return { root, directory, packageName, platform, binary, toolchain, helpers }; +} + +test("selects only supported native host packages, including Linux libc", () => { + expect(nativeCliPackage("darwin", "arm64", null)).toBe("@scriptc/cli-darwin-arm64"); + expect(nativeCliPackage("win32", "x64", null)).toBe("@scriptc/cli-win32-x64-msvc"); + expect(nativeCliPackage("linux", "x64", "glibc")).toBe("@scriptc/cli-linux-x64-gnu"); + expect(nativeCliPackage("linux", "arm64", "musl")).toBe("@scriptc/cli-linux-arm64-musl"); + expect(() => nativeCliPackage("linux", "x64", null)).toThrow("identify"); + expect(() => nativeCliPackage("win32", "arm64", null)).toThrow("no native command"); +}); + +test("installs a direct executable and relocatable references to platform assets", () => { + const f = fixture(); + const command = installNativeCli(f.directory, f.packageName); + expect(readFileSync(command, "utf8")).toBe("native compiler payload"); + if (process.platform !== "win32") expect(statSync(command).mode & 0o111).toBe(0o111); + if (process.platform !== "win32") { + for (const helper of f.helpers) expect(statSync(helper).mode & 0o111).toBe(0o111); + } + const moved = join(f.root, "relocated"); + renameSync(f.directory, moved); + const installed = join(moved, "bin/scriptc.exe"); + const manifest = JSON.parse(readFileSync(installed + ".json", "utf8")); + const platform = join(moved, "node_modules", f.packageName, "dist/lib"); + expect(resolve(dirname(installed), manifest.ts7)).toBe(join(platform, "typescript/lib/tsc")); + expect(resolve(dirname(installed), manifest.wasi_node_runner)).toBe(join(platform, "wasi/cli/wasi-runner.js")); + expect(resolve(dirname(installed), manifest.runtime_packs[0].path)).toBe(join(platform, "runtime")); + expect(manifest.linker).toBe("clang"); + expect(JSON.stringify(manifest)).not.toContain(f.directory); +}); + +test("missing and mismatched packages fail without replacing an installed command", () => { + const f = fixture(); + const command = installNativeCli(f.directory, f.packageName); + writeFileSync(join(f.platform, "package.json"), JSON.stringify({ name: f.packageName, version: "1.2.4" })); + expect(() => installNativeCli(f.directory, f.packageName)).toThrow("found 1.2.4"); + expect(() => installNativeCli(f.directory, "@scriptc/cli-missing")).toThrow("optional dependencies"); + expect(readFileSync(command, "utf8")).toBe("native compiler payload"); +}); + +test("npm links the installed native executable without an interpreter", () => { + const f = fixture(); + // Keep the package payload small while exercising npm's native shim. + // Production command coverage builds the real compiler in the bootstrap gate. + const windows = process.platform === "win32"; + if (windows) copyFileSync(process.execPath, f.binary); + else { + const source = join(f.root, "payload.c"); + writeFileSync(source, '#include \nint main(void) { puts("native compiler"); return 0; }\n'); + execFileSync("clang", [source, "-o", f.binary]); + } + for (const helper of f.helpers) writeFileSync(helper, "#!/bin/sh\nprintf 'native helper\\n'\n"); + const packageName = "scriptc-native-install-test"; + const scripts = join(f.directory, "scripts"); + mkdirSync(scripts); + copyFileSync(join(import.meta.dirname, "../scripts/install-native.mjs"), join(scripts, "install-native.mjs")); + writeFileSync(join(scripts, "fixture-install.mjs"), + `import { installNativeCli } from './install-native.mjs'; installNativeCli(process.cwd(), ${JSON.stringify(f.packageName)});\n`); + const npm = process.platform === "win32" ? "npm.cmd" : "npm"; + const npmEnv = { ...process.env, NODE_PATH: "", npm_config_cache: join(f.root, "npm-cache"), npm_config_update_notifier: "false" }; + const pack = (directory: string): string => { + const result = execFileSync(npm, ["pack", "--json", "--ignore-scripts"], { + cwd: directory, env: npmEnv, encoding: "utf8", shell: process.platform === "win32", + }); + return join(directory, (JSON.parse(result) as { filename: string }[])[0]!.filename); + }; + const platformTarball = pack(f.platform); + writeFileSync(join(f.directory, "package.json"), JSON.stringify({ + name: packageName, version: "1.2.3", type: "module", + bin: { [packageName]: "bin/scriptc.exe" }, files: ["bin", "scripts"], + scripts: { postinstall: "node scripts/fixture-install.mjs" }, + optionalDependencies: { [f.packageName]: "file:" + platformTarball }, + })); + prepareNativeCommand(f.directory); + const tarball = pack(f.directory); + const installed = join(f.root, "npm-install"); + mkdirSync(installed); + execFileSync(npm, ["install", "--offline", "--no-audit", "--no-fund", tarball], { + cwd: installed, env: npmEnv, shell: process.platform === "win32", stdio: "pipe", + }); + const command = join(installed, "node_modules/.bin", packageName + (process.platform === "win32" ? ".cmd" : "")); + const result = execFileSync(command, windows ? ["--version"] : ["native compiler"], { + env: { ...process.env, PATH: "" }, encoding: "utf8", shell: process.platform === "win32", + }); + expect(result.trim()).toBe(windows ? process.version : "native compiler"); + if (process.platform !== "win32") { + const bin = join(installed, "node_modules", packageName, "bin"); + const manifest = JSON.parse(readFileSync(join(bin, "scriptc.exe.json"), "utf8")); + for (const helper of [manifest.ts7, manifest.comptime, join(manifest.llvm_package, "bin/scriptc-llvm-codegen")]) { + expect(execFileSync(resolve(bin, helper), [], { env: { ...process.env, PATH: "" }, encoding: "utf8" }).trim()).toBe("native helper"); + } + } +}, 60_000); diff --git a/packages/cli/test/wasi-runtime-pack.test.ts b/packages/cli/test/wasi-runtime-pack.test.ts index 1a36bfbf..9faaa32a 100644 --- a/packages/cli/test/wasi-runtime-pack.test.ts +++ b/packages/cli/test/wasi-runtime-pack.test.ts @@ -42,7 +42,7 @@ test.runIf(supported)("WASI helper object plus runtime pack builds and runs with })).resolves.toMatchObject({ stdout: "hello world\n" }); }); -test.runIf(supported)("switching WASI LLVM and native C builds preserves both translation units", async () => { +test.runIf(supported)("switching WASI and native builds retains both executables", async () => { const dir = await mkdtemp(join(tmpdir(), "scriptc-wasi-native-output-")); dirs.push(dir); const entry = join(dir, "hello.ts"); @@ -59,10 +59,12 @@ test.runIf(supported)("switching WASI LLVM and native C builds preserves both tr const llvm = await readFile(join(outDir, "hello.ll")); const wasm = await readFile(join(outDir, "hello.wasm")); await build(["--backend=llvm", "--keep-llvm"], nativeEnv); - const c = await readFile(join(outDir, "hello.c")); - expect(await readFile(join(outDir, "hello.ll"))).toEqual(llvm); + const native = await readFile(join(outDir, process.platform === "win32" ? "hello.exe" : "hello")); + const nativeLlvm = await readFile(join(outDir, "hello.ll")); + expect(nativeLlvm.equals(llvm)).toBe(false); expect(await readFile(join(outDir, "hello.wasm"))).toEqual(wasm); await build([], wasiEnv); - expect(await readFile(join(outDir, "hello.c"))).toEqual(c); + expect(await readFile(join(outDir, "hello.ll"))).toEqual(llvm); + expect(await readFile(join(outDir, process.platform === "win32" ? "hello.exe" : "hello"))).toEqual(native); }); diff --git a/packages/compiler/ambient/scriptc-node-fallback.d.ts b/packages/compiler/ambient/scriptc-node-fallback.d.ts index 5bb51257..6197935a 100644 --- a/packages/compiler/ambient/scriptc-node-fallback.d.ts +++ b/packages/compiler/ambient/scriptc-node-fallback.d.ts @@ -1069,9 +1069,8 @@ declare module "node:fs" { ): void; /* The bare-encoding spelling — the options record's encoding key alone. */ export function writeFileSync(path: string, data: string, encoding: "utf8" | "utf-8"): void; - export function writeFileSync(path: string, data: Uint8Array): void; - export function appendFileSync(path: string, data: string): void; - export function appendFileSync(path: string, data: Uint8Array): void; + export function writeFileSync(path: string, data: string | Uint8Array): void; + export function appendFileSync(path: string, data: string | Uint8Array): void; export function existsSync(path: string): boolean; export function mkdirSync(path: string): void; export function mkdirSync(path: string, options: { recursive?: boolean; mode?: number }): void; diff --git a/packages/compiler/native/comptime.c b/packages/compiler/native/comptime.c new file mode 100644 index 00000000..8b3ff3ef --- /dev/null +++ b/packages/compiler/native/comptime.c @@ -0,0 +1,189 @@ +/* Isolated JavaScript evaluation for comptime callbacks. No filesystem, + * process, module loader, or console bindings enter the guest context. */ +#ifndef _WIN32 +#define _POSIX_C_SOURCE 200809L +#endif +#include "quickjs.h" +#include +#include +#include +#include +#include +#include +#ifdef _WIN32 +#include +#else +#include +#endif + +static uint64_t now_ms(void) { +#ifdef _WIN32 + return GetTickCount64(); +#else + struct timespec ts; + if (clock_gettime(CLOCK_MONOTONIC, &ts) != 0) return 0; + return (uint64_t)ts.tv_sec * 1000 + (uint64_t)ts.tv_nsec / 1000000; +#endif +} + +typedef struct { uint64_t deadline; int interrupted; } Budget; +static int interrupt(JSRuntime *rt, void *opaque) { + (void)rt; + Budget *budget = opaque; + if (now_ms() >= budget->deadline) budget->interrupted = 1; + return budget->interrupted; +} + +typedef struct { + char *data; + size_t length, capacity; + void *active[512]; + size_t depth; +} Output; + +static int append(Output *out, const char *text) { + size_t size = strlen(text); + if (size > 64 * 1024 * 1024 - out->length) return -1; + size_t required = out->length + size + 1; + if (required > out->capacity) { + size_t capacity = out->capacity ? out->capacity : 1024; + while (capacity < required) capacity *= 2; + char *data = realloc(out->data, capacity); + if (!data) return -1; + out->data = data; + out->capacity = capacity; + } + memcpy(out->data + out->length, text, size + 1); + out->length += size; + return 0; +} + +static int quote(JSContext *ctx, Output *out, JSValueConst value) { + size_t length; + const uint16_t *text = JS_ToCStringLenUTF16(ctx, &length, value); + if (!text) return -1; + int status = append(out, "\""); + for (size_t i = 0; status == 0 && i < length; i++) { + char escaped[7]; + unsigned ch = text[i]; + if (ch >= 32 && ch < 127 && ch != '"' && ch != '\\') { + escaped[0] = (char)ch; escaped[1] = 0; + } else snprintf(escaped, sizeof(escaped), "\\u%04x", ch); + status = append(out, escaped); + } + JS_FreeCStringUTF16(ctx, text); + return status == 0 ? append(out, "\"") : status; +} + +/* Serialize through engine APIs, outside the callback's mutable globals. + * The guest cannot replace JSON.stringify or install a toJSON hook that + * changes the value the compiler validates. */ +static int encode(JSContext *ctx, Output *out, JSValueConst value) { + if (JS_IsException(value)) return -1; + if (JS_IsUndefined(value)) return append(out, "[\"undefined\"]"); + if (JS_IsNull(value)) return append(out, "[\"null\"]"); + if (JS_IsBool(value)) return append(out, JS_ToBool(ctx, value) ? "[\"boolean\",true]" : "[\"boolean\",false]"); + if (JS_IsFunction(ctx, value)) return append(out, "[\"function\"]"); + if (JS_IsSymbol(value)) return append(out, "[\"symbol\"]"); + if (JS_IsString(value) || JS_IsNumber(value) || JS_IsBigInt(value)) { + const char *tag = JS_IsString(value) ? "[\"string\"," : JS_IsNumber(value) ? "[\"number\"," : "[\"bigint\","; + if (append(out, tag) != 0) return -1; + double number = 1; + if (JS_IsNumber(value) && JS_ToFloat64(ctx, &number, value) < 0) return -1; + int status = number == 0 && signbit(number) ? append(out, "\"-0\"") : quote(ctx, out, value); + return status == 0 ? append(out, "]") : status; + } + if (out->depth == 512) { JS_ThrowTypeError(ctx, "compile-time result is nested too deeply"); return -1; } + void *identity = JS_VALUE_GET_PTR(value); + for (size_t i = 0; i < out->depth; i++) { + if (out->active[i] == identity) { JS_ThrowTypeError(ctx, "cyclic compile-time result"); return -1; } + } + out->active[out->depth++] = identity; + int status = 0; + if (JS_IsArray(value)) { + int64_t length = 0; + if (JS_GetLength(ctx, value, &length) < 0 || length < 0 || length > 1000000) status = -1; + if (status == 0) status = append(out, "[\"array\",["); + for (int64_t i = 0; status == 0 && i < length; i++) { + if (i != 0) status = append(out, ","); + JSValue item = JS_GetPropertyUint32(ctx, value, (uint32_t)i); + if (status == 0) status = encode(ctx, out, item); + JS_FreeValue(ctx, item); + } + } else { + JSPropertyEnum *properties = NULL; + uint32_t length = 0; + status = JS_GetOwnPropertyNames(ctx, &properties, &length, value, JS_GPN_STRING_MASK | JS_GPN_ENUM_ONLY); + if (status == 0) status = append(out, "[\"object\",["); + for (uint32_t i = 0; status == 0 && i < length; i++) { + if (i != 0) status = append(out, ","); + if (status == 0) status = append(out, "["); + JSValue key = JS_AtomToString(ctx, properties[i].atom); + if (status == 0) status = quote(ctx, out, key); + JS_FreeValue(ctx, key); + if (status == 0) status = append(out, ","); + JSValue item = JS_GetProperty(ctx, value, properties[i].atom); + if (status == 0) status = encode(ctx, out, item); + JS_FreeValue(ctx, item); + if (status == 0) status = append(out, "]"); + } + JS_FreePropertyEnum(ctx, properties, length); + } + out->depth--; + return status == 0 ? append(out, "]]") : status; +} + +int main(int argc, char **argv) { + if (argc != 3) { fputs("usage: scriptc-comptime \n", stderr); return 2; } + char *end = NULL; + errno = 0; + long timeout = strtol(argv[2], &end, 10); + if (errno || !end || *end || timeout < 1 || timeout > 60000) { + fputs("invalid compile-time budget\n", stderr); return 2; + } + FILE *file = fopen(argv[1], "rb"); + if (!file) { perror("compile-time input"); return 2; } + if (fseek(file, 0, SEEK_END) != 0) { fclose(file); return 2; } + long size = ftell(file); + if (size < 0 || size > 16 * 1024 * 1024 || fseek(file, 0, SEEK_SET) != 0) { fclose(file); return 2; } + char *source = malloc((size_t)size + 1); + if (!source) { fclose(file); return 2; } + if (fread(source, 1, (size_t)size, file) != (size_t)size) { free(source); fclose(file); return 2; } + fclose(file); + source[size] = 0; + JSRuntime *rt = JS_NewRuntime(); + if (!rt) { free(source); return 2; } + JS_SetMemoryLimit(rt, 256 * 1024 * 1024); + JS_SetMaxStackSize(rt, 2 * 1024 * 1024); + Budget budget = { now_ms() + (uint64_t)timeout, 0 }; + JS_SetInterruptHandler(rt, interrupt, &budget); + JSContext *ctx = JS_NewContext(rt); + if (!ctx) { JS_FreeRuntime(rt); free(source); return 2; } + JSValue result = JS_Eval(ctx, source, (size_t)size, "comptime", JS_EVAL_TYPE_GLOBAL); + free(source); + int status = 0; + Output out = {0}; + if (JS_IsException(result) || encode(ctx, &out, result) != 0) { + free(out.data); + memset(&out, 0, sizeof(out)); + JSValue exception = JS_GetException(ctx); + if (budget.interrupted) status = append(&out, "[\"error\",\"ERR_SCRIPT_EXECUTION_TIMEOUT\",\"compile-time evaluation timed out\"]"); + else { + JSValue message = JS_GetPropertyStr(ctx, exception, "message"); + JSValue detail = JS_IsUndefined(message) ? JS_IsNull(exception) + ? JS_NewString(ctx, "compile-time result exceeds the output limit") : JS_DupValue(ctx, exception) : JS_DupValue(ctx, message); + status = append(&out, "[\"error\",\"\","); + if (status == 0) status = quote(ctx, &out, detail); + if (status == 0) status = append(&out, "]"); + JS_FreeValue(ctx, detail); + JS_FreeValue(ctx, message); + } + JS_FreeValue(ctx, exception); + } + if (status == 0 && (fwrite(out.data, 1, out.length, stdout) != out.length || fputc('\n', stdout) == EOF)) status = 2; + free(out.data); + JS_FreeValue(ctx, result); + JS_FreeContext(ctx); + JS_FreeRuntime(rt); + return status == 0 ? 0 : 2; +} diff --git a/packages/compiler/native/host.c b/packages/compiler/native/host.c new file mode 100644 index 00000000..75c2dcba --- /dev/null +++ b/packages/compiler/native/host.c @@ -0,0 +1,46 @@ +#ifndef _WIN32 +#define _POSIX_C_SOURCE 200809L +#endif +#include +#include +#include +#include +#include +#ifdef _WIN32 +#define WIN32_LEAN_AND_MEAN +#include +#else +#include +#include +#endif + +/* Accept only directories owned by this user with private POSIX access. + * Windows inherits the user's cache ACL; reject reparse points so an + * installed cache directory cannot redirect compiler payloads elsewhere. */ +uint8_t scriptc_native_private_directory(const uint8_t *bytes, size_t size, uint8_t harden) { + if (size == 0 || memchr(bytes, 0, size) != NULL) return 0; +#ifdef _WIN32 + (void)harden; + if (size > INT_MAX) return 0; + int length = MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, (const char *)bytes, (int)size, NULL, 0); + if (length <= 0) return 0; + wchar_t *path = malloc(((size_t)length + 1) * sizeof(wchar_t)); + if (!path) return 0; + if (MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, (const char *)bytes, (int)size, path, length) != length) { free(path); return 0; } + path[length] = 0; + DWORD attrs = GetFileAttributesW(path); + free(path); + return attrs != INVALID_FILE_ATTRIBUTES && (attrs & FILE_ATTRIBUTE_DIRECTORY) != 0 && (attrs & FILE_ATTRIBUTE_REPARSE_POINT) == 0; +#else + char *path = malloc(size + 1); + if (!path) return 0; + memcpy(path, bytes, size); + path[size] = 0; + struct stat info; + int ok = lstat(path, &info) == 0 && S_ISDIR(info.st_mode) && info.st_uid == getuid(); + if (ok && harden && (info.st_mode & 077) != 0) ok = chmod(path, 0700) == 0; + if (ok) ok = lstat(path, &info) == 0 && S_ISDIR(info.st_mode) && info.st_uid == getuid() && (info.st_mode & 077) == 0; + free(path); + return ok; +#endif +} diff --git a/packages/compiler/native/host.ffi.json b/packages/compiler/native/host.ffi.json new file mode 100644 index 00000000..f2a73805 --- /dev/null +++ b/packages/compiler/native/host.ffi.json @@ -0,0 +1,6 @@ +{ + "ffi_format": 6, + "functions": [ + { "name": "compilerNativePrivateDirectory", "symbol": "scriptc_native_private_directory", "params": ["string", "bool"], "returns": "bool" } + ] +} diff --git a/packages/compiler/package.json b/packages/compiler/package.json index b77894ff..0c7a0ba0 100644 --- a/packages/compiler/package.json +++ b/packages/compiler/package.json @@ -18,7 +18,13 @@ "./scriptc.d.ts": "./ambient/scriptc.d.ts", "./scriptc-overrides.d.ts": "./ambient/scriptc-overrides.d.ts", "./scriptc-node-fallback.d.ts": "./ambient/scriptc-node-fallback.d.ts", - "./surface-manifest.json": "./surface-manifest.json" + "./surface-manifest.json": "./surface-manifest.json", + "./cli/command": "./dist/cli/command.js", + "./cli/host": "./dist/cli/host.js", + "./cli/paths": "./dist/cli/paths.js", + "./cli/output-options": "./dist/cli/output-options.js", + "./cli/usage": "./dist/cli/usage.js", + "./cli/wasi-runner": "./dist/cli/wasi-runner.js" }, "files": [ "dist", diff --git a/packages/compiler/src/backend/build-cache.ts b/packages/compiler/src/backend/build-cache.ts index cb4044f9..f81ca81a 100644 --- a/packages/compiler/src/backend/build-cache.ts +++ b/packages/compiler/src/backend/build-cache.ts @@ -1,7 +1,8 @@ +import { resolveBuildCacheRoot } from "./cache-root.js"; +export { resolveBuildCacheRoot } from "./cache-root.js"; import { createHash } from "node:crypto"; import { chmod, copyFile, mkdir, readdir, readFile, rename, rm, stat, unlink, utimes, writeFile } from "node:fs/promises"; -import { homedir } from "node:os"; -import { basename, dirname, join, resolve } from "node:path"; +import { basename, dirname, join } from "node:path"; export async function fileExists(path: string): Promise { return stat(path).then( @@ -44,28 +45,6 @@ export async function installArtifact( } } -/** Resolve the build cache without touching the filesystem. Exported from this - * internal module so its platform and override behavior can be pinned directly. */ -export function resolveBuildCacheRoot( - env: NodeJS.ProcessEnv = process.env, - platform: NodeJS.Platform = process.platform, - userHome: string = homedir(), -): string | null { - if (env["SCRIPTC_NO_CACHE"] === "1") return null; - const configured = env["SCRIPTC_CACHE_DIR"]; - if (configured !== undefined) return configured === "" ? null : resolve(configured); - - const xdg = env["XDG_CACHE_HOME"]; - if (xdg !== undefined && xdg !== "") return resolve(xdg, "scriptc", "build"); - if (platform === "win32") { - const local = env["LOCALAPPDATA"]; - if (local !== undefined && local !== "") return resolve(local, "scriptc", "cache", "build"); - } - return platform === "darwin" - ? resolve(userHome, "Library", "Caches", "scriptc", "build") - : resolve(userHome, ".cache", "scriptc", "build"); -} - /** Shared persistent-cache root for compiler-level tiers. The early library * cache deliberately follows the native cache's activation and hard-disable * contract, while native compilation retains ownership of toolchain safety. */ diff --git a/packages/compiler/src/backend/cache-root.ts b/packages/compiler/src/backend/cache-root.ts new file mode 100644 index 00000000..3d668713 --- /dev/null +++ b/packages/compiler/src/backend/cache-root.ts @@ -0,0 +1,24 @@ +import { homedir } from "node:os"; +import { resolve } from "node:path"; + +/** Resolve the build cache without touching the filesystem. Exported from this + * internal module so its platform and override behavior can be pinned directly. */ +export function resolveBuildCacheRoot( + env: NodeJS.ProcessEnv = process.env, + platform: NodeJS.Platform = process.platform, + userHome: string = homedir(), +): string | null { + if (env["SCRIPTC_NO_CACHE"] === "1") return null; + const configured = env["SCRIPTC_CACHE_DIR"]; + if (configured !== undefined) return configured === "" ? null : resolve(configured); + + const xdg = env["XDG_CACHE_HOME"]; + if (xdg !== undefined && xdg !== "") return resolve(xdg, "scriptc", "build"); + if (platform === "win32") { + const local = env["LOCALAPPDATA"]; + if (local !== undefined && local !== "") return resolve(local, "scriptc", "cache", "build"); + } + return platform === "darwin" + ? resolve(userHome, "Library", "Caches", "scriptc", "build") + : resolve(userHome, ".cache", "scriptc", "build"); +} diff --git a/packages/compiler/src/backend/link-trace.ts b/packages/compiler/src/backend/link-trace.ts new file mode 100644 index 00000000..fc576f5c --- /dev/null +++ b/packages/compiler/src/backend/link-trace.ts @@ -0,0 +1,22 @@ +export function linkTraceCandidate(line: string): string[] { + const trimmed = line.trim().replace(/^(?:LOAD|load)\s+/, ""); + if (trimmed === "") return []; + const unquoted = + (trimmed.startsWith('"') && trimmed.endsWith('"')) || + (trimmed.startsWith("'") && trimmed.endsWith("'")) + ? trimmed.slice(1, -1) + : trimmed; + const candidates = [unquoted]; + const member = unquoted.lastIndexOf("("); + if (member > 0 && unquoted.endsWith(")")) candidates.push(unquoted.slice(0, member)); + return candidates; +} + +export function driverTraceCandidates(line: string): string[] { + const candidates: string[] = []; + for (const match of line.matchAll(/"((?:\\.|[^"\\])*)"|'([^']*)'|(\S+)/g)) { + const token = (match[1] ?? match[2] ?? match[3] ?? "").replace(/\\(["\\])/g, "$1"); + if (token !== "") candidates.push(token); + } + return candidates; +} diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index b7c2c011..ecb16353 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -576,7 +576,7 @@ export class LlEmitter { `}`, ``, ); - defs.push(...dispatchBody); + for (const line of dispatchBody) defs.push(line); this.declare(`declare ptr @scr_ffi_call_new(ptr, ptr, ptr, ${this.sizeType})`); this.declare(`declare void @scr_ffi_post(ptr)`); @@ -1027,7 +1027,7 @@ export class LlEmitter { // finish_top_level initially notes 13. Replace that hint before exit // listeners run when a higher-priority verdict was already selected. lines.push(` call void @scr_exit_code_note(i32 ${exitStatus})`); - lines.push(...exitListenerLines("xp")); + for (const line of exitListenerLines("xp")) lines.push(line); if (tracksIslandExit) { lines.push( ` %tla_exit_version_after = call ${this.sizeType} @scr_island_exit_code_version()`, @@ -1043,7 +1043,7 @@ export class LlEmitter { ` %tla_final_exit = phi i32 [ %tla_listener_exit, %tla_exit_updated ], [ ${exitStatus}, %tla_exit_unchanged ]`, ); } - lines.push(...topPendingReleases); + for (const line of topPendingReleases) lines.push(line); lines.push(` ret i32 ${tracksIslandExit ? "%tla_final_exit" : exitStatus}`); return lines; }; @@ -1147,8 +1147,8 @@ export class LlEmitter { `%ScrIslandModule = type { ptr, ptr, ${this.sizeType}, ${this.sizeType}, i32, ptr, ${this.sizeType}, ${this.sizeType} }`, `%ScrIslandEdge = type { ptr, ptr, ptr, i32 }`, ]; - out.push(...shapes.typeDefs); - out.push(...classShapes.typeDefs); + for (const line of shapes.typeDefs) out.push(line); + for (const line of classShapes.typeDefs) out.push(line); // Thread-instanced library state (abi.instance_per_thread): the // program TU's mutable globals — module globals, run-once guards, the // lazily-compiled regex literal caches — and the runtime globals its @@ -1293,7 +1293,7 @@ export class LlEmitter { } out.push(``); } - out.push(...ffiCallbacks.globals); + for (const line of ffiCallbacks.globals) out.push(line); if (ffiCallbacks.globals.length > 0) out.push(``); for (const g of globals) { const ty = this.llType(g.type); @@ -1302,16 +1302,16 @@ export class LlEmitter { out.push(`@${mangleGlobal(g.id)} = internal ${tl}global ${ty} ${zero}${debug ? `, !dbg ${debug}` : ""} ; ${g.name}`); } if (globals.length > 0) out.push(``); - out.push(...helpers); - out.push(...ffiCallbacks.defs); - out.push(...shapes.defs); - out.push(...classShapes.defs); - out.push(...classObjDefs); - out.push(...this.walkers.defs); - out.push(...this.dyn.defs); - out.push(...wrappers); - out.push(...asyncDefs); - out.push(...this.resolveThunkDefs); + for (const line of helpers) out.push(line); + for (const line of ffiCallbacks.defs) out.push(line); + for (const line of shapes.defs) out.push(line); + for (const line of classShapes.defs) out.push(line); + for (const line of classObjDefs) out.push(line); + for (const line of this.walkers.defs) out.push(line); + for (const line of this.dyn.defs) out.push(line); + for (const line of wrappers) out.push(line); + for (const line of asyncDefs) out.push(line); + for (const line of this.resolveThunkDefs) out.push(line); out.push(fnDefs.join("\n\n"), ``); // main(): scr_init, the program-dependent error-vt interval stamps, @@ -1367,7 +1367,7 @@ export class LlEmitter { if (this.mod.lib !== undefined) { // LIBRARY mode: no @main — the profile-declared external // symbols specified by the library IR instead. - out.push(...this.emitLibDefs(globals, globalReleaseLines, stamps)); + for (const line of this.emitLibDefs(globals, globalReleaseLines, stamps)) out.push(line); out.push(`attributes #0 = { sanitize_address }`); if (this.wasi) out.push(`attributes #1 = { sanitize_address presplitcoroutine }`); if (hasNoInlineRecordClone) out.push(`attributes #2 = { noinline sanitize_address }`); @@ -1661,7 +1661,7 @@ export class LlEmitter { // from the poisoned guard and every runtime touch (ratified), so a // host can read them before init and after a trap. The u64 rides // i64 two's-complement (LLVM integer constants are signed). - out.push(...emitLibraryIdentityLines(lib.identity, FN_ATTRS)); + for (const line of emitLibraryIdentityLines(lib.identity, FN_ATTRS)) out.push(line); } if (lib.resultResetSymbol !== null) { out.push( @@ -2006,7 +2006,7 @@ export class LlEmitter { if (fn.async !== true || fn.generator !== undefined) continue; const { definitions, ret, tr, spawnParams, argPackLines } = this.emitArgPackAndTrampolinePrologue(fn); - out.push(...definitions); + for (const line of definitions) out.push(line); this.declare(`declare ptr @scr_fiber_promise(ptr)`); this.declare(`declare ptr @scr_async_spawn(ptr, ptr)`); this.needOom(); @@ -2058,7 +2058,7 @@ export class LlEmitter { } tr.push(` ret void`, `}`, ``); } - out.push(...tr); + for (const line of tr) out.push(line); // Spawn wrapper: pack the args (+1 moves in), spawn the fiber. const cache = fn.asyncCacheGlobal !== undefined ? mangleGlobal(fn.asyncCacheGlobal) : null; @@ -2128,9 +2128,9 @@ export class LlEmitter { `}`, ``, ); - out.push(...sp); + for (const line of sp) out.push(line); } - out.push(...this.emitGenScaffolding()); + for (const line of this.emitGenScaffolding()) out.push(line); return out; } @@ -2154,7 +2154,7 @@ export class LlEmitter { spawnParams, argPackLines, } = this.emitArgPackAndTrampolinePrologue(fn); - out.push(...definitions); + for (const line of definitions) out.push(line); this.declare(`declare zeroext i1 @scr_exc_genret_pending()`); this.declare(`declare void @scr_exc_clear()`); this.declare(`declare ptr @scr_gen_of_fiber(ptr)`); @@ -2211,7 +2211,7 @@ export class LlEmitter { } tr.push(` br label %done`, `done:`, ` ret void`, `}`, ``); } - out.push(...tr); + for (const line of tr) out.push(line); let settleAsync: string | null = null; if (fn.async) { @@ -2264,7 +2264,7 @@ export class LlEmitter { } }); dr.push(` call void @free(ptr %ap)`, ` ret void`, `}`, ``); - out.push(...dr); + for (const line of dr) out.push(line); // Spawn wrapper: pack the args (+1 moves in), allocate the // SUSPENDED fiber — nothing runs until the first .next(). @@ -2284,7 +2284,7 @@ export class LlEmitter { `}`, ``, ); - out.push(...sp); + for (const line of sp) out.push(line); } return out; } diff --git a/packages/compiler/src/backend/llvm/expr-async.ts b/packages/compiler/src/backend/llvm/expr-async.ts index 70eedb34..47d0cf2f 100644 --- a/packages/compiler/src/backend/llvm/expr-async.ts +++ b/packages/compiler/src/backend/llvm/expr-async.ts @@ -235,7 +235,7 @@ export function emitSerializationExpr(host: LlvmEmitterContext, e: ExprOf<"jsonS // clamp/truncate rules); the interned re-indenter rewrites the // compact text with Node's gap algorithm. Compact temp stays // frame-owned; the pretty string is a fresh +1. - const indent = (e as { indent?: string }).indent; + const indent = e.indent; if (indent === undefined || indent === "") return compact; const rewriter = host.walkers.jsonIndentHelper(); const t2 = B.tmp(); diff --git a/packages/compiler/src/backend/native-link-info-core.ts b/packages/compiler/src/backend/native-link-info-core.ts new file mode 100644 index 00000000..83629213 --- /dev/null +++ b/packages/compiler/src/backend/native-link-info-core.ts @@ -0,0 +1,155 @@ +import { join } from "node:path"; +import type { FfiProfile } from "../ffi/ffi-manifest.js"; +import { EXTERNAL_OBJECT_ABI_STABILITY, RUNTIME_ABI_MARKER, RUNTIME_ABI_VERSION } from "./runtime-abi.js"; +import { executableLinkInputs } from "./link-plan-core.js"; +import type { RuntimePackArtifact, RuntimePackArtifacts, RuntimePackManifest } from "./runtime-pack-core.js"; +import type { NativeTargetSpec } from "./targets.js"; + +export interface NativeLinkFeatures { + dynamic: boolean; + regex: boolean; + copying: boolean; + textDecoderLegacy: boolean; + fileHandle: boolean; + fetch: boolean; + netIsland: boolean; + zlib: boolean; + assert: boolean; + inspect: boolean; + dynInvoke: boolean; + dc: boolean; + dynAsync: boolean; + events: boolean; + emitter: boolean; + symbol: boolean; + bigint: boolean; + searchParams: boolean; + qs: boolean; + parseArgs: boolean; + stream: boolean; + net: boolean; + http: boolean; + http2: boolean; + dgram: boolean; + watch: boolean; + foreignFfi: boolean; + nodeTest: boolean; + tls: boolean; + tlsCa: boolean; +} + +export interface NativeLinkInfo { + schema: "scriptc.native-link-info.v1"; + format: 1; + compiler_version: string; + object_abi: { + stability: "experimental"; + compatibility: "exact-runtime-version"; + }; + target: { + name: NativeTargetSpec["name"]; + llvm_triple: NativeTargetSpec["llvmTriple"]; + architecture: NativeTargetSpec["architecture"]; + object_format: NativeTargetSpec["objectFormat"]; + minimum_os: NativeTargetSpec["minimumOs"]; + relocation_model: NativeTargetSpec["relocationModel"]; + }; + program: { + object: string; + entry_symbol: "main"; + undefined_runtime_symbol_prefix: "scr_"; + }; + runtime_abi: { + version: typeof RUNTIME_ABI_VERSION; + marker: typeof RUNTIME_ABI_MARKER; + }; + runtime_pack: { + kind: "precompiled"; + package: string; + version: string; + root: string; + path_base: "runtime_pack.root"; + flavor: "release" | "dev"; + objects: RuntimePackArtifact[]; + archives: RuntimePackArtifact[]; + }; + ffi: { + format: number | null; + symbols: string[]; + libraries: string[]; + }; + link: { + input_order: string[]; + driver_flags: string[]; + system_libraries: string[]; + frameworks: string[]; + }; +} + +/** Format the same external-link contract after either host verifies a pack. */ +export function formatNativeLinkInfo(options: { + programObject: string; + target: NativeTargetSpec; + ffi: FfiProfile | null; +}, compilerVersion: string, pack: { + root: string; + manifest: RuntimePackManifest; + selected: RuntimePackArtifacts; + flavor: "release" | "dev"; +}): NativeLinkInfo { + const ffiLibraries = options.ffi?.libraries ?? []; + const plan = executableLinkInputs({ + target: options.target, programObject: options.programObject, + ffiLibraries, ffiSystemLibraries: options.ffi?.systemLibraries ?? [], + ffiFrameworks: options.ffi?.frameworks ?? [], runtimeObjects: pack.selected.runtime.map((artifact) => join(pack.root, artifact.path)), + runtimeArchives: pack.selected.archives.map((artifact) => join(pack.root, artifact.path)), runtimeSystemLibraries: pack.selected.systemLibraries, + optimization: pack.flavor, + }); + return { + schema: "scriptc.native-link-info.v1", + format: 1, + compiler_version: compilerVersion, + object_abi: { + stability: EXTERNAL_OBJECT_ABI_STABILITY, + compatibility: "exact-runtime-version", + }, + target: { + name: options.target.name, + llvm_triple: options.target.llvmTriple, + architecture: options.target.architecture, + object_format: options.target.objectFormat, + minimum_os: options.target.minimumOs, + relocation_model: options.target.relocationModel, + }, + program: { + object: options.programObject, + entry_symbol: "main", + undefined_runtime_symbol_prefix: "scr_", + }, + runtime_abi: { + version: RUNTIME_ABI_VERSION, + marker: RUNTIME_ABI_MARKER, + }, + runtime_pack: { + kind: "precompiled", + package: pack.manifest.package, + version: pack.manifest.version, + root: pack.root, + path_base: "runtime_pack.root", + flavor: pack.flavor, + objects: pack.selected.runtime, + archives: pack.selected.archives, + }, + ffi: { + format: options.ffi?.ffiFormat ?? null, + symbols: options.ffi?.functions.filter((fn) => !fn.callbackOperation).map((fn) => fn.symbol) ?? [], + libraries: [...ffiLibraries], + }, + link: { + input_order: plan.inputs, + driver_flags: plan.driverFlags, + system_libraries: plan.systemLibraries, + frameworks: [...(options.ffi?.frameworks ?? [])], + }, + }; +} diff --git a/packages/compiler/src/backend/native-link-info.ts b/packages/compiler/src/backend/native-link-info.ts index 9bb52201..45f28877 100644 --- a/packages/compiler/src/backend/native-link-info.ts +++ b/packages/compiler/src/backend/native-link-info.ts @@ -1,92 +1,10 @@ +import { formatNativeLinkInfo, type NativeLinkInfo, type NativeLinkFeatures } from "./native-link-info-core.js"; +export type { NativeLinkInfo, NativeLinkFeatures } from "./native-link-info-core.js"; import type { FfiProfile } from "../ffi/ffi-manifest.js"; import { compilerReleaseVersion } from "../library/sidecar.js"; -import { EXTERNAL_OBJECT_ABI_STABILITY, RUNTIME_ABI_MARKER, RUNTIME_ABI_VERSION } from "./runtime-abi.js"; -import { executableLinkInputs } from "./link-plan-core.js"; import { loadRuntimePack } from "./runtime-pack.js"; -import type { RuntimePackArtifact } from "./runtime-pack-core.js"; import type { NativeTargetSpec } from "./targets.js"; -export interface NativeLinkFeatures { - dynamic: boolean; - regex: boolean; - copying: boolean; - textDecoderLegacy: boolean; - fileHandle: boolean; - fetch: boolean; - netIsland: boolean; - zlib: boolean; - assert: boolean; - inspect: boolean; - dynInvoke: boolean; - dc: boolean; - dynAsync: boolean; - events: boolean; - emitter: boolean; - symbol: boolean; - bigint: boolean; - searchParams: boolean; - qs: boolean; - parseArgs: boolean; - stream: boolean; - net: boolean; - http: boolean; - http2: boolean; - dgram: boolean; - watch: boolean; - foreignFfi: boolean; - nodeTest: boolean; - tls: boolean; - tlsCa: boolean; -} - -export interface NativeLinkInfo { - schema: "scriptc.native-link-info.v1"; - format: 1; - compiler_version: string; - object_abi: { - stability: "experimental"; - compatibility: "exact-runtime-version"; - }; - target: { - name: NativeTargetSpec["name"]; - llvm_triple: NativeTargetSpec["llvmTriple"]; - architecture: NativeTargetSpec["architecture"]; - object_format: NativeTargetSpec["objectFormat"]; - minimum_os: NativeTargetSpec["minimumOs"]; - relocation_model: NativeTargetSpec["relocationModel"]; - }; - program: { - object: string; - entry_symbol: "main"; - undefined_runtime_symbol_prefix: "scr_"; - }; - runtime_abi: { - version: typeof RUNTIME_ABI_VERSION; - marker: typeof RUNTIME_ABI_MARKER; - }; - runtime_pack: { - kind: "precompiled"; - package: string; - version: string; - root: string; - path_base: "runtime_pack.root"; - flavor: "release" | "dev"; - objects: RuntimePackArtifact[]; - archives: RuntimePackArtifact[]; - }; - ffi: { - format: number | null; - symbols: string[]; - libraries: string[]; - }; - link: { - input_order: string[]; - driver_flags: string[]; - system_libraries: string[]; - frameworks: string[]; - }; -} - export async function createNativeLinkInfo(options: { programObject: string; target: NativeTargetSpec; @@ -100,59 +18,8 @@ export async function createNativeLinkInfo(options: { target: options.target, features: options.features, optimization: options.optimization ?? "release", ...(options.env === undefined ? {} : { env: options.env }), }); - const ffiLibraries = options.ffi?.libraries ?? []; - const plan = executableLinkInputs({ - target: options.target, programObject: options.programObject, - ffiLibraries, ffiSystemLibraries: options.ffi?.systemLibraries ?? [], - ffiFrameworks: options.ffi?.frameworks ?? [], runtimeObjects: pack.runtimeObjects, - runtimeArchives: pack.archives, runtimeSystemLibraries: pack.systemLibraries, - optimization: pack.flavor, + return formatNativeLinkInfo(options, compilerReleaseVersion(), { + root: pack.root, manifest: pack.manifest, flavor: pack.flavor, + selected: { features: pack.features, runtime: pack.selectedRuntimeArtifacts, archives: pack.selectedArchiveArtifacts, systemLibraries: pack.systemLibraries }, }); - return { - schema: "scriptc.native-link-info.v1", - format: 1, - compiler_version: compilerReleaseVersion(), - object_abi: { - stability: EXTERNAL_OBJECT_ABI_STABILITY, - compatibility: "exact-runtime-version", - }, - target: { - name: options.target.name, - llvm_triple: options.target.llvmTriple, - architecture: options.target.architecture, - object_format: options.target.objectFormat, - minimum_os: options.target.minimumOs, - relocation_model: options.target.relocationModel, - }, - program: { - object: options.programObject, - entry_symbol: "main", - undefined_runtime_symbol_prefix: "scr_", - }, - runtime_abi: { - version: RUNTIME_ABI_VERSION, - marker: RUNTIME_ABI_MARKER, - }, - runtime_pack: { - kind: "precompiled", - package: pack.manifest.package, - version: pack.manifest.version, - root: pack.root, - path_base: "runtime_pack.root", - flavor: pack.flavor, - objects: pack.selectedRuntimeArtifacts, - archives: pack.selectedArchiveArtifacts, - }, - ffi: { - format: options.ffi?.ffiFormat ?? null, - symbols: options.ffi?.functions.filter((fn) => !fn.callbackOperation).map((fn) => fn.symbol) ?? [], - libraries: [...ffiLibraries], - }, - link: { - input_order: plan.inputs, - driver_flags: plan.driverFlags, - system_libraries: plan.systemLibraries, - frameworks: [...(options.ffi?.frameworks ?? [])], - }, - }; } diff --git a/packages/compiler/src/backend/native-toolchain.ts b/packages/compiler/src/backend/native-toolchain.ts index d8d35f65..61587ee0 100644 --- a/packages/compiler/src/backend/native-toolchain.ts +++ b/packages/compiler/src/backend/native-toolchain.ts @@ -1,3 +1,8 @@ +import { driverTraceCandidates, linkTraceCandidate } from "./link-trace.js"; +import { toolchainEnvironmentCachePolicy, toolchainEnvironmentFingerprint } from "./toolchain-environment.js"; +export { toolchainEnvironmentCachePolicy, toolchainEnvironmentFingerprint, type ToolchainEnvironmentCachePolicy } from "./toolchain-environment.js"; +import { IPHONEOS_MIN_VERSION, ANDROID_MIN_API, isIosTarget, isAndroidTarget, isMobileTarget, mobileLibraryTarget, mobileTargetRefusal, configuredTargetPlatform } from "./target-platform.js"; +export { IPHONEOS_MIN_VERSION, ANDROID_MIN_API, isIosTarget, isAndroidTarget, isMobileTarget, mobileLibraryTarget, mobileTargetRefusal, configuredTargetPlatform } from "./target-platform.js"; import { InternalCompilerError } from "../errors.js"; import { execFile, spawnSync } from "node:child_process"; import { createHash, randomUUID } from "node:crypto"; @@ -123,114 +128,6 @@ export function executableSectionEliminationFlags(platform: string): { } } -/** Environment variables consumed by clang, its linker/subtools, or the - * platform SDK selection. They are implicit command-line inputs: changing one - * must never reuse an artifact produced under the old toolchain posture. */ -const TOOLCHAIN_ENV_KEYS = [ - "COMPILER_PATH", - "GCC_EXEC_PREFIX", - "CPATH", - "C_INCLUDE_PATH", - "CPLUS_INCLUDE_PATH", - "OBJC_INCLUDE_PATH", - "OBJCPLUS_INCLUDE_PATH", - "LIBRARY_PATH", - "LD_LIBRARY_PATH", - "LD_RUN_PATH", - "DYLD_LIBRARY_PATH", - "DYLD_FRAMEWORK_PATH", - "DYLD_FALLBACK_LIBRARY_PATH", - "DYLD_FALLBACK_FRAMEWORK_PATH", - "SDKROOT", - "DEVELOPER_DIR", - "MACOSX_DEPLOYMENT_TARGET", - "IPHONEOS_DEPLOYMENT_TARGET", - "TVOS_DEPLOYMENT_TARGET", - "WATCHOS_DEPLOYMENT_TARGET", - "DRIVERKIT_DEPLOYMENT_TARGET", - "XROS_DEPLOYMENT_TARGET", - "CCC_OVERRIDE_OPTIONS", - "CCC_ADD_ARGS", - "CLANG_CONFIG_FILE_SYSTEM_DIR", - "CLANG_CONFIG_FILE_USER_DIR", - "CC", - "CFLAGS", - "CPPFLAGS", - "LDFLAGS", - "AR", - "RANLIB", - "CMAKE_GENERATOR", - "CMAKE_TOOLCHAIN_FILE", - "ZIG_LIB_DIR", - "ZIG_LIBC", - "SOURCE_DATE_EPOCH", - "ZERO_AR_DATE", - "LANG", - "LC_ALL", - "LC_CTYPE", -] as const; - -/** Toolchain variables whose values name mutable files/directories consumed - * while compiling a TU (or can inject arbitrary compiler options). Hashing the - * value is insufficient: a header, SDK, config, compiler helper, or loaded - * dylib can change in place while the spelling remains stable. In that posture - * neither complete artifacts nor per-TU runtime objects are safe to reuse. */ -const MUTABLE_COMPILE_ENV_KEYS = [ - "COMPILER_PATH", - "GCC_EXEC_PREFIX", - "CPATH", - "C_INCLUDE_PATH", - "CPLUS_INCLUDE_PATH", - "OBJC_INCLUDE_PATH", - "OBJCPLUS_INCLUDE_PATH", - "LD_LIBRARY_PATH", - "DYLD_LIBRARY_PATH", - "DYLD_FRAMEWORK_PATH", - "DYLD_FALLBACK_LIBRARY_PATH", - "DYLD_FALLBACK_FRAMEWORK_PATH", - "SDKROOT", - "DEVELOPER_DIR", - "CCC_OVERRIDE_OPTIONS", - "CCC_ADD_ARGS", - "CLANG_CONFIG_FILE_SYSTEM_DIR", - "CLANG_CONFIG_FILE_USER_DIR", - // `zig cc` resolves its bundled headers/runtime through ZIG_LIB_DIR and a - // caller-selected native libc description through ZIG_LIBC. Both values name - // mutable compiler inputs whose contents can change behind a stable path. - "ZIG_LIB_DIR", - "ZIG_LIBC", -] as const; - -/** These variables only redirect link-time inputs. Runtime objects remain - * reusable, but a complete executable could otherwise retain a library that - * was rebuilt in place behind the same search-path spelling. */ -const MUTABLE_LINK_ENV_KEYS = ["LIBRARY_PATH", "LD_RUN_PATH"] as const; - -export interface ToolchainEnvironmentCachePolicy { - completeArtifacts: boolean; - runtimeObjects: boolean; -} - -export function toolchainEnvironmentCachePolicy( - env: NodeJS.ProcessEnv = process.env, -): ToolchainEnvironmentCachePolicy { - const mutableCompileInput = MUTABLE_COMPILE_ENV_KEYS.some((name) => env[name] !== undefined); - const mutableLinkInput = MUTABLE_LINK_ENV_KEYS.some((name) => env[name] !== undefined); - return { - completeArtifacts: !mutableCompileInput && !mutableLinkInput, - runtimeObjects: !mutableCompileInput, - }; -} - -export function toolchainEnvironmentFingerprint(env: NodeJS.ProcessEnv = process.env): string { - const hash = createHash("sha256").update("toolchain-env-v1\0"); - for (const name of TOOLCHAIN_ENV_KEYS) { - const value = env[name]; - hash.update(name).update(value === undefined ? "\0unset\0" : "\0set\0").update(value ?? "").update("\0"); - } - return hash.digest("hex"); -} - /** Inputs that can change which native tool/runtime implementation an * executable build selects before compileC has a chance to rediscover it. * The early whole-program cache keys this exact posture before restoring a @@ -645,59 +542,6 @@ export function isZigDriver(driver: Pick): boolean { * embedder's side of the contract: Xcode links iOS archives against the * selected SDK, and Gradle/NDK builds link Android archives against the * API-26+ bionic stubs. */ -export const IPHONEOS_MIN_VERSION = "15.0"; -export const ANDROID_MIN_API = 26; - -const MOBILE_LIBRARY_TARGETS = [ - "aarch64-apple-ios", - "aarch64-apple-ios-simulator", - "aarch64-linux-android", -] as const; - -export function isIosTarget(target: string | null): boolean { - return target === "aarch64-apple-ios" || target === "aarch64-apple-ios-simulator"; -} - -export function isAndroidTarget(target: string | null): boolean { - return target === "aarch64-linux-android"; -} - -export function isMobileTarget(target: string | null): boolean { - return isIosTarget(target) || isAndroidTarget(target); -} - -/** The canonical mobile triple SCRIPTC_TARGET selects, or null when the - * environment names none. Pure string inspection — safe to consult before - * any toolchain discovery runs. */ -export function mobileLibraryTarget(env: NodeJS.ProcessEnv = process.env): string | null { - const target = env["SCRIPTC_TARGET"] ?? ""; - return isMobileTarget(target) ? target : null; -} - -/** The admission verdict for a mobile-family triple: null when the spelling - * and host pairing are supported, otherwise the refusal text (the same text - * resolveCc throws and compileLibrary reports as SC3002). Pure string/host - * inspection — no discovery, no subprocess. */ -export function mobileTargetRefusal( - target: string, - hostPlatform: NodeJS.Platform = process.platform, -): string | null { - if (isIosTarget(target)) { - return hostPlatform === "darwin" - ? null - : `${target} library archives build on macOS hosts only (the Apple iOS SDK sysroot and Mach-O symbol localization live there); this host is ${hostPlatform}`; - } - if (isAndroidTarget(target)) return null; - // A near-miss mobile spelling must refuse with the supported set named, - // never reach zig with no sysroot wired (the compile would fail on the - // first libc header) or produce an artifact for an unverified device - // class. - if (/(?:^|-)(?:ios|tvos|watchos|visionos|android)/.test(target)) { - return `unsupported mobile target '${target}' (supported: ${MOBILE_LIBRARY_TARGETS.join(", ")})`; - } - return null; -} - /** The Apple SDK root for one mobile platform, discovered through xcrun the * way Xcode's own build system selects it. Memoized per SDK name and * selection environment: production rediscovers per process, and the two @@ -906,34 +750,6 @@ function isMuslTarget(driver: Pick): boolean { * analyze(): the FRONTEND consults it too (path.sep / os.EOL literals and * the path-module binding follow the target — a win32 triple compiles * Node-on-Windows semantics, path.win32 backing the bare module). */ -export function configuredTargetPlatform( - env: NodeJS.ProcessEnv = process.env, - hostPlatform: NodeJS.Platform = process.platform, -): string { - const target = env["SCRIPTC_TARGET"] ?? ""; - if (target === "") return hostPlatform; - if (target === "wasm32-wasi") return "wasi"; - if (target.includes("wasi")) { - throw new Error(`unsupported WASI target '${target}' (supported: wasm32-wasi)`); - } - // iOS is a darwin-family target: Mach-O objects, ld64 localization, - // POSIX path/EOL semantics. Android falls to the linux arm below — - // bionic is a linux libc and its archives are ordinary ELF. - if (isIosTarget(target)) return "darwin"; - if (isAndroidTarget(target)) return "linux"; - if (/(?:^|-)(?:ios|tvos|watchos|visionos|android)/.test(target)) { - throw new Error( - `unsupported mobile target '${target}' (supported: ${MOBILE_LIBRARY_TARGETS.join(", ")})`, - ); - } - if (target.includes("linux")) return "linux"; - if (target.includes("windows")) return "win32"; - if (target.includes("macos") || target.includes("darwin")) return "darwin"; - throw new Error( - `unsupported target '${target}' (supported OS families: linux, windows, macos/darwin, wasm32-wasi)`, - ); -} - export function targetPlatform(driver: CcDriver): string { if (driver.target === null) return process.platform; return configuredTargetPlatform({ SCRIPTC_TARGET: driver.target }); @@ -2980,29 +2796,6 @@ function implicitToolchainFingerprint( ); } -function linkTraceCandidate(line: string): string[] { - const trimmed = line.trim().replace(/^(?:LOAD|load)\s+/, ""); - if (trimmed === "") return []; - const unquoted = - (trimmed.startsWith('"') && trimmed.endsWith('"')) || - (trimmed.startsWith("'") && trimmed.endsWith("'")) - ? trimmed.slice(1, -1) - : trimmed; - const candidates = [unquoted]; - const member = unquoted.lastIndexOf("("); - if (member > 0 && unquoted.endsWith(")")) candidates.push(unquoted.slice(0, member)); - return candidates; -} - -function driverTraceCandidates(line: string): string[] { - const candidates: string[] = []; - for (const match of line.matchAll(/"((?:\\.|[^"\\])*)"|'([^']*)'|(\S+)/g)) { - const token = (match[1] ?? match[2] ?? match[3] ?? "").replace(/\\(["\\])/g, "$1"); - if (token !== "") candidates.push(token); - } - return candidates; -} - async function existingDriverTracePaths( output: string, cwd: string, diff --git a/packages/compiler/src/backend/native-tools.ts b/packages/compiler/src/backend/native-tools.ts index ff30e97b..ab9e693a 100644 --- a/packages/compiler/src/backend/native-tools.ts +++ b/packages/compiler/src/backend/native-tools.ts @@ -1,16 +1,12 @@ /** Native tool invocation without a JavaScript host or shell command construction. */ -import { spawnSync } from "node:child_process"; +import { execFileSync } from "node:child_process"; import { readFileSync, statSync } from "node:fs"; import type { NativeHelperSpec, NativeTargetSpec } from "./targets.js"; -import { validateNativeCodegenVersion } from "./native-codegen-core.js"; +import { validateNativeCodegenVersion, type NativeCodegenOutputKind } from "./native-codegen-core.js"; -export function runNativeTool(executable: string, args: string[]): string { - const result = spawnSync(executable, args, { encoding: "utf8" }); - if (result.error) throw new Error(`${executable}: ${result.error.message}`); - if (result.status !== 0) { - throw new Error(`${executable} failed (${result.signal ?? String(result.status)}): ${result.stderr.trim()}`); - } - return result.stdout; +export function runNativeTool(executable: string, args: string[], cwd?: string, env?: NodeJS.ProcessEnv): string { + return execFileSync(executable, args, { encoding: "utf8", stdio: "pipe", maxBuffer: 64 * 1024 * 1024, + cwd: cwd ?? process.cwd(), env: env ?? process.env }); } export function requireNativeArtifact(path: string): void { @@ -28,6 +24,22 @@ export function emitNativeObject(options: { outputPath: string; sourcePath: string; optimization: "release" | "dev"; + outputKind?: NativeCodegenOutputKind; +}): void { + verifyNativeHelper(options); + runNativeTool(options.executable, [ + "emit", "--input", options.inputPath, "--output", options.outputPath, "--filetype", options.outputKind ?? "obj", + "--target", options.target.llvmTriple, "--opt-level", options.optimization === "dev" ? "0" : "2", + "--relocation-model", options.target.relocationModel, "--diagnostic-format", "json", + "--source-path", options.sourcePath, + ]); + requireNativeArtifact(options.outputPath); +} + +/** Verify installed identity even when a caller can reuse a cached object. */ +export function verifyNativeHelper(options: { + executable: string; packageRoot: string; compilerVersion: string; + target: NativeTargetSpec; helper: NativeHelperSpec; }): void { const identity = JSON.parse(readFileSync(options.packageRoot + "/package.json", "utf8")) as { name: string; version: string }; if (identity.name !== options.helper.packageName || identity.version !== options.compilerVersion) { @@ -35,11 +47,4 @@ export function emitNativeObject(options: { } const version = JSON.parse(runNativeTool(options.executable, ["version", "--format=json"])) as Record; validateNativeCodegenVersion(version, options.target, options.helper, options.compilerVersion); - runNativeTool(options.executable, [ - "emit", "--input", options.inputPath, "--output", options.outputPath, "--filetype", "obj", - "--target", options.target.llvmTriple, "--opt-level", options.optimization === "dev" ? "0" : "2", - "--relocation-model", options.target.relocationModel, "--diagnostic-format", "json", - "--source-path", options.sourcePath, - ]); - requireNativeArtifact(options.outputPath); } diff --git a/packages/compiler/src/backend/object-localize.ts b/packages/compiler/src/backend/object-localize.ts index 5b79448d..e1f50a69 100644 --- a/packages/compiler/src/backend/object-localize.ts +++ b/packages/compiler/src/backend/object-localize.ts @@ -199,7 +199,10 @@ export function localizeElfObject(object: Uint8Array, keep: ReadonlySet) for (let i = 0; i < sections.length; i++) sectionMap.push(sectionKept[i] === true ? next++ : -1); } for (let i = 0; i < sections.length; i++) { - if (sectionKept[i] === true) sections[i]!.flags &= ~SHF_GROUP; + if (sectionKept[i] === true) { + const section = sections[i]!; + section.flags = section.flags & ~SHF_GROUP; + } } // Demote and drop decisions per symbol. A symbol anchored to a dropped @@ -228,7 +231,9 @@ export function localizeElfObject(object: Uint8Array, keep: ReadonlySet) } const symbolOrder = [...localOrder, ...globalOrder]; const symbolMap: number[] = new Array(symCount).fill(-1); - symbolOrder.forEach((oldIndex, newIndex) => (symbolMap[oldIndex] = newIndex)); + symbolOrder.forEach((oldIndex, newIndex) => { + symbolMap[oldIndex] = newIndex; + }); const newSymtabData = new Uint8Array(symbolOrder.length * 24); const newSymtabView = new DataView(newSymtabData.buffer); @@ -412,7 +417,8 @@ function parseCoff(object: Uint8Array, label: string): CoffObject { const sectionName = (raw: Uint8Array): string => { if (raw[0] === 0x2f /* '/' */) { const spelled = textDecoder.decode(raw.subarray(1)).replace(/\0+$/, "").trim(); - const offset = Number.parseInt(spelled, 10); + const decimal = /^\d+/.exec(spelled); + const offset = decimal === null ? NaN : Number(decimal[0]); if (!Number.isFinite(offset)) fail(`${label}: malformed long section name`); return readCString(strtab, offset); } @@ -563,7 +569,7 @@ export function mergeAndLocalizeCoffObjects( ]; for (const object of objects) { if (object.machine !== IMAGE_FILE_MACHINE_AMD64) { - fail(`${object.label}: unsupported COFF machine 0x${object.machine.toString(16)}`); + fail(`${object.label}: unsupported COFF machine ${object.machine}`); } } @@ -601,7 +607,7 @@ export function mergeAndLocalizeCoffObjects( for (const sym of object.symbols) { if (!coffIsUndefined(sym)) continue; if (selectedDefinitions.has(sym.name)) continue; - for (const candidate of definers.get(sym.name) ?? []) { + for (const candidate of definers.get(sym.name) ?? new Array()) { if (included[candidate] !== true) { included[candidate] = true; addDefinitions(objects[candidate]!); diff --git a/packages/compiler/src/backend/runtime-pack-native.ts b/packages/compiler/src/backend/runtime-pack-native.ts index c6b223e7..2fc94730 100644 --- a/packages/compiler/src/backend/runtime-pack-native.ts +++ b/packages/compiler/src/backend/runtime-pack-native.ts @@ -8,7 +8,7 @@ import type { NativeLinkFeatures } from "./native-link-info.js"; import type { NativeTargetSpec } from "./targets.js"; import { RuntimePackError, parseRuntimePackManifest, selectRuntimePackArtifacts, validateRuntimePackIdentity, - type RuntimePackArtifact, + type RuntimePackArtifact, type RuntimePackArtifacts, type RuntimePackManifest, type RuntimePackMode, } from "./runtime-pack-core.js"; export interface NativeRuntimePack { @@ -17,6 +17,28 @@ export interface NativeRuntimePack { systemLibraries: string[]; } +export interface NativeRuntimeSelection { + root: string; + manifest: RuntimePackManifest; + packageText: string; + manifestText: string; + selected: RuntimePackArtifacts; + flavor: "release" | "dev"; +} + +export function selectNativeRuntimePack( + root: string, target: NativeTargetSpec, compilerVersion: string, + features: NativeLinkFeatures, flavor: "release" | "dev", mode: RuntimePackMode = "executable", +): NativeRuntimeSelection { + const packageText = readFileSync(join(root, "package.json"), "utf8"); + const manifestText = readFileSync(join(root, "runtime-pack.json"), "utf8"); + const identity = JSON.parse(packageText) as { name?: string; version?: string }; + const manifest = parseRuntimePackManifest(JSON.parse(manifestText)); + validateRuntimePackIdentity(manifest, identity.name, identity.version, target, compilerVersion); + const selected = selectRuntimePackArtifacts(manifest, features, flavor, process.env, mode); + return { root, manifest, packageText, manifestText, selected, flavor }; +} + function stageArtifact(root: string, stage: string, artifact: RuntimePackArtifact): string { const source = join(root, artifact.path); const destination = join(stage, artifact.path); @@ -48,15 +70,15 @@ export function stageNativeRuntimePack( compilerVersion: string, features: NativeLinkFeatures, flavor: "release" | "dev", + mode: RuntimePackMode = "executable", ): NativeRuntimePack { + return stageNativeRuntimeSelection(selectNativeRuntimePack(root, target, compilerVersion, features, flavor, mode), stageRoot); +} + +export function stageNativeRuntimeSelection(selection: NativeRuntimeSelection, stageRoot: string): NativeRuntimePack { + const { root, manifest, selected, packageText, manifestText } = selection; const packagePath = join(root, "package.json"); const manifestPath = join(root, "runtime-pack.json"); - const packageText = readFileSync(packagePath, "utf8"); - const manifestText = readFileSync(manifestPath, "utf8"); - const identity = JSON.parse(packageText) as { name?: string; version?: string }; - const manifest = parseRuntimePackManifest(JSON.parse(manifestText)); - validateRuntimePackIdentity(manifest, identity.name, identity.version, target, compilerVersion); - const selected = selectRuntimePackArtifacts(manifest, features, flavor); const runtimeObjects = selected.runtime.map((artifact) => stageArtifact(root, stageRoot, artifact)); const archives = selected.archives.map((artifact) => stageArtifact(root, stageRoot, artifact)); for (const license of manifest.licenses) { diff --git a/packages/compiler/src/backend/target-diagnostics.ts b/packages/compiler/src/backend/target-diagnostics.ts new file mode 100644 index 00000000..6095a12e --- /dev/null +++ b/packages/compiler/src/backend/target-diagnostics.ts @@ -0,0 +1,117 @@ +import type { LlvmUnsupportedError } from "./llvm/emitter.js"; +import type { ScrDiagnostic } from "../diagnostics/diagnostic.js"; +import { moduleUsesFetch, moduleEmbedsBuiltin, type IrModule, type SrcLoc } from "../ir/ir.js"; + +/** The LLVM backend's tier refusal as a diagnostic. SC3xxx = backend + * coverage (the program is fine — this backend doesn't compile it yet); + * the parenthesized kind tag is machine-readable for the differential + * harness's histogram. */ +export function llvmRefusalDiag(err: LlvmUnsupportedError, entryPath: string): ScrDiagnostic { + return { + code: "SC3001", + message: err.message, + loc: err.loc ?? { file: entryPath, start: 0, end: 0 }, + }; +} + +/** A valid program surface that the selected execution target cannot host. + * SC3xxx stays the backend/target-coverage family: source semantics are + * valid, but this target deliberately refuses them instead of emitting a + * binary that traps later. */ +export function targetRefusalDiag(target: string, surface: string, loc: SrcLoc): ScrDiagnostic { + return { + code: "SC3002", + message: `${target} target does not support ${surface}`, + loc, + }; +} + +/** APIs that require host capabilities absent from portable WASI Preview 1. + * These are target diagnostics, not backend-tier gaps: the same language IR + * (including async, generators, and the dynamic island) is otherwise valid. + * Keep the fine-grained walk first so diagnostics point at the API use; the + * embedded-module checks are the entry-anchored safety net for island code. */ +export function moduleWasiUnavailableSurface(mod: IrModule): { surface: string; loc: SrcLoc } | null { + const entryLoc: SrcLoc = { file: mod.sourceFile, start: 0, end: 0 }; + const prefixes: readonly (readonly [string, string])[] = [ + ["cp.", "child processes (WASI Preview 1 has no process-spawning API)"], + ["child.", "child processes (WASI Preview 1 has no process-spawning API)"], + ["spawnRes.", "child processes (WASI Preview 1 has no process-spawning API)"], + ["net.", "network sockets (WASI Preview 1 has no socket API)"], + ["http.", "network sockets (WASI Preview 1 has no socket API)"], + ["https.", "network sockets (WASI Preview 1 has no socket API)"], + ["http2.", "network sockets (WASI Preview 1 has no socket API)"], + ["h2.", "network sockets (WASI Preview 1 has no socket API)"], + ["dgram.", "network sockets (WASI Preview 1 has no socket API)"], + ["dns.", "network sockets (WASI Preview 1 has no socket API)"], + ["tls.", "network sockets (WASI Preview 1 has no socket API)"], + ["fetch.", "network-backed fetch (WASI Preview 1 has no socket API)"], + ["fs.watch", "filesystem watching (WASI Preview 1 has no notification API)"], + ["watcher.", "filesystem watching (WASI Preview 1 has no notification API)"], + ]; + const kinds: ReadonlyMap = new Map([ + ["child", "child processes (WASI Preview 1 has no process-spawning API)"], + ["spawnRes", "child processes (WASI Preview 1 has no process-spawning API)"], + ["childStream", "child processes (WASI Preview 1 has no process-spawning API)"], + ["childWriter", "child processes (WASI Preview 1 has no process-spawning API)"], + ["netServer", "network sockets (WASI Preview 1 has no socket API)"], + ["netSocket", "network sockets (WASI Preview 1 has no socket API)"], + ["http2Session", "network sockets (WASI Preview 1 has no socket API)"], + ["http2Stream", "network sockets (WASI Preview 1 has no socket API)"], + ["dgramSocket", "network sockets (WASI Preview 1 has no socket API)"], + ["fsWatcher", "filesystem watching (WASI Preview 1 has no notification API)"], + ["httpReq", "network sockets (WASI Preview 1 has no socket API)"], + ["httpRes", "network sockets (WASI Preview 1 has no socket API)"], + ["httpClientReq", "network sockets (WASI Preview 1 has no socket API)"], + ["secureCtx", "network sockets (WASI Preview 1 has no socket API)"], + ]); + let found: { surface: string; loc: SrcLoc } | null = null; + const visit = (value: unknown, inheritedLoc: SrcLoc): void => { + if (found !== null || value === null || typeof value !== "object") return; + if (Array.isArray(value)) { + for (const item of value) visit(item, inheritedLoc); + return; + } + const node = value as { kind?: unknown; fn?: unknown; loc?: SrcLoc }; + const loc = node.loc ?? inheritedLoc; + if (typeof node.kind === "string") { + const kindSurface = kinds.get(node.kind); + if (kindSurface !== undefined) { + found = { surface: kindSurface, loc }; + return; + } + if (node.kind === "libCall" && typeof node.fn === "string") { + if (node.fn === "process.kill" || node.fn === "process.killNum" || + node.fn === "process.onSignal" || node.fn === "process.offSignal") { + found = { surface: "OS signals (WASI Preview 1 has no signal API)", loc }; + return; + } + if (node.fn === "os.networkInterfaces") { + found = { surface: "network-interface enumeration (WASI Preview 1 has no interface API)", loc }; + return; + } + for (const [prefix, surface] of prefixes) { + if (node.fn.startsWith(prefix)) { + found = { surface, loc }; + return; + } + } + } + } + for (const key of Object.keys(value)) { + visit((value as Record)[key], loc); + } + }; + visit(mod, entryLoc); + if (found !== null) return found; + + if (moduleUsesFetch(mod)) { + return { surface: "network-backed fetch (WASI Preview 1 has no socket API)", loc: entryLoc }; + } + for (const builtin of ["node:http", "node:https", "node:net", "node:tls"]) { + if (moduleEmbedsBuiltin(mod, builtin)) { + return { surface: `${builtin} networking (WASI Preview 1 has no socket API)`, loc: entryLoc }; + } + } + return null; +} diff --git a/packages/compiler/src/backend/target-platform.ts b/packages/compiler/src/backend/target-platform.ts new file mode 100644 index 00000000..84f49eec --- /dev/null +++ b/packages/compiler/src/backend/target-platform.ts @@ -0,0 +1,80 @@ +export const IPHONEOS_MIN_VERSION = "15.0"; +export const ANDROID_MIN_API = 26; + +const MOBILE_LIBRARY_TARGETS = [ + "aarch64-apple-ios", + "aarch64-apple-ios-simulator", + "aarch64-linux-android", +] as const; + +export function isIosTarget(target: string | null): boolean { + return target === "aarch64-apple-ios" || target === "aarch64-apple-ios-simulator"; +} + +export function isAndroidTarget(target: string | null): boolean { + return target === "aarch64-linux-android"; +} + +export function isMobileTarget(target: string | null): boolean { + return isIosTarget(target) || isAndroidTarget(target); +} + +/** The canonical mobile triple SCRIPTC_TARGET selects, or null when the + * environment names none. Pure string inspection — safe to consult before + * any toolchain discovery runs. */ +export function mobileLibraryTarget(env: NodeJS.ProcessEnv = process.env): string | null { + const target = env["SCRIPTC_TARGET"] ?? ""; + return isMobileTarget(target) ? target : null; +} + +/** The admission verdict for a mobile-family triple: null when the spelling + * and host pairing are supported, otherwise the refusal text (the same text + * resolveCc throws and compileLibrary reports as SC3002). Pure string/host + * inspection — no discovery, no subprocess. */ +export function mobileTargetRefusal( + target: string, + hostPlatform: NodeJS.Platform = process.platform, +): string | null { + if (isIosTarget(target)) { + return hostPlatform === "darwin" + ? null + : `${target} library archives build on macOS hosts only (the Apple iOS SDK sysroot and Mach-O symbol localization live there); this host is ${hostPlatform}`; + } + if (isAndroidTarget(target)) return null; + // A near-miss mobile spelling must refuse with the supported set named, + // never reach zig with no sysroot wired (the compile would fail on the + // first libc header) or produce an artifact for an unverified device + // class. + if (/(?:^|-)(?:ios|tvos|watchos|visionos|android)/.test(target)) { + return `unsupported mobile target '${target}' (supported: ${MOBILE_LIBRARY_TARGETS.join(", ")})`; + } + return null; +} + +export function configuredTargetPlatform( + env: NodeJS.ProcessEnv = process.env, + hostPlatform: NodeJS.Platform = process.platform, +): string { + const target = env["SCRIPTC_TARGET"] ?? ""; + if (target === "") return hostPlatform; + if (target === "wasm32-wasi") return "wasi"; + if (target.includes("wasi")) { + throw new Error(`unsupported WASI target '${target}' (supported: wasm32-wasi)`); + } + // iOS is a darwin-family target: Mach-O objects, ld64 localization, + // POSIX path/EOL semantics. Android falls to the linux arm below — + // bionic is a linux libc and its archives are ordinary ELF. + if (isIosTarget(target)) return "darwin"; + if (isAndroidTarget(target)) return "linux"; + if (/(?:^|-)(?:ios|tvos|watchos|visionos|android)/.test(target)) { + throw new Error( + `unsupported mobile target '${target}' (supported: ${MOBILE_LIBRARY_TARGETS.join(", ")})`, + ); + } + if (target.includes("linux")) return "linux"; + if (target.includes("windows")) return "win32"; + if (target.includes("macos") || target.includes("darwin")) return "darwin"; + throw new Error( + `unsupported target '${target}' (supported OS families: linux, windows, macos/darwin, wasm32-wasi)`, + ); +} diff --git a/packages/compiler/src/backend/targets.ts b/packages/compiler/src/backend/targets.ts index 85cc6fa5..cdd96a5d 100644 --- a/packages/compiler/src/backend/targets.ts +++ b/packages/compiler/src/backend/targets.ts @@ -439,6 +439,24 @@ function requestedTarget( } } +/** Native distributions already know their host ABI, including Linux libc. */ +export function selectNativeTarget( + raw: string, host: NativeTargetSpec, hostPlatform: string = process.platform, +): NativeTargetSpec | null { + const target = requestedTarget(raw, host, hostPlatform as NodeJS.Platform); + if (target === null) return null; + if (target.platform === "linux" && target.name !== host.name && target.name !== "android-arm64") { + return { + ...target, defaultLinker: "zig", defaultLinkerArgs: ["cc"], + linkerTargetTriple: `${target.architecture === "x64" ? "x86_64" : "aarch64"}-linux-${target.name.endsWith("musl") ? "musl" : "gnu.2.36"}`, + }; + } + if (target.platform === "darwin" && hostPlatform !== "darwin") { + return { ...target, defaultLinker: "zig", defaultLinkerArgs: ["cc"], linkerTargetTriple: `${target.architecture === "x64" ? "x86_64" : "aarch64"}-macos.14.0` }; + } + return target; +} + /** Select only a fully described scriptc target. LLVM accepting an arbitrary * triple is never evidence of its object ABI, runtime pack, link, or run. */ export function nativeCodegenTarget( @@ -449,19 +467,8 @@ export function nativeCodegenTarget( linuxLibc?: LinuxLibc, ): NativeTargetSpec | null { const host = nativeHostTarget(hostPlatform, hostArch, hostRelease, linuxLibc); - const target = requestedTarget(env["SCRIPTC_TARGET"] ?? "", host, hostPlatform); + const target = host === null ? null : selectNativeTarget(env["SCRIPTC_TARGET"] ?? "", host, hostPlatform); if (host === null || target === null || nativeHelperForTarget(target, hostPlatform, hostArch, linuxLibc) === null) return null; - // Cross ELF links use Zig's target libc. A native clang driver cannot - // infer or provide another architecture's CRT and sysroot. - if (target.platform === "linux" && target.name !== host.name && target.name !== "android-arm64") { - return { - ...target, defaultLinker: "zig", defaultLinkerArgs: ["cc"], - linkerTargetTriple: `${target.architecture === "x64" ? "x86_64" : "aarch64"}-linux-${target.name.endsWith("musl") ? "musl" : "gnu.2.36"}`, - }; - } - if (target.platform === "darwin" && hostPlatform !== "darwin") { - return { ...target, defaultLinker: "zig", defaultLinkerArgs: ["cc"], linkerTargetTriple: `${target.architecture === "x64" ? "x86_64" : "aarch64"}-macos.14.0` }; - } return target; } diff --git a/packages/compiler/src/backend/toolchain-environment.ts b/packages/compiler/src/backend/toolchain-environment.ts new file mode 100644 index 00000000..7c8a749d --- /dev/null +++ b/packages/compiler/src/backend/toolchain-environment.ts @@ -0,0 +1,110 @@ +import { createHash } from "node:crypto"; + +/** Environment variables consumed by clang, its linker/subtools, or the + * platform SDK selection. They are implicit command-line inputs: changing one + * must never reuse an artifact produced under the old toolchain posture. */ +const TOOLCHAIN_ENV_KEYS: readonly string[] = [ + "COMPILER_PATH", + "GCC_EXEC_PREFIX", + "CPATH", + "C_INCLUDE_PATH", + "CPLUS_INCLUDE_PATH", + "OBJC_INCLUDE_PATH", + "OBJCPLUS_INCLUDE_PATH", + "LIBRARY_PATH", + "LD_LIBRARY_PATH", + "LD_RUN_PATH", + "DYLD_LIBRARY_PATH", + "DYLD_FRAMEWORK_PATH", + "DYLD_FALLBACK_LIBRARY_PATH", + "DYLD_FALLBACK_FRAMEWORK_PATH", + "SDKROOT", + "DEVELOPER_DIR", + "MACOSX_DEPLOYMENT_TARGET", + "IPHONEOS_DEPLOYMENT_TARGET", + "TVOS_DEPLOYMENT_TARGET", + "WATCHOS_DEPLOYMENT_TARGET", + "DRIVERKIT_DEPLOYMENT_TARGET", + "XROS_DEPLOYMENT_TARGET", + "CCC_OVERRIDE_OPTIONS", + "CCC_ADD_ARGS", + "CLANG_CONFIG_FILE_SYSTEM_DIR", + "CLANG_CONFIG_FILE_USER_DIR", + "CC", + "CFLAGS", + "CPPFLAGS", + "LDFLAGS", + "AR", + "RANLIB", + "CMAKE_GENERATOR", + "CMAKE_TOOLCHAIN_FILE", + "ZIG_LIB_DIR", + "ZIG_LIBC", + "SOURCE_DATE_EPOCH", + "ZERO_AR_DATE", + "LANG", + "LC_ALL", + "LC_CTYPE", +]; + +/** Toolchain variables whose values name mutable files/directories consumed + * while compiling a TU (or can inject arbitrary compiler options). Hashing the + * value is insufficient: a header, SDK, config, compiler helper, or loaded + * dylib can change in place while the spelling remains stable. In that posture + * neither complete artifacts nor per-TU runtime objects are safe to reuse. */ +const MUTABLE_COMPILE_ENV_KEYS: readonly string[] = [ + "COMPILER_PATH", + "GCC_EXEC_PREFIX", + "CPATH", + "C_INCLUDE_PATH", + "CPLUS_INCLUDE_PATH", + "OBJC_INCLUDE_PATH", + "OBJCPLUS_INCLUDE_PATH", + "LD_LIBRARY_PATH", + "DYLD_LIBRARY_PATH", + "DYLD_FRAMEWORK_PATH", + "DYLD_FALLBACK_LIBRARY_PATH", + "DYLD_FALLBACK_FRAMEWORK_PATH", + "SDKROOT", + "DEVELOPER_DIR", + "CCC_OVERRIDE_OPTIONS", + "CCC_ADD_ARGS", + "CLANG_CONFIG_FILE_SYSTEM_DIR", + "CLANG_CONFIG_FILE_USER_DIR", + // `zig cc` resolves its bundled headers/runtime through ZIG_LIB_DIR and a + // caller-selected native libc description through ZIG_LIBC. Both values name + // mutable compiler inputs whose contents can change behind a stable path. + "ZIG_LIB_DIR", + "ZIG_LIBC", +]; + +/** These variables only redirect link-time inputs. Runtime objects remain + * reusable, but a complete executable could otherwise retain a library that + * was rebuilt in place behind the same search-path spelling. */ +const MUTABLE_LINK_ENV_KEYS: readonly string[] = ["LIBRARY_PATH", "LD_RUN_PATH"]; + +export interface ToolchainEnvironmentCachePolicy { + completeArtifacts: boolean; + runtimeObjects: boolean; +} + +export function toolchainEnvironmentCachePolicy( + env: NodeJS.ProcessEnv = process.env, +): ToolchainEnvironmentCachePolicy { + const mutableCompileInput = MUTABLE_COMPILE_ENV_KEYS.some((name) => env[name] !== undefined); + const mutableLinkInput = MUTABLE_LINK_ENV_KEYS.some((name) => env[name] !== undefined); + return { + completeArtifacts: !mutableCompileInput && !mutableLinkInput, + runtimeObjects: !mutableCompileInput, + }; +} + +export function toolchainEnvironmentFingerprint(env: NodeJS.ProcessEnv = process.env): string { + const hash = createHash("sha256").update("toolchain-env-v1\0"); + for (const name of TOOLCHAIN_ENV_KEYS) { + const value = env[name]; + const text: string = value ?? ""; + hash.update(name).update(value === undefined ? "\0unset\0" : "\0set\0").update(text).update("\0"); + } + return hash.digest("hex"); +} diff --git a/packages/compiler/src/backend/vendor-archives.ts b/packages/compiler/src/backend/vendor-archives.ts index 608de233..cfc4cf18 100644 --- a/packages/compiler/src/backend/vendor-archives.ts +++ b/packages/compiler/src/backend/vendor-archives.ts @@ -9,15 +9,8 @@ import type { CcDriver } from "./native-toolchain.js"; const execFileAsync = promisify(execFile); -/** The pinned QuickJS, mbedTLS, and zlib inputs used by native recipes. */ -export const QJS_COMMIT = "3c8f3d68953955950074c41c6e4d999562ae82a7"; -export const MBEDTLS_VERSION = "3.6.7"; -export const ZLIB_VERSION = "1.3.1"; -/** Exact translation-unit membership shared by cache builds and external - * source-pack recipes. */ -export const QJS_ENGINE_SOURCES = ["dtoa.c", "libregexp.c", "libunicode.c", "quickjs.c"] as const; -export const LRE_SOURCES = ["libregexp.c", "libunicode.c"] as const; -export const ZLIB_SOURCES = ["adler32.c", "compress.c", "crc32.c", "deflate.c", "infback.c", "inffast.c", "inflate.c", "inftrees.c", "trees.c", "uncompr.c", "zutil.c"] as const; +import { QJS_COMMIT, MBEDTLS_VERSION, ZLIB_VERSION, QJS_ENGINE_SOURCES, LRE_SOURCES, ZLIB_SOURCES } from "./vendor-inputs.js"; +export { QJS_COMMIT, MBEDTLS_VERSION, ZLIB_VERSION, QJS_ENGINE_SOURCES, LRE_SOURCES, ZLIB_SOURCES } from "./vendor-inputs.js"; export interface VendorArchiveContext { runtimeSrcDir(): string; diff --git a/packages/compiler/src/backend/vendor-inputs.ts b/packages/compiler/src/backend/vendor-inputs.ts new file mode 100644 index 00000000..a9b4b4ad --- /dev/null +++ b/packages/compiler/src/backend/vendor-inputs.ts @@ -0,0 +1,9 @@ +/** The pinned QuickJS, mbedTLS, and zlib inputs used by native recipes. */ +export const QJS_COMMIT = "3c8f3d68953955950074c41c6e4d999562ae82a7"; +export const MBEDTLS_VERSION = "3.6.7"; +export const ZLIB_VERSION = "1.3.1"; +/** Exact translation-unit membership shared by cache builds and external + * source-pack recipes. */ +export const QJS_ENGINE_SOURCES = ["dtoa.c", "libregexp.c", "libunicode.c", "quickjs.c"] as const; +export const LRE_SOURCES = ["libregexp.c", "libunicode.c"] as const; +export const ZLIB_SOURCES = ["adler32.c", "compress.c", "crc32.c", "deflate.c", "infback.c", "inffast.c", "inflate.c", "inftrees.c", "trees.c", "uncompr.c", "zutil.c"] as const; diff --git a/packages/compiler/src/cli/command.ts b/packages/compiler/src/cli/command.ts new file mode 100644 index 00000000..12d9fdce --- /dev/null +++ b/packages/compiler/src/cli/command.ts @@ -0,0 +1,325 @@ +import { rmSync, statSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { parseArgs } from "node:util"; +import { isExactExternalTypeSpecifier } from "../frontend/program.js"; +import { renderDiagnostics } from "../diagnostics/render.js"; +import { renderCoverage } from "../coverage/report.js"; +import { setProvenanceSources } from "../frontend/provenance-registry.js"; +import type { CliHost, NativeCacheWarmProfile } from "./host.js"; +import { resolveOutputOptions } from "./output-options.js"; +import { selectOutputPaths } from "./paths.js"; +import { CLI_OPTIONS, USAGE } from "./usage.js"; + +/* The exit discipline: NEVER process.exit() after writing output. stdout/ + * stderr to a PIPE are async streams — process.exit() drops whatever libuv + * hasn't flushed yet, which truncates large diagnostic renders at the pipe + * buffer (observed: 64KB cut mid-code-frame). Every path sets + * process.exitCode and returns instead; Node exits naturally once the + * streams drain. */ +class CliExit extends Error { + constructor(readonly code: number) { + super(`exit ${code}`); + this.name = "CliExit"; + } +} + +function fail(msg: string): never { + process.stderr.write(msg + "\n"); + throw new CliExit(1); +} + +/** parseArgs, with its throw turned into the CLI's own one-line error. + * Unparseable arguments are a USER error — an unknown flag or a missing + * value used to reach the top level as an uncaught ERR_PARSE_ARGS_* and + * print a Node stack trace over the user's terminal. */ +function parseCli(args: string[]): ReturnType> { + try { + return parseArgs({ args, options: CLI_OPTIONS, allowPositionals: true, allowNegative: true }); + } catch (err) { + // parseArgs appends a paragraph about `--` and positionals to the + // unknown-option message; the first sentence is the part that names + // what was wrong, and USAGE below already covers what was meant. + const raw = err instanceof Error ? err.message : String(err); + const msg = raw.split("\n")[0]!.split(". ")[0]!; + fail(`scriptc: ${msg}\n\n${USAGE}`); + } +} + +async function main(args: string[], host: CliHost): Promise { + const { values, positionals } = parseCli(args); + const externalTypeArgs = values["external-types"] ?? []; + + if (values.version) { + process.stdout.write(`${host.version()}\n`); + return 0; + } + + if (values.help || positionals.length === 0) { + process.stdout.write(USAGE); + return values.help ? 0 : 1; + } + + const [command, inputArg] = positionals; + if (command === "cache") { + if (inputArg !== "warm") fail(`unknown cache command "${inputArg ?? ""}" (supported: warm)\n\n${USAGE}`); + if (values.lib || values.dynamic || values.backend !== undefined || values.emit !== undefined || values.print !== undefined || values.ffi !== undefined || values.profile !== undefined || values.strip || values["windows-subsystem"] !== undefined || (values["npm-static"] ?? []).length > 0 || values["provenance-sources"] || externalTypeArgs.length > 0 || values.out !== undefined || values["emit-ir"] || !values["keep-llvm"]) { + fail(`scriptc cache warm takes only native optimization/sanitizer options and profile names\n\n${USAGE}`); + } + const optimization = values.optimization; + if (optimization !== undefined && optimization !== "release" && optimization !== "dev") { + fail(`unknown optimization "${optimization}" (supported: release, dev)\n\n${USAGE}`); + } + const profileArgs = positionals.slice(2); + const knownProfiles = new Set(["runtime", "tls", "dynamic"]); + for (const profile of profileArgs) { + if (!knownProfiles.has(profile as NativeCacheWarmProfile)) { + fail(`unknown cache warm profile "${profile}" (supported: runtime, tls, dynamic)`); + } + } + let result; + try { + result = await host.warmNativeCaches({ + ...(optimization === undefined ? {} : { optimization }), + sanitize: values.sanitize, + ...(profileArgs.length === 0 + ? {} + : { profiles: profileArgs as NativeCacheWarmProfile[] }), + }); + } catch (error) { + fail(`scriptc: ${error instanceof Error ? error.message : String(error)}`); + } + process.stdout.write(`${result.cacheRoot}\n`); + for (const profile of result.profiles) { + process.stdout.write(`${profile.profile}\t${Math.round(profile.elapsedMs)}ms\n`); + } + return 0; + } + if (command !== "build" && command !== "run" && command !== "coverage") { + fail(`unknown command "${command}"\n\n${USAGE}`); + } + if (values.lib) { + // LIBRARY mode: the profile names the entry module and pins the + // emission; the executable lane's mode flags have no meaning here + // (library artifacts are static-tier only, and there is no fallback + // concept — bare npm specifiers are static-or-refuse: the npm-static + // eligibility bar runs automatically, eligible packages compile into + // the graph, ineligible ones refuse with SC4013). + if (command !== "build") fail(`--lib is a build mode (scriptc build --lib --profile )\n\n${USAGE}`); + const profileArg = values.profile; + if (!profileArg) fail(`scriptc build --lib needs --profile \n\n${USAGE}`); + if (inputArg) { + fail("scriptc build --lib takes no input positional: the profile names the entry module"); + } + if (values.dynamic || values.backend !== undefined || values.emit !== undefined || values.print !== undefined || values.optimization !== undefined || values.strip || values.ffi !== undefined || values["windows-subsystem"] !== undefined || (values["npm-static"] ?? []).length > 0 || externalTypeArgs.length > 0) { + fail( + "scriptc build --lib takes no --dynamic/--backend/--emit/--print/--optimization/--strip/--windows-subsystem/--npm-static/--ffi/--external-types: the profile pins the emission and optimization, npm imports are judged automatically, outbound FFI belongs to executable builds, and external type mappings belong to coverage", + ); + } + const profilePath = resolve(profileArg); + const libOutDir = values.out ? dirname(resolve(values.out)) : join(dirname(profilePath), ".scriptc"); + const result = await host.compileLibrary({ + profilePath, + outDir: libOutDir, + ...(values.out ? { outPath: resolve(values.out) } : {}), + emitIr: values["emit-ir"], + sanitize: values.sanitize, + }); + if (!result.ok) { + const color = process.stderr.isTTY ?? false; + process.stderr.write(renderDiagnostics(result.diagnostics, result.sourceTexts, { color }) + "\n"); + const n = result.diagnostics.length; + process.stderr.write(`\n${n} error${n === 1 ? "" : "s"}.\n`); + return 1; + } + if (!values["keep-llvm"]) rmSync(result.llvmPath, { force: true }); + process.stdout.write(`${result.archivePath}\n`); + // The contract sidecar rides the same invocation when the profile + // declares one — name it so the embedder's tooling knows where to look. + if (result.sidecarPath !== undefined) process.stdout.write(`${result.sidecarPath}\n`); + return 0; + } + if (values["emit-ir"] && (command === "build" || command === "run")) { + process.stderr.write("scriptc: warning: --emit-ir is deprecated; use --emit=ir for IR as the primary output\n"); + } + if (!inputArg) fail(`missing input file\n\n${USAGE}`); + const input = resolve(inputArg); + if (command === "coverage" && values.emit !== undefined) { + fail(`--emit is a build/run option\n\n${USAGE}`); + } + if (command === "coverage" && values.strip) { + fail(`--strip is a build/run option\n\n${USAGE}`); + } + if (values.print !== undefined && values.print !== "native-link-info") { + fail(`unknown print kind "${values.print}" (supported: native-link-info)\n\n${USAGE}`); + } + const printNativeLinkInfo = values.print === "native-link-info"; + if (printNativeLinkInfo && command !== "build") { + fail(`--print=native-link-info is a build option\n\n${USAGE}`); + } + if (printNativeLinkInfo && values.emit !== undefined && values.emit !== "obj") { + fail(`--print=native-link-info requires --emit=obj\n\n${USAGE}`); + } + if (externalTypeArgs.length > 0 && command !== "coverage") { + fail(`--external-types is a coverage-only option\n\n${USAGE}`); + } + const externalTypes: Record = Object.create(null) as Record; + for (const mapping of externalTypeArgs) { + const equals = mapping.indexOf("="); + if (equals <= 0 || equals === mapping.length - 1) { + fail(`invalid --external-types mapping ${JSON.stringify(mapping)} (expected )`); + } + const specifier = mapping.slice(0, equals).trim(); + const declarationArg = mapping.slice(equals + 1).trim(); + if (!isExactExternalTypeSpecifier(specifier)) { + fail(`invalid --external-types specifier ${JSON.stringify(specifier)} (expected an exact bare package specifier)`); + } + if (!/\.d\.(?:ts|mts|cts)$/.test(declarationArg)) { + fail(`invalid --external-types declaration ${JSON.stringify(declarationArg)} (expected a .d.ts, .d.mts, or .d.cts file)`); + } + if (externalTypes[specifier] !== undefined) { + fail(`duplicate --external-types mapping for ${JSON.stringify(specifier)}`); + } + const declarationPath = resolve(declarationArg); + try { + if (!statSync(declarationPath).isFile()) throw new Error("not a file"); + } catch { + fail(`--external-types declaration does not name a readable file: ${declarationPath}`); + } + externalTypes[specifier] = declarationPath; + } + const ffiProfilePath = values.ffi !== undefined ? resolve(values.ffi) : undefined; + if (values.backend !== undefined && values.backend !== "llvm") { + fail(`unknown backend "${values.backend}" (supported: llvm)\n\n${USAGE}`); + } + const optimization = values.optimization; + if (optimization !== undefined && optimization !== "release" && optimization !== "dev") { + fail(`unknown optimization "${optimization}" (supported: release, dev)\n\n${USAGE}`); + } + const windowsSubsystem = values["windows-subsystem"]; + if (windowsSubsystem !== undefined && windowsSubsystem !== "console" && windowsSubsystem !== "gui") { + fail(`unknown Windows subsystem "${windowsSubsystem}" (supported: console, gui)\n\n${USAGE}`); + } + if (windowsSubsystem !== undefined && command === "coverage") { + fail(`--windows-subsystem is only supported for executable builds\n\n${USAGE}`); + } + const output = command === "coverage" + ? null + : resolveOutputOptions(command, { + ...(values.emit === undefined && !printNativeLinkInfo + ? {} + : { emit: values.emit ?? "obj" }), + emitIr: values["emit-ir"], + ...(values.backend === undefined ? {} : { backend: values.backend }), + keepLlvm: values["keep-llvm"], + sanitize: values.sanitize, + ...(values.optimization === undefined ? {} : { optimization: values.optimization }), + strip: values.strip, + ...(windowsSubsystem === undefined ? {} : { windowsSubsystem }), + ...(values.ffi === undefined ? {} : { ffi: values.ffi }), + }); + if (output !== null && !output.ok) fail(`${output.message}\n\n${USAGE}`); + const backend = output === null ? undefined : output.backend; + + // --npm-static: repeatable and comma-splittable; the literal "auto" + // switches to eligibility-based detection (mixing "auto" with names + // is rejected — the shapes answer different questions). + const npmStaticRaw = (values["npm-static"] ?? []).flatMap((v) => v.split(",")).map((v) => v.trim()).filter((v) => v !== ""); + let npmStatic: string[] | "auto" | undefined; + if (npmStaticRaw.includes("auto")) { + if (npmStaticRaw.length > 1) fail(`--npm-static auto cannot be combined with package names\n\n${USAGE}`); + npmStatic = "auto"; + } else if (npmStaticRaw.length > 0) { + npmStatic = npmStaticRaw; + } + + // --provenance-sources resolves BEFORE the program loads (tsgo needs the + // source "paths" at creation): attestations and source trees fetch (or + // ride the content-addressed cache / the offline manifest), the registry + // installs, and every fallback prints as a note — never a failure. + const provenance = values["provenance-sources"] ? await host.resolveProvenanceSources(input) : null; + if (provenance !== null) { + setProvenanceSources(provenance); + for (const pkg of provenance.packages) { + process.stderr.write( + `provenance: ${pkg.name}@${pkg.version} ← ${pkg.repo.replace(/^git\+/, "")} @ ${pkg.commit.slice(0, 12)} (source compiles statically)\n`, + ); + } + for (const note of provenance.notes) process.stderr.write(`provenance: ${note}\n`); + } + + if (command === "coverage") { + const { coverage, sourceTexts } = await host.analyze(input, { + dynamic: values.dynamic, + ...(npmStatic !== undefined ? { npmStatic } : {}), + ...(ffiProfilePath !== undefined ? { ffiProfilePath } : {}), + ...(Object.keys(externalTypes).length > 0 ? { externalTypes } : {}), + }); + const color = process.stdout.isTTY ?? false; + process.stdout.write(renderCoverage(coverage, { color, sourceTexts }) + "\n"); + return coverage.preflightFailed ? 1 : 0; + } + + if (output === null || !output.ok) throw new Error("internal output-option state"); + if (windowsSubsystem !== undefined && host.sourceTargetPlatform() !== "win32") { + fail(`--windows-subsystem requires a Windows executable target\n\n${USAGE}`); + } + const { outDir, outPath } = selectOutputPaths(input, output.cliOutputKind, values.out, + output.cliOutputKind === "exe" ? host.sourceTargetPlatform() : undefined); + + let nativeLinkInfo: object | undefined; + const build = async (): Promise => { + const result = await host.compile(input, { + outPath, + outDir, + outputKind: output.outputKind, + emitIr: output.emitIr, + sanitize: values.sanitize, + dynamic: values.dynamic, + ...(backend !== undefined ? { backend } : {}), + ...(optimization !== undefined ? { optimization } : {}), + ...(values.strip ? { strip: true } : {}), + ...(windowsSubsystem !== undefined ? { windowsSubsystem } : {}), + ...(npmStatic !== undefined ? { npmStatic } : {}), + ...(ffiProfilePath !== undefined ? { ffiProfilePath } : {}), + ...(printNativeLinkInfo ? { nativeLinkInfo: true } : {}), + }); + if (!result.ok) { + const color = process.stderr.isTTY ?? false; + process.stderr.write(renderDiagnostics(result.diagnostics, result.sourceTexts, { color }) + "\n"); + const n = result.diagnostics.length; + process.stderr.write(`\n${n} error${n === 1 ? "" : "s"}.\n`); + throw new CliExit(1); + } + if (result.artifact.kind === "exe") { + if (!values["keep-llvm"]) rmSync(result.artifact.translationUnitPath, { force: true }); + } else if (result.artifact.kind === "obj") { + nativeLinkInfo = result.artifact.nativeLinkInfo; + } + return result.artifact.path; + }; + + const binary = await build(); + + if (command === "run") { + return host.run(binary); + } + + if (printNativeLinkInfo) { + if (nativeLinkInfo === undefined) throw new Error("internal native-link-info state"); + // Keep stdout pure JSON for tooling; the ordinary artifact path is in + // program.object inside the document. + process.stdout.write(`${JSON.stringify(nativeLinkInfo, null, 2)}\n`); + } else { + process.stdout.write(`${binary}\n`); + } + return 0; +} + +/** Both installed and seed commands use this argument and diagnostic contract. */ +export async function runCli(args: string[], host: CliHost): Promise { + try { return await main(args, host); } + catch (err) { + if (err instanceof Error && err.name === "CliExit") return Number(err.message.slice(5)); + throw err; + } +} diff --git a/packages/compiler/src/cli/host.ts b/packages/compiler/src/cli/host.ts new file mode 100644 index 00000000..f2cd6c8b --- /dev/null +++ b/packages/compiler/src/cli/host.ts @@ -0,0 +1,20 @@ +import type { AnalyzeOptions, AnalyzeResult, CompileLibraryOptions, CompileLibraryResult, CompileRequestOptions, CompileRequestResult } from "../compile-types.js"; +import type { ProvenanceSources } from "../frontend/provenance-registry.js"; + +export type NativeCacheWarmProfile = "runtime" | "tls" | "dynamic"; + +/** Only host operations vary between the seed and installed compiler. */ +export interface CliHost { + version: () => string; + sourceTargetPlatform: () => string; + analyze: (entry: string, options: AnalyzeOptions) => Promise; + compile: (entry: string, options: CompileRequestOptions) => Promise; + compileLibrary: (options: CompileLibraryOptions) => Promise; + resolveProvenanceSources: (entry: string) => Promise; + warmNativeCaches: (options: { + optimization?: "release" | "dev"; + sanitize: boolean; + profiles?: NativeCacheWarmProfile[]; + }) => Promise<{ cacheRoot: string; profiles: { profile: NativeCacheWarmProfile; elapsedMs: number }[] }>; + run: (binary: string) => Promise; +} diff --git a/packages/compiler/src/cli/output-options.ts b/packages/compiler/src/cli/output-options.ts new file mode 100644 index 00000000..9dec4c91 --- /dev/null +++ b/packages/compiler/src/cli/output-options.ts @@ -0,0 +1,91 @@ +import type { CompileOutputKind } from "../compile-types.js"; +import type { CliOutputKind } from "./paths.js"; + +export interface OutputOptionValues { + emit?: string; + emitIr: boolean; + backend?: string; + keepLlvm: boolean; + sanitize: boolean; + optimization?: string; + strip?: boolean; + windowsSubsystem?: string; + ffi?: string; +} + +export type OutputOptionResolution = + | { + ok: true; + outputKind: CompileOutputKind; + cliOutputKind: CliOutputKind; + backend?: "llvm"; + emitIr: boolean; + deprecateEmitIr: boolean; + } + | { ok: false; message: string }; + +const SOURCE_KINDS = new Set(["ir", "llvm"]); +const NATIVE_ARTIFACT_KINDS = new Set(["asm", "obj"]); + +/** Pure compatibility/validation matrix for build/run output selection. */ +export function resolveOutputOptions( + command: "build" | "run", + values: OutputOptionValues, +): OutputOptionResolution { + if (values.backend !== undefined && values.backend !== "llvm") { + return { ok: false, message: `unknown backend "${values.backend}" (supported: llvm)` }; + } + const backend = values.backend as "llvm" | undefined; + const rawEmit = values.emit; + if ( + rawEmit !== undefined && rawEmit !== "ir" && rawEmit !== "llvm" && + rawEmit !== "asm" && rawEmit !== "obj" && rawEmit !== "exe" + ) { + return { + ok: false, + message: `unknown emit kind "${rawEmit}" (supported: ir, llvm, asm, obj, exe)`, + }; + } + const emit = (rawEmit ?? "exe") as CliOutputKind; + if (command === "run" && emit !== "exe") { + return { ok: false, message: `scriptc run requires --emit=exe` }; + } + if (values.emitIr && rawEmit !== undefined && emit !== "ir" && emit !== "exe") { + return { ok: false, message: `--emit-ir cannot be combined with --emit=${emit}; use --emit=ir` }; + } + if (values.emitIr && emit === "ir") { + return { ok: false, message: `--emit-ir and --emit=ir select the same output; use --emit=ir` }; + } + if (values.windowsSubsystem !== undefined && emit !== "exe") { + return { ok: false, message: `--windows-subsystem is only supported with --emit=exe` }; + } + if (values.strip && emit !== "exe") { + return { ok: false, message: `--strip is only supported with --emit=exe` }; + } + if (emit === "ir" && backend !== undefined) { + return { ok: false, message: `--emit=ir cannot be combined with --backend; IR is emitted before backend selection` }; + } + if (SOURCE_KINDS.has(emit)) { + if (!values.keepLlvm) { + return { ok: false, message: `--no-keep-llvm is only meaningful with --emit=exe` }; + } + if (values.sanitize) { + return { ok: false, message: `--sanitize is only meaningful with --emit=exe` }; + } + if (values.optimization !== undefined) { + return { ok: false, message: `--optimization is only meaningful with --emit=exe` }; + } + } + if (NATIVE_ARTIFACT_KINDS.has(emit) && !values.keepLlvm) { + return { ok: false, message: `--no-keep-llvm is only meaningful with --emit=exe` }; + } + const outputKind = emit as CompileOutputKind; + return { + ok: true, + outputKind, + cliOutputKind: emit, + ...(emit === "ir" && backend === undefined ? {} : { backend: "llvm" as const }), + emitIr: values.emitIr && emit === "exe", + deprecateEmitIr: values.emitIr, + }; +} diff --git a/packages/compiler/src/cli/paths.ts b/packages/compiler/src/cli/paths.ts new file mode 100644 index 00000000..3ea4c664 --- /dev/null +++ b/packages/compiler/src/cli/paths.ts @@ -0,0 +1,61 @@ +import { basename, dirname, join, resolve } from "node:path"; +import { configuredTargetPlatform as sourceTargetPlatform } from "../backend/target-platform.js"; +export { wasiEnvironment, wasiPreopens } from "./wasi-paths.js"; + +export type CliOutputKind = "ir" | "llvm" | "asm" | "obj" | "exe"; + +const POSIX_SUFFIXES: Record = { + ir: ".ir.json", + llvm: ".ll", + asm: ".s", + obj: ".o", + exe: "", +}; + +const WINDOWS_SUFFIXES: Record = { + ...POSIX_SUFFIXES, + asm: ".asm", + obj: ".obj", + exe: ".exe", +}; + +export function defaultOutputName( + stem: string, + kind: CliOutputKind, + platform?: string, +): string { + const selectedPlatform = platform ?? (kind === "exe" ? sourceTargetPlatform() : process.platform); + if (kind === "exe" && selectedPlatform === "wasi") return `${stem}.wasm`; + return `${stem}${(selectedPlatform === "win32" ? WINDOWS_SUFFIXES : POSIX_SUFFIXES)[kind]}`; +} + +export interface OutputPaths { + outDir: string; + outPath: string; +} + +/** One authority for explicit and default primary artifact paths. */ +export function selectOutputPaths( + input: string, + kind: CliOutputKind, + explicitOut?: string, + platform?: string, +): OutputPaths { + const absoluteInput = resolve(input); + const outDir = explicitOut === undefined + ? join(dirname(absoluteInput), ".scriptc") + : dirname(resolve(explicitOut)); + const stem = basename(absoluteInput).replace(/\.(ts|mts|cts|js|mjs|cjs|c|ll)$/, ""); + return { + outDir, + outPath: explicitOut === undefined + ? join(outDir, defaultOutputName(stem, kind, platform)) + : resolve(explicitOut), + }; +} + +/** Default executable filename for the build target. Explicit --out paths + * stay exact; only scriptc's generated default needs the Windows PE suffix. */ +export function defaultExecutableName(stem: string, platform: string = sourceTargetPlatform()): string { + return defaultOutputName(stem, "exe", platform); +} diff --git a/packages/compiler/src/cli/usage.ts b/packages/compiler/src/cli/usage.ts new file mode 100644 index 00000000..062e2653 --- /dev/null +++ b/packages/compiler/src/cli/usage.ts @@ -0,0 +1,86 @@ +export const USAGE = `scriptc — TypeScript/JavaScript to native and WebAssembly executables (experimental) + +Usage: + scriptc build [options] compile to an executable or source artifact + scriptc run [options] compile and run + scriptc coverage how much compiles statically, and why not + scriptc coverage --dynamic what a --dynamic build compiles, and what still blocks it + scriptc coverage --external-types + type-resolve an embedder-provided module for analysis + scriptc build --lib --profile library mode: compile the profile's entry + module to a linkable static archive + (.lib.a), or a Wasm reactor + (.wasm) on wasm32-wasi, + exporting the profile symbols; a profile + with a sidecar section also gets the + contract sidecar JSON beside the archive + scriptc cache warm [runtime|tls|dynamic…] prebuild expensive native cache families + for the current compiler/SDK/target + +Options: + -o, --out primary output path (default: .scriptc/) + --emit primary output: ir, llvm, asm, obj, or exe + (default: exe). asm/obj use the matching platform helper + --print print machine-readable metadata instead of the output path + (native-link-info implies --emit=obj and never links) + --backend code generator (llvm) + --optimization + native optimization posture (default: release/-O2). dev + uses -O0, source breakpoints, and cached LLVM object shards; + macOS executable builds also produce an adjacent .dSYM + --strip remove symbol/debug payload from the linked executable + for smaller builds (opt in; --emit=exe only) + --windows-subsystem + Windows executable subsystem (default: console). gui + prevents Windows from opening a console window + --keep-llvm keep generated LLVM IR beside the executable (default) + --no-keep-llvm delete generated LLVM IR after compiling + --emit-ir also write IR beside an executable or library archive; + deprecated for executables: use --emit=ir for primary IR + --sanitize build with ASan + runtime RC audit + --dynamic embed the dynamic engine (adds ~620KB; static stays the default) + --ffi bind signature-only TypeScript declarations to native + C symbols and link the manifest's archives/libraries + --npm-static + compile the named npm packages' shipped JS statically as + program modules (repeatable; "auto" opts in every eligible + direct import: own .d.ts, unminified JS, no build-transform + markers). A package preflight refuses falls back to the + island (--dynamic) with a coverage-report note — opt-in, + experimental + --provenance-sources + EXPERIMENTAL: compile npm dependencies from their + provenance-attested SOURCE (fetched at the attested + commit) as static program modules; packages without a + usable attestation keep the island path (a note, never + a failure) + --external-types + coverage only: map an exact bare module specifier to a + local declaration file. The declaration supplies types + for analysis; the host module remains an explicit + external-boundary blocker (repeatable) + -h, --help show this help + -v, --version print the version +`; + +export const CLI_OPTIONS = { + out: { type: "string", short: "o" }, + emit: { type: "string" }, + print: { type: "string" }, + backend: { type: "string" }, + optimization: { type: "string" }, + strip: { type: "boolean", default: false }, + "windows-subsystem": { type: "string" }, + "keep-llvm": { type: "boolean", default: true }, + "emit-ir": { type: "boolean", default: false }, + sanitize: { type: "boolean", default: false }, + dynamic: { type: "boolean", default: false }, + ffi: { type: "string" }, + "npm-static": { type: "string", multiple: true }, + "provenance-sources": { type: "boolean", default: false }, + "external-types": { type: "string", multiple: true }, + lib: { type: "boolean", default: false }, + profile: { type: "string" }, + help: { type: "boolean", short: "h", default: false }, + version: { type: "boolean", short: "v", default: false }, +} as const; diff --git a/packages/compiler/src/cli/wasi-paths.ts b/packages/compiler/src/cli/wasi-paths.ts new file mode 100644 index 00000000..07108936 --- /dev/null +++ b/packages/compiler/src/cli/wasi-paths.ts @@ -0,0 +1,45 @@ +import { tmpdir } from "node:os"; +import { wasiGuestPath } from "../wasi-paths.js"; + +/** Host paths exposed by `scriptc run` to a WASI Preview 1 module. Guest + * `/tmp` maps to the host's real platform temp directory instead of assuming + * the POSIX spelling exists (notably false on Windows). */ +export function wasiPreopens( + cwd: string = process.cwd(), + hostTmp: string = tmpdir(), +): Record { + return { "/": cwd, "/tmp": hostTmp }; +} + +/** Environment inherited by a WASI module. Host-absolute directory values + * must not claim paths outside the guest namespace: `/` is the module's + * capability root/home/cwd and `/tmp` is its writable temporary directory. */ +export function wasiEnvironment( + env: NodeJS.ProcessEnv = process.env, + cwd: string = process.cwd(), + hostTmp: string = tmpdir(), +): Record { + const guest: Record = {}; + for (const [name, value] of Object.entries(env)) { + if (value !== undefined) guest[name] = value; + } + + guest["PWD"] = "/"; + guest["HOME"] = "/"; + guest["TMPDIR"] = "/tmp"; + if (guest["USERPROFILE"] !== undefined) guest["USERPROFILE"] = "/"; + if (guest["TMP"] !== undefined) guest["TMP"] = "/tmp"; + if (guest["TEMP"] !== undefined) guest["TEMP"] = "/tmp"; + + // These optional shell/package-manager paths retain their meaning only + // when they fall under a capability the runner actually exposes. + for (const key of ["OLDPWD", "INIT_CWD"]) { + const value = guest[key]; + if (value === undefined) continue; + const mapped = wasiGuestPath(value, cwd, hostTmp); + if (mapped === null) delete guest[key]; + else guest[key] = mapped; + } + + return guest; +} diff --git a/packages/compiler/src/cli/wasi-runner.ts b/packages/compiler/src/cli/wasi-runner.ts new file mode 100644 index 00000000..6fb88bc8 --- /dev/null +++ b/packages/compiler/src/cli/wasi-runner.ts @@ -0,0 +1,39 @@ +#!/usr/bin/env node +/* The subprocess host for `scriptc run` on wasm32-wasi. Keeping the WASI + * instance outside the CLI process preserves native run's exit isolation: + * process.exit(), traps, and signals cannot take the compiler process down. + */ +import { readFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { WASI } from "node:wasi"; +import { wasiGuestPath } from "../wasi-paths.js"; +import { wasiEnvironment, wasiPreopens } from "./wasi-paths.js"; + +type WasiInstance = Parameters[0]; +declare const WebAssembly: { + instantiate( + bytes: Uint8Array, + imports: ReturnType, + ): Promise<{ instance: WasiInstance }>; +}; + +const binary = process.argv[2]; +if (binary === undefined) throw new Error("scriptc WASI runner needs a module path"); +const cwd = process.cwd(); +const hostTmp = tmpdir(); + +const wasi = new WASI({ + version: "preview1", + args: [wasiGuestPath(binary, cwd, hostTmp) ?? binary], + env: wasiEnvironment(process.env, cwd, hostTmp), + // A native scriptc executable inherits access to the caller's filesystem. + // WASI is capability-based, so expose the caller's working tree as `/` + // and the platform's actual temporary directory as guest `/tmp`. + preopens: wasiPreopens(cwd, hostTmp), + returnOnExit: true, +}); +const instantiated = await WebAssembly.instantiate( + await readFile(binary), + wasi.getImportObject(), +); +process.exitCode = wasi.start(instantiated.instance); diff --git a/packages/compiler/src/compile-types.ts b/packages/compiler/src/compile-types.ts new file mode 100644 index 00000000..d39e3a11 --- /dev/null +++ b/packages/compiler/src/compile-types.ts @@ -0,0 +1,157 @@ +import type { WindowsSubsystem } from "./backend/targets.js"; +import type { NativeLinkInfo } from "./backend/native-link-info.js"; +import type { ScrDiagnostic } from "./diagnostics/diagnostic.js"; +import type { CoverageInput } from "./coverage/report.js"; + +export type CompileOutputKind = "ir" | "llvm" | "asm" | "obj" | "exe"; + +export interface CompileBaseOptions { + /** Primary artifact path. The CLI supplies the output-kind default. */ + outPath: string; + /** Where generated intermediates and compatibility side artifacts land. */ + outDir: string; + /** @deprecated This option no longer controls output cleanup; sibling artifacts are retained. */ + defaultOutputPath?: boolean; + /** Compatibility-only additive IR side artifact for executable builds. + * The CLI's deprecated --emit-ir flag supplies this option. */ + emitIr?: boolean; + sanitize?: boolean; + /** Embed the dynamic-island engine (--dynamic). Off = the static default: + * island constructs are diagnostics and nothing about codegen or linking + * changes. */ + dynamic?: boolean; + /** LLVM is the production code generator. */ + backend?: "llvm"; + /** Native optimization posture. Release is the shipped -O2 default; dev + * uses -O0, source line tables, and stable multi-TU object caching for + * large LLVM programs. Darwin executables include an adjacent .dSYM. */ + optimization?: "release" | "dev"; + /** Remove symbol/debug payload from an executable at link time. */ + strip?: boolean; + /** Windows PE executable subsystem. Console is the default; GUI suppresses + * automatic console-window creation. Only valid for Windows executables. */ + windowsSubsystem?: WindowsSubsystem; + /** --npm-static: package names whose shipped, unminified JS compiles + * STATICALLY as program modules (inference types the bodies; statements + * the lowering cannot prove become runtime fences). "auto" opts in every + * directly-imported package passing the eligibility heuristics (own + * .d.ts, unminified JS, no build-transform markers). A package whose + * preflight refuses marks itself an offender and falls back to the + * island (--dynamic) or the requires-dynamic diagnostic (static builds) + * — never a silent misbuild. Off by default: nothing changes without + * the flag. */ + npmStatic?: readonly string[] | "auto"; + /** Outbound native FFI manifest. Its signature-only TypeScript bindings + * lower to direct C ABI calls. Source outputs retain those declarations; + * archive/system-library inputs join only an executable link. */ + ffiProfilePath?: string; + /** Attach the machine-readable external link recipe to an object result. + * Valid only with outputKind "obj"; it never invokes a linker. */ + nativeLinkInfo?: boolean; +} + +/** Executable compile options. This remains the compatibility type for the + * historical compile() API, whose omitted output kind means executable. */ +export interface CompileOptions extends CompileBaseOptions { + outputKind?: "exe"; + /** Internal validation lane retained for helper-object artifact tests. + * Supported ordinary LLVM executable builds select this path automatically. */ + nativeProgramObject?: boolean; +} + +/** Source-artifact compile options, discriminated by the required kind. */ +export interface CompileSourceOptions extends CompileBaseOptions { + outputKind: Exclude; +} + +/** Internal/dynamic request shape for callers that select the kind at runtime. + * Statically executable/source callers should prefer the narrower interfaces. */ +export interface CompileRequestOptions extends CompileBaseOptions { + outputKind?: CompileOutputKind; + /** Internal validation lane for executable requests. */ + nativeProgramObject?: boolean; +} + +export type CompileArtifact = + | { kind: "ir"; path: string } + | { kind: "llvm"; path: string } + | { kind: "asm"; path: string } + | { kind: "obj"; path: string; nativeLinkInfo?: NativeLinkInfo } + | { + kind: "exe"; + path: string; + translationUnitPath: string; + backend: "llvm"; + + }; + +export type CompileFailure = { + ok: false; + diagnostics: ScrDiagnostic[]; + sourceTexts: Map; +}; + +export type CompileSourceResult = + | { ok: true; artifact: Extract } + | CompileFailure; + +/** Historical executable result shape retained for source compatibility. */ +export type CompileResult = + | { + ok: true; + binaryPath: string; + llvmPath: string; + irPath?: string; + backend: "llvm"; + + } + | CompileFailure; + +export type CompileExecutableResult = + /** The generated LLVM source is retained beside the executable. */ + | (Extract & { + artifact: Extract; + }) + | CompileFailure; + +/** Result union for callers that choose outputKind dynamically. */ +export type CompileRequestResult = CompileSourceResult | CompileExecutableResult; + +export interface CompileLibraryOptions { + profilePath: string; + /** Where the archive and the kept program TU land. */ + outDir: string; + /** Artifact path. Default: .lib.a, or .wasm for wasm32-wasi. */ + outPath?: string; + emitIr?: boolean; + sanitize?: boolean; +} + +export type CompileLibraryResult = + /** `sidecarPath` is present exactly when the profile declares a + * `sidecar` section: the contract JSON written beside the archive by + * the same invocation (ask 2). */ + | { ok: true; archivePath: string; llvmPath: string; backend: "llvm"; irPath?: string; sidecarPath?: string } + | { ok: false; diagnostics: ScrDiagnostic[]; sourceTexts: Map }; + +export interface AnalyzeOptions { + /** Analyze as a --dynamic build (island constructs lower instead of + * producing requires-dynamic diagnostics). */ + dynamic?: boolean; + /** --npm-static (see CompileOptions.npmStatic): the analysis compiles + * opted-in packages' JS as program modules and the coverage report + * carries each package's static/fallback status. */ + npmStatic?: readonly string[] | "auto"; + /** Analyze with the outbound native bindings from this FFI manifest. */ + ffiProfilePath?: string; + /** Coverage-only external host type surfaces: exact bare module + * specifier → local declaration file. The checker uses the declarations + * to analyze project code, but imported runtime values remain explicit + * SC1010 blockers rather than being counted as executable. */ + externalTypes?: Readonly>; +} + +export interface AnalyzeResult { + coverage: CoverageInput; + sourceTexts: Map; +} diff --git a/packages/compiler/src/coverage/report.ts b/packages/compiler/src/coverage/report.ts index 3af5092d..778b8024 100644 --- a/packages/compiler/src/coverage/report.ts +++ b/packages/compiler/src/coverage/report.ts @@ -190,21 +190,21 @@ export function renderCoverage(input: CoverageInput, opts: { color?: boolean; so // surface as SC2030 blockers below; "lazy trap" rows DON'T fail the // build — the binary embeds Node's call-time error and the call throws // at runtime, exactly where Node would have failed. - const builtins = (input.dynamic && input.npmBuiltins) || []; - const traps = (input.dynamic && input.npmLazyTraps) || []; + const builtins = input.dynamic ? input.npmBuiltins ?? [] : []; + const traps = input.dynamic ? input.npmLazyTraps ?? [] : []; if (builtins.length > 0 || traps.length > 0) { out.push( ` ${c(DIM, traps.length > 0 ? "embedded npm code imports Node builtins and unresolved specifiers:" : "embedded npm code imports Node builtins:")}`, ); - const widestB = Math.max( + const widestB = Math.max(...[ ...builtins.map((b) => b.builtin.length), ...traps.map((t) => t.specifier.length), - ); + ]); // Pad the plain words before coloring — escape codes have no width. - const widestS = Math.max( + const widestS = Math.max(...[ ...builtins.map((b) => (b.shimmed ? 7 : b.lazy ? 23 : 11)), ...traps.map(() => 24), - ); + ]); for (const b of builtins) { const status = b.shimmed ? c(GREEN, "shimmed".padEnd(widestS)) @@ -247,7 +247,7 @@ export function renderCoverage(input: CoverageInput, opts: { color?: boolean; so let pFailed = 0; let pIsland = 0; const dir = pkg.dir.endsWith("/") ? pkg.dir : `${pkg.dir}/`; - for (const [file, s] of input.statsByFile ?? []) { + for (const [file, s] of input.statsByFile ?? new Map()) { if (!file.startsWith(dir)) continue; pTotal += s.total; pFailed += s.failed; diff --git a/packages/compiler/src/coverage/surface-manifest.ts b/packages/compiler/src/coverage/surface-manifest.ts index bafa96ba..55cb8e24 100644 --- a/packages/compiler/src/coverage/surface-manifest.ts +++ b/packages/compiler/src/coverage/surface-manifest.ts @@ -187,7 +187,7 @@ export function generateSurfaceManifest(compilerVersion: string): SurfaceManifes ["for-using-of", "for (using ... of ...) over arrays", "each array element is disposed at the end of its iteration, including break and continue paths"], ["finally-abrupt-completions", "abrupt completions through finally", "return, throw, break, continue, and labeled jumps run crossed finally blocks; a finally completion replaces the pending one"], ["spread-arguments", "spread arguments", "non-empty fixed tuples flatten into fixed signatures with evaluate-once ordering; arrays, Sets, and statically represented class iterables spread into typed rest parameters"], - ] as const) { + ] as [string, string, string][]) { add({ id: `syntax.${id}`, kind: "syntax", name, status: "static", note }); } diff --git a/packages/compiler/src/executable/prepare.ts b/packages/compiler/src/executable/prepare.ts new file mode 100644 index 00000000..350d40f8 --- /dev/null +++ b/packages/compiler/src/executable/prepare.ts @@ -0,0 +1,79 @@ +import type { CompileFailure, CompileRequestOptions } from "../compile-types.js"; +import type { FfiProfile } from "../ffi/ffi-manifest.js"; +import { checkerPanicDiag, iceDiag, isCheckerPanic, type ScrDiagnostic } from "../diagnostics/diagnostic.js"; +import type { IrModule, SrcLoc } from "../ir/ir.js"; +import { validateModule } from "../ir/validate.js"; +import { moduleWasiUnavailableSurface, targetRefusalDiag } from "../backend/target-diagnostics.js"; +import type { LowerResult, LowerStats } from "../frontend/lowering/lowerer.js"; +import type { FrontendFactory } from "../frontend/pipeline.js"; + +export interface PreparedExecutableModule { + ok: true; + mod: IrModule; + sourceTexts: Map; + stats: LowerStats; +} + +/** Release the parser/checker before native object generation begins. */ +export function prepareExecutableModule( + entryPath: string, + opts: CompileRequestOptions, + ffi: FfiProfile | null, + buildPlatform: string, + createFrontend: FrontendFactory, +): PreparedExecutableModule | CompileFailure { + const fe = createFrontend(entryPath, opts.npmStatic); + let lowered: LowerResult; + let sourceTexts: Map; + // The frontend (and its tsgo server) is released as soon as lowering + // ends — clang and the link never hold it open. + try { + const fail = (diagnostics: ScrDiagnostic[]): CompileFailure => ({ + ok: false, + diagnostics, + sourceTexts: fe.sourceTexts(), + }); + + if (fe.preflight.length > 0) return fail(fe.preflight); + + try { + lowered = fe.lower({ + dynamic: opts.dynamic ?? false, + targetPlatform: buildPlatform, + ...(ffi !== null ? { ffiImports: ffi.functions } : {}), + }); + } catch (e) { + // The last-resort panic fence: an upstream tsgo panic that crossed a + // checker call no statement/collection fence wrapped still becomes a + // clean failed compile (anchored at the entry), never a crashed CLI. + if (!isCheckerPanic(e)) throw e; + return fail([ + checkerPanicDiag(e.message.split("\n", 1)[0]!, { file: entryPath, start: 0, end: 0 }), + ]); + } + if (lowered.module === null) return fail(lowered.diagnostics); + + const validation = validateModule(lowered.module); + if (validation.length > 0) { + return fail(validation.map((v) => iceDiag(v.message, v.loc))); + } + if (buildPlatform === "wasi") { + const entryLoc: SrcLoc = { file: entryPath, start: 0, end: 0 }; + if (opts.sanitize) { + return fail([targetRefusalDiag("wasm32-wasi", "--sanitize", entryLoc)]); + } + if (ffi !== null) { + return fail([targetRefusalDiag("wasm32-wasi", "native FFI manifests", entryLoc)]); + } + const unavailable = moduleWasiUnavailableSurface(lowered.module); + if (unavailable !== null) { + return fail([targetRefusalDiag("wasm32-wasi", unavailable.surface, unavailable.loc)]); + } + } + sourceTexts = fe.sourceTexts(); + } finally { + fe.dispose(); + } + + return { ok: true, mod: lowered.module!, sourceTexts, stats: lowered.stats }; +} diff --git a/packages/compiler/src/frontend/analysis.ts b/packages/compiler/src/frontend/analysis.ts new file mode 100644 index 00000000..32b92413 --- /dev/null +++ b/packages/compiler/src/frontend/analysis.ts @@ -0,0 +1,90 @@ +import type { AnalyzeOptions, AnalyzeResult } from "../compile-types.js"; +import { loadFfiProfile, type FfiProfile } from "../ffi/ffi-manifest.js"; +import { provenanceSources } from "./provenance-registry.js"; +import type { FrontendFactory } from "./pipeline.js"; + +/** Analysis without codegen: how much of the program compiles statically. + * Unlike compile(), lowering diagnostics are data here, not failure. */ +export function analyzeWithFrontend(entryPath: string, opts: AnalyzeOptions, buildPlatform: string, createFrontend: FrontendFactory): AnalyzeResult { + let ffi: FfiProfile | null = null; + if (opts.ffiProfilePath !== undefined) { + const loaded = loadFfiProfile(opts.ffiProfilePath); + if (!loaded.ok) { + return { + coverage: { + file: entryPath, + dynamic: opts.dynamic ?? false, + stats: { statementsTotal: 0, statementsFailed: 0, statementsIsland: 0, functionsSkipped: 0 }, + diagnostics: loaded.diagnostics, + preflightFailed: true, + }, + sourceTexts: new Map(), + }; + } + ffi = loaded.profile; + } + const fe = createFrontend(entryPath, opts.npmStatic, opts.externalTypes); + try { + const emptyStats = { statementsTotal: 0, statementsFailed: 0, statementsIsland: 0, functionsSkipped: 0 }; + + const preflight = fe.preflight; + // Import-FORM fences don't stop the analysis: the module graph is still + // computable (a fenced import contributes no edges), the imported + // bindings poison at their use sites, and the fences join the blockers + // list beside statement-level ones — the report shows a statement + // percentage instead of stopping at the import lines. Everything else — + // tsc errors, config incompatibilities, circular imports — still stops + // at preflight (no trustworthy program to lower). Builds are unchanged: + // compile() fails on every preflight diagnostic exactly as before. + const IMPORT_FENCES = new Set(["SC1010", "SC1012", "SC1013", "SC1014", "SC1015"]); + if (preflight.some((d) => !IMPORT_FENCES.has(d.code))) { + return { + coverage: { + file: entryPath, + dynamic: opts.dynamic ?? false, + stats: emptyStats, + diagnostics: preflight, + ...(fe.npmStatic.length > 0 ? { npmStatic: fe.npmStatic } : {}), + preflightFailed: true, + }, + sourceTexts: fe.sourceTexts(), + }; + } + // Coverage is whole-program by design: builds stop at what the entry + // reaches, but the analysis additionally lowers the unreached remainder + // (throwaway) so the report covers everything the source declares — with + // the unreached share in its own group. + const lowered = fe.lower({ + dynamic: opts.dynamic ?? false, + coverage: true, + targetPlatform: buildPlatform, + ...(ffi !== null ? { ffiImports: ffi.functions } : {}), + }); + const provenance = provenanceSources(); + return { + coverage: { + file: entryPath, + dynamic: opts.dynamic ?? false, + stats: lowered.stats, + // The import fences report as blockers alongside the statement-level + // ones (use sites of the fenced bindings emit matching diagnostics, + // which the report groups with these). + diagnostics: [...preflight, ...lowered.diagnostics], + ...(lowered.runtimeFences.length > 0 ? { runtimeFences: lowered.runtimeFences } : {}), + ...(lowered.unreached ? { unreached: lowered.unreached } : {}), + ...(lowered.npmBuiltins ? { npmBuiltins: lowered.npmBuiltins } : {}), + ...(lowered.npmLazyTraps ? { npmLazyTraps: lowered.npmLazyTraps } : {}), + ...(fe.npmStatic.length > 0 ? { npmStatic: fe.npmStatic } : {}), + // --provenance-sources: the per-package attribution inputs (the + // report aggregates statsByFile under each package's source dir). + ...(provenance !== null ? { provenance } : {}), + ...(lowered.statsByFile ? { statsByFile: lowered.statsByFile } : {}), + ...(lowered.provenanceElided ? { provenanceElided: lowered.provenanceElided } : {}), + preflightFailed: false, + }, + sourceTexts: fe.sourceTexts(), + }; + } finally { + fe.dispose(); + } +} diff --git a/packages/compiler/src/frontend/dts-paths.ts b/packages/compiler/src/frontend/dts-paths.ts index 34cbf4b5..79113b56 100644 --- a/packages/compiler/src/frontend/dts-paths.ts +++ b/packages/compiler/src/frontend/dts-paths.ts @@ -1,16 +1,29 @@ import { createRequire } from "node:module"; +import { join } from "node:path"; import { npmPackageNameOf } from "./workspace-registry.js"; import { tsgoPath } from "./ts7/session-path.js"; export { tsgoPath } from "./ts7/session-path.js"; const require = createRequire(import.meta.url); +let installedDeclarationRoot: string | null = null; + +/** A native distribution supplies its relocatable declaration directory. */ +export function setDeclarationRoot(root: string): void { installedDeclarationRoot = root; } + +function declarationPath(name: string): string { + return tsgoPath(installedDeclarationRoot === null + ? name === "scriptc.d.ts" ? require.resolve("@scriptc/compiler/scriptc.d.ts") + : name === "scriptc-overrides.d.ts" ? require.resolve("@scriptc/compiler/scriptc-overrides.d.ts") + : require.resolve("@scriptc/compiler/scriptc-node-fallback.d.ts") + : join(installedDeclarationRoot, name)); +} /** Path of the shipped ambient declarations — the always-shipped CORE * (comptime/__island_eval, setTimeout). Part of EVERY program scriptc * builds, the project-world preflight program included. */ export function ambientDtsPath(): string { - return tsgoPath(require.resolve("@scriptc/compiler/scriptc.d.ts")); + return declarationPath("scriptc.d.ts"); } /** Path of the shipped divergence/precision OVERRIDES (JSON.parse(): @@ -19,7 +32,7 @@ export function ambientDtsPath(): string { * so a project that typechecks under its own tsc never fails preflight over * an override-manufactured error (checkPreflight). */ export function overridesDtsPath(): string { - return tsgoPath(require.resolve("@scriptc/compiler/scriptc-overrides.d.ts")); + return declarationPath("scriptc-overrides.d.ts"); } /** Path of the shipped FALLBACK declarations (console, process, node:fs) — @@ -27,7 +40,7 @@ export function overridesDtsPath(): string { * With @types/node, the project's real Node types stand in and this file * stands down (its declaration forms would collide). */ export function fallbackDtsPath(): string { - return tsgoPath(require.resolve("@scriptc/compiler/scriptc-node-fallback.d.ts")); + return declarationPath("scriptc-node-fallback.d.ts"); } /** True for files belonging to the adopted Node type surface: the diff --git a/packages/compiler/src/frontend/input-tracker.test.ts b/packages/compiler/src/frontend/input-tracker.test.ts index 59f5ef98..085f443d 100644 --- a/packages/compiler/src/frontend/input-tracker.test.ts +++ b/packages/compiler/src/frontend/input-tracker.test.ts @@ -176,3 +176,27 @@ test("failed directory enumeration invalidates when access is restored", async ( await chmod(packages, 0o700); } }); + +test("synchronous tracking restores parents after throws and propagates unstable inputs", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-inputs-")); + scratch.push(dir); + const first = join(dir, "first.ts"); + const second = join(dir, "second.ts"); + await writeFile(first, "export const first = 1;"); + await writeFile(second, "export const second = 2;"); + const parent = new FrontendInputTracker(); + const child = new FrontendInputTracker(); + parent.runSynchronous(() => { + expect(() => child.runSynchronous(() => { + trackedReadFile(first); + markFrontendInputsUnstable(); + throw new Error("interrupted"); + })).toThrow("interrupted"); + trackedReadFile(second); + }); + expect(parent.snapshot().probes.filter((probe) => probe.op === "file").map((probe) => probe.path)).toEqual([first, second]); + expect(parent.snapshot().stable).toBe(false); + const before = parent.snapshot(); + trackedFileExists(join(dir, "outside.ts")); + expect(parent.snapshot()).toEqual(before); +}); diff --git a/packages/compiler/src/frontend/input-tracker.ts b/packages/compiler/src/frontend/input-tracker.ts index 685e6768..ab18b373 100644 --- a/packages/compiler/src/frontend/input-tracker.ts +++ b/packages/compiler/src/frontend/input-tracker.ts @@ -41,8 +41,8 @@ export interface FrontendSemanticMatch { * named artifacts and a generated directory's formerly-missing observation * are excluded. */ export interface FrontendInputExclusions { - outputPaths?: Iterable; - outputDirectories?: Iterable; + outputPaths?: readonly string[]; + outputDirectories?: readonly string[]; } function frontendSourceDigest(text: string): string { @@ -73,6 +73,7 @@ function systemRealpath(path: string): string { } const activeTracker = new AsyncLocalStorage(); +let synchronousTracker: FrontendInputTracker | undefined; export class FrontendInputTracker { private readonly probes = new Map(); @@ -88,6 +89,23 @@ export class FrontendInputTracker { }); } + /** The native frontend is synchronous; its typed result need not cross an + * async-context callback boundary. Nested trackers still propagate probes. */ + runSynchronous(fn: () => T): T { + const parent = synchronousTracker; + // Install the active tracker so filesystem callbacks can record probes. + // eslint-disable-next-line @typescript-eslint/no-this-alias + synchronousTracker = this; + try { return fn(); } + finally { + synchronousTracker = parent; + if (parent !== undefined && parent !== this) { + for (const probe of this.probes.values()) parent.record(probe); + if (!this.stable) parent.markUnstable(); + } + } + } + record(probe: FrontendInputProbe): void { const key = `${probe.op}\0${probe.path}`; const previous = this.probes.get(key); @@ -140,13 +158,13 @@ export class FrontendInputTracker { } function record(probe: FrontendInputProbe): void { - activeTracker.getStore()?.record(probe); + (synchronousTracker ?? activeTracker.getStore())?.record(probe); } /** Mark the active frontend as unsafe to persist because an exact host query * bypassed the tracked filesystem wrappers. No-op outside a frontend run. */ export function markFrontendInputsUnstable(): void { - activeTracker.getStore()?.markUnstable(); + (synchronousTracker ?? activeTracker.getStore())?.markUnstable(); } export function trackedReadFile(path: string): string | null { @@ -411,11 +429,11 @@ export function frontendInputsSemanticallyMatch( /** Pure validator used by the persistent-cache reader before any path probes. */ export function validFrontendInputSnapshot(snapshot: unknown): snapshot is FrontendInputSnapshot { if (snapshot === null || typeof snapshot !== "object") return false; - const candidate = snapshot as Partial; + const candidate = snapshot as { version?: unknown; stable?: unknown; probes?: unknown }; if (candidate.version !== 1 || candidate.stable !== true || !Array.isArray(candidate.probes)) return false; return candidate.probes.every((probe) => { if (probe === null || typeof probe !== "object") return false; - const value = probe as Partial; + const value = probe as Record; if (typeof value.path !== "string" || typeof value.op !== "string") return false; switch (value.op) { case "file": diff --git a/packages/compiler/src/frontend/lowering/array-indexed-mutation.ts b/packages/compiler/src/frontend/lowering/array-indexed-mutation.ts index e2319323..19bd6ab9 100644 --- a/packages/compiler/src/frontend/lowering/array-indexed-mutation.ts +++ b/packages/compiler/src/frontend/lowering/array-indexed-mutation.ts @@ -1,4 +1,4 @@ -import { BOOL, F64, type IrExpr, type IrStmt, type IrType, type SrcLoc } from "../../ir/ir.js"; +import { BOOL, DYN, F64, NULL_T, type IrExpr, type IrStmt, type IrType, type SrcLoc } from "../../ir/ir.js"; import { numLit, varRef } from "../../ir/build.js"; import { typeKey } from "../type-mapper.js"; import type { Lowerer } from "./lowerer.js"; @@ -40,8 +40,12 @@ export function lowerArrayFill( arrType: IrType & { kind: "array" }, loc: SrcLoc, ): IrExpr { - const valueType = writeUndefined ? F64 : value!.type; - const key = "indexed:fill:" + typeKey(arrType.elem) + ":" + typeKey(valueType) + ":" + writeUndefined; + const writeNull = value?.type.kind === "nullT"; + // Unit literals have no parameter ABI. Pass a numeric placeholder and + // synthesize the unit inside the helper's correctly tagged store. + const discardValue = writeUndefined || writeNull; + const valueType = discardValue ? F64 : value!.type; + const key = "indexed:fill:" + typeKey(arrType.elem) + ":" + typeKey(valueType) + ":" + writeUndefined + ":" + writeNull; let name = lowerer.arrHofHelpers.get(key); if (!name) { name = "%arr.fill." + lowerer.arrHofHelpers.size; @@ -51,7 +55,7 @@ export function lowerArrayFill( const i = varRef("i.0", F64, loc); const body: IrStmt[] = writeUndefined ? [{ kind: "arraySetUndefined", arr: a, index: i, loc }] - : [arrayValueStore(lowerer, a, i, varRef("v.0", valueType, loc), arrType.elem, loc)]; + : [arrayValueStore(lowerer, a, i, writeNull ? { kind: "unitLit", unit: "null", type: NULL_T, loc } : varRef("v.0", valueType, loc), arrType.elem, loc)]; lowerer.liftedFns.push({ name, params: [ @@ -88,8 +92,9 @@ export function lowerArrayFill( loc, }); } - const valueArg: IrExpr = value && writeUndefined - ? { kind: "seqExpr", stmts: [{ kind: "exprStmt", expr: value, loc }], result: numLit(0, loc), type: F64, loc } + const valueArg: IrExpr = discardValue + ? value === null || value.kind === "unitLit" ? numLit(0, loc) + : { kind: "seqExpr", stmts: [{ kind: "exprStmt", expr: lowerer.coerceToExpected(value, DYN), loc }], result: numLit(0, loc), type: F64, loc } : value ?? numLit(0, loc); return { kind: "call", callee: name, args: [receiver, valueArg, start, end], type: arrType, loc }; } diff --git a/packages/compiler/src/frontend/lowering/lower-builtins.ts b/packages/compiler/src/frontend/lowering/lower-builtins.ts index a720faf4..ae946d25 100644 --- a/packages/compiler/src/frontend/lowering/lower-builtins.ts +++ b/packages/compiler/src/frontend/lowering/lower-builtins.ts @@ -2295,6 +2295,12 @@ function lowerFsSyncBufferWindow( } if (bi.module === "fs" && (bi.member === "writeFileSync" || bi.member === "appendFileSync") && expr.arguments.length === 2) { const dataIr = lowerer.mapTypeOf(lowerer.typeOf(expr.arguments[1]!)); + if (dataIr?.kind === "union") { + const arms = lowerer.unions.get(dataIr.unionId)?.arms; + if (arms && arms.every((arm) => arm.kind === "string" || (arm.kind === "bytes" && arm.elem === "u8"))) { + return lowerStringOrBytesWrite(lowerer, expr, dataIr, arms, bi.member === "appendFileSync"); + } + } if (dataIr?.kind === "bytes") { if (dataIr.elem !== "u8") { lowerer.noLowering( @@ -4987,17 +4993,49 @@ export function lowerForkCall(lowerer: Lowerer, expr: ts.CallExpression, loc: Sr ); } const node: IrExpr = { kind: "jsonStringify", value, type: STRING, loc }; - if (indent !== "") { - // The compile-time-resolved indent rides as an extra property (the - // node shape in ir/ir.ts is unchanged); the backend re-indents - // the compact serializer output with Node's gap algorithm. - (node as { indent?: string }).indent = indent; - } + if (indent !== "") node.indent = indent; return node; } return null; // unknown members are tsc errors before lowering } +/** A valid string/byte overload must select its runtime entry from the live + * union tag. Coercing the whole argument to the string overload loses bytes. */ +function lowerStringOrBytesWrite(lowerer: Lowerer, call: ts.CallExpression, + dataType: Extract, arms: readonly IrType[], append: boolean): IrExpr { + const loc = locOf(call); + const key = `fs.${append ? "append" : "write"}:${dataType.unionId}`; + let helper = lowerer.widthHelpers.get(key); + if (!helper) { + helper = `%fs.writeData.${lowerer.widthHelpers.size}`; + lowerer.widthHelpers.set(key, helper); + const body: IrStmt[] = []; + for (let tag = 0; tag < arms.length; tag++) { + const arm = arms[tag]!; + const fn: IrLibFn = arm.kind === "bytes" + ? append ? "fs.appendFileSyncBytes" : "fs.writeFileSyncBytes" + : append ? "fs.appendFileSync" : "fs.writeFileSync"; + body.push({ kind: "if", loc, + cond: { kind: "unionIsTag", unionId: dataType.unionId, tag, negated: false, + value: varRef("data.0", dataType, loc), type: BOOL, loc }, + then: [ + { kind: "exprStmt", expr: { kind: "libCall", fn, args: [varRef("path.0", STRING, loc), + { kind: "unionNarrow", unionId: dataType.unionId, tag, value: varRef("data.0", dataType, loc), type: arm, loc }], type: VOID, loc }, loc }, + { kind: "return", value: null, loc }, + ], else_: null, + }); + } + body.push({ kind: "return", value: null, loc }); + lowerer.liftedFns.push({ name: helper, returnType: VOID, loc, + params: [{ localId: "path.0", name: "path", type: STRING }, { localId: "data.0", name: "data", type: dataType }], + locals: [{ id: "path.0", name: "path", type: STRING, mutable: false }, { id: "data.0", name: "data", type: dataType, mutable: false }], + body, + }); + } + return { kind: "call", callee: helper, args: [lowerer.lowerExprExpecting(call.arguments[0]!, STRING), + lowerer.lowerExprExpecting(call.arguments[1]!, dataType)], type: VOID, loc }; +} + function lowerOptionalStringifyRoot(lowerer: Lowerer, value: IrExpr, indent: string, loc: SrcLoc): IrExpr | null { const tags = optionalStringTags(lowerer, value.type); if (!tags || value.type.kind !== "union") return null; @@ -5014,7 +5052,7 @@ function lowerOptionalStringifyRoot(lowerer: Lowerer, value: IrExpr, indent: str type: STRING, loc, }; - if (indent !== "") (serialized as { indent?: string }).indent = indent; + if (indent !== "") serialized.indent = indent; const missing = lowerer.wrappedUndefined(resultT, loc); if (!missing) throw new InternalCompilerError("optional JSON.stringify result needs an undefined arm"); lowerer.liftedFns.push({ @@ -6129,7 +6167,11 @@ function lowerOptionalStringSearchParams(lowerer: Lowerer, init: IrExpr, loc: Sr lowerer.noLowering(`${receiverType.kind === "cryptoHash" ? "Hash" : "Hmac"}.update with ${call.arguments.length} arguments`, call, "update(stringOrBuffer[, inputEncoding]) is supported"); } const dataNode = call.arguments[0]!; - const data = lowerer.lowerExpr(dataNode); + const dataType = lowerer.mapTypeOf(lowerer.typeOf(dataNode)); + // Indexed reads can retain optional storage after a nullish fallback + // or narrowing. Use the proven argument type through a checked coercion. + const data = dataType?.kind === "string" || dataType?.kind === "bytes" + ? lowerer.lowerExprExpecting(dataNode, dataType) : lowerer.lowerExpr(dataNode); const prefix = receiverType.kind === "cryptoHash" ? "crypto.hashUpdate" : "crypto.hmacUpdate"; if (data.type.kind === "bytes" && data.type.elem === "u8") { if (call.arguments.length !== 1) { diff --git a/packages/compiler/src/frontend/lowering/lower-calls.ts b/packages/compiler/src/frontend/lowering/lower-calls.ts index 4441021a..46638f6a 100644 --- a/packages/compiler/src/frontend/lowering/lower-calls.ts +++ b/packages/compiler/src/frontend/lowering/lower-calls.ts @@ -1,3 +1,4 @@ +import { objectEnumerationReceiver } from "./object-enumeration-receiver.js"; import { InternalCompilerError } from "../../errors.js"; /* Call lowering: the lowerCall dispatch chain, parameter-shape analysis and * argument completion (optional/default/rest, explicit-undefined ≡ omission), @@ -9576,7 +9577,7 @@ export function lowerPromiseMethodCall(lowerer: Lowerer, call: ts.CallExpression const loc = locOf(call); const resultT = lowerer.irTypeOf(call); if (resultT.kind !== "array") lowerer.badType(call, lowerer.typeOf(call)); // defensive - const receiver = lowerer.lowerExpr(argNode); + const receiver = objectEnumerationReceiver(lowerer, lowerer.lowerExpr(argNode), argIr, loc); if (member === "keys") { // The keys walk is shared with for-in (which iterates exactly the // keys Object.keys answers — one construction, one intern key). diff --git a/packages/compiler/src/frontend/lowering/lower-containers.ts b/packages/compiler/src/frontend/lowering/lower-containers.ts index 5f81e9fb..15d84a66 100644 --- a/packages/compiler/src/frontend/lowering/lower-containers.ts +++ b/packages/compiler/src/frontend/lowering/lower-containers.ts @@ -1,3 +1,4 @@ +import { objectEnumerationReceiver } from "./object-enumeration-receiver.js"; import { InternalCompilerError } from "../../errors.js"; /* Container-surface call lowering: array methods (including the HOF family * map/filter/forEach with their synthesized helper functions), Map/Set @@ -5958,7 +5959,8 @@ function mapFromSeedValue(lowerer: Lowerer, seed: IrExpr, mapT: IrType & { kind: shape: IrRecordShape,): IrExpr { const resultT = lowerer.irTypeOf(call); if (resultT.kind !== "array") lowerer.badType(call, lowerer.typeOf(call)); // defensive - return objectIterOverIndexShape(lowerer, call, member, argIr, shape, lowerer.lowerExpr(call.arguments[0]!), resultT, locOf(call)); + return objectIterOverIndexShape(lowerer, call, member, argIr, shape, + objectEnumerationReceiver(lowerer, lowerer.lowerExpr(call.arguments[0]!), argIr, locOf(call)), resultT, locOf(call)); } /** The construction core, receiver/result pre-resolved — `node` anchors diff --git a/packages/compiler/src/frontend/lowering/lower-exprs.ts b/packages/compiler/src/frontend/lowering/lower-exprs.ts index 4afad556..8147acda 100644 --- a/packages/compiler/src/frontend/lowering/lower-exprs.ts +++ b/packages/compiler/src/frontend/lowering/lower-exprs.ts @@ -5791,7 +5791,7 @@ export function lowerElementCompound(lowerer: Lowerer, expr: ts.BinaryExpression if (arr.type.kind !== "array") { lowerer.unsupported("SC1090", target.expression, "assignment through a possibly missing nested array receiver"); } - const index = lowerer.lowerExpr(target.argumentExpression); + const index = lowerOptionalNumber(lowerer, lowerer.lowerExpr(target.argumentExpression), locOf(target.argumentExpression), target.argumentExpression); if (index.type.kind !== "f64") { lowerer.unsupported("SC1090", target.argumentExpression, "indexing with non-number keys"); } diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index ccc22169..8ab24452 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -1,4 +1,5 @@ import { everyExprChild, everyStmtChild } from "../../ir/traverse.js"; +import { sanitizeUnregisteredClassTypes } from "./sanitize-class-types.js"; import { buildUnionNarrow } from "./union-narrow.js"; import { planUnionRetag, buildUnionRetag, planRecordUnionWrap, buildRecordUnionWrap } from "./union-retag.js"; import type { WidthLift } from "./width-lift.js"; @@ -3817,9 +3818,6 @@ export class Lowerer { // slots), uniformly across params/locals/globals/fields/body types so // every producer and consumer agrees. Programs with no unregistered // reference are untouched — byte-stability holds. - if (this.diags.length === 0) { - this.sanitizeUnregisteredClassTypes([functions, this.globalsList, artifacts.classes, artifacts.records, artifacts.unions]); - } const module: IrModule | null = this.diags.length > 0 ? null @@ -3835,6 +3833,7 @@ export class Lowerer { entry: ENTRY_NAME, ...(this.ffiImports.length > 0 ? { ffiImports: [...this.ffiImports] } : {}), }; + if (module) sanitizeUnregisteredClassTypes(module, (name) => this.classes.has(name)); return { module, diagnostics: this.diags, @@ -3847,40 +3846,6 @@ export class Lowerer { }; } - /** The unregistered-class type sweep (run()'s last step before the - * module assembles): every `{kind:"object"}` TYPE naming a class with - * no registered ClassInfo is rewritten IN PLACE to the f64 dummy. - * classval types are exempt (they emit the class-independent - * `ScrClassObj *` — inert-but-valid storage, the validator's own - * stance), and only type objects rewrite — node-level classNames - * (`new`, upcasts) cannot reach here (their lowerings fence without a - * registered class), so the validator still backstops those. */ - sanitizeUnregisteredClassTypes(roots: unknown[]): void { - const isUnregisteredObjectType = (v: unknown): boolean => - typeof v === "object" && v !== null && - (v as { kind?: unknown }).kind === "object" && - typeof (v as { className?: unknown }).className === "string" && - !this.classes.has((v as { className: string }).className); - const sweep = (node: unknown): void => { - if (node === null || typeof node !== "object") return; - if (Array.isArray(node)) { - node.forEach((item, i) => { - if (isUnregisteredObjectType(item)) node[i] = F64; - else sweep(item); - }); - return; - } - const rec = node as Record; - for (const key of Object.keys(rec)) { - if (key === "loc") continue; - const v = rec[key]; - if (isUnregisteredObjectType(v)) rec[key] = F64; - else sweep(v); - } - }; - for (const root of roots) sweep(root); - } - /** True when `t` (recursively) names a class instance type with no * registered ClassInfo — the shape of a JS class whose collection fenced. * Used by run()'s global pruning; shapes/unions recurse with a seen-set @@ -7812,6 +7777,17 @@ export class Lowerer { * lowering goes through here (via lowerExprExpecting) or calls this * directly when the expression was already lowered. */ coerceInto(node: ts.Node, expr: IrExpr, expected: IrType): IrExpr { + // A union destination can accept the binding's tagged storage directly. + // The checker's single-arm flow type can be stale after a callback writes + // the binding. Extracting that arm and wrapping it again would discard the + // actual tag and could read a different record layout. Retag the stored + // value instead; ordinary coercion checks any excluded destination arms. + // Explicit assertions keep their own conversion and validation. + if (expected.kind === "union" && expr.kind === "unionNarrow") { + let source = node; + while (ts.isParenthesizedExpression(source) || ts.isSatisfiesExpression(source)) source = source.expression; + if (ts.isIdentifier(source) || ts.isShorthandPropertyAssignment(source)) expr = expr.value; + } // A fresh literal can retain a wider runtime-optional field after its // initial contextual layout was chosen. Its known discriminator still // selects the destination arm; validate that payload before wrapping. diff --git a/packages/compiler/src/frontend/lowering/object-enumeration-receiver.ts b/packages/compiler/src/frontend/lowering/object-enumeration-receiver.ts new file mode 100644 index 00000000..d3e596a9 --- /dev/null +++ b/packages/compiler/src/frontend/lowering/object-enumeration-receiver.ts @@ -0,0 +1,34 @@ +import { BOOL, STRING, type IrExpr, type IrType, type SrcLoc, isUnitType, typeEquals } from "../../ir/ir.js"; +import { varRef } from "../../ir/build.js"; +import type { Lowerer } from "./lowerer.js"; + +/** Array/property reads retain a missing-value arm even when the checker + * promises a record. Object enumeration must check it before reading fields. */ +export function objectEnumerationReceiver( + lowerer: Lowerer, receiver: IrExpr, expected: IrType & { kind: "record" }, loc: SrcLoc, +): IrExpr { + if (receiver.type.kind !== "union") return receiver; + const source = receiver.type; + const arms = lowerer.unions.get(source.unionId)?.arms; + const tag = lowerer.armTag(source.unionId, expected); + if (tag < 0 || !arms?.every((arm) => isUnitType(arm) || typeEquals(arm, expected))) return receiver; + const key = `obj.receiver:${source.unionId}:${expected.shapeId}`; + let name = lowerer.arrHofHelpers.get(key); + if (name === undefined) { + name = `%obj.receiver.${lowerer.arrHofHelpers.size}`; + lowerer.arrHofHelpers.set(key, name); + const value = varRef("value.0", source, loc); + lowerer.liftedFns.push({ + name, params: [{ localId: "value.0", name: "value", type: source }], + returnType: expected, locals: [{ id: "value.0", name: "value", type: source, mutable: false }], + body: [{ + kind: "if", cond: { kind: "unionIsTag", unionId: source.unionId, tag, value, negated: false, type: BOOL, loc }, + then: [{ kind: "return", value: { kind: "unionNarrow", unionId: source.unionId, tag, value, type: expected, loc }, loc }], + else_: [{ kind: "throw", value: { kind: "libCall", fn: "error.new", + args: [{ kind: "strLit", value: "Cannot convert undefined or null to object", type: STRING, loc }], + type: { kind: "object", className: "%TypeError" }, loc }, loc }], loc, + }], loc, + }); + } + return { kind: "call", callee: name, args: [receiver], type: expected, loc }; +} diff --git a/packages/compiler/src/frontend/lowering/sanitize-class-types.test.ts b/packages/compiler/src/frontend/lowering/sanitize-class-types.test.ts new file mode 100644 index 00000000..ad7c2389 --- /dev/null +++ b/packages/compiler/src/frontend/lowering/sanitize-class-types.test.ts @@ -0,0 +1,61 @@ +import { expect, test } from "vitest"; +import { F64, type IrExpr, type IrModule, type IrType } from "../../ir/ir.js"; +import { sanitizeUnregisteredClassTypes } from "./sanitize-class-types.js"; + +test("fenced instance slots are erased throughout signatures, storage and nested expressions", () => { + const loc = { file: "classes.js", start: 0, end: 1 }; + const missing: IrType = { kind: "object", className: "Fenced" }; + const kept: IrType = { kind: "object", className: "Registered" }; + const constructor: IrType = { kind: "classval", className: "Fenced" }; + const composite: IrType = { kind: "func", rest: true, restAbi: "typed", argumentsAll: true, + params: [{ kind: "array", elem: missing }, { kind: "set", elem: missing }, + { kind: "map", key: missing, value: { kind: "promise", inner: missing } }], + ret: { kind: "generator", async: true, yieldT: missing, retT: missing, nextT: missing } }; + const ref = (type: IrType): IrExpr => ({ kind: "varRef", localId: "x", type, loc }); + const module: IrModule = { + irVersion: 13, sourceFile: loc.file, entry: "main", + globals: [{ id: "g", name: "g", type: composite, mutable: false }], + classes: [{ name: "Registered", fields: [{ name: "value", type: missing }], + localCaptures: [{ localId: "x", name: "x", type: missing }], loc }], + records: [{ id: "r", fields: [{ name: "value", type: missing }, + { name: "next", type: { kind: "union", unionId: "u" } }], indexValue: missing }], + unions: [{ id: "u", arms: [missing, { kind: "record", shapeId: "r" }, kept, constructor] }], + functions: [{ name: "main", returnType: missing, loc, + params: [{ localId: "x", name: "x", type: missing }], + locals: [{ id: "x", name: "x", type: missing, mutable: false }], + captures: [{ localId: "y", name: "y", type: missing }], + classCaptures: [{ localId: "z", name: "z", type: missing, slot: 3 }], + generator: { yieldT: missing, nextT: missing, resultType: { kind: "record", shapeId: "r" } }, + body: [{ kind: "if", cond: ref(kept), else_: null, loc, then: [{ kind: "exprStmt", loc, + expr: { kind: "seqExpr", type: missing, loc, + stmts: [{ kind: "exprStmt", expr: ref(composite), loc }], + result: { kind: "new", className: "Fenced", args: [ref(missing)], type: missing, loc } } }] }], + }], + }; + // An independent structural oracle checks every nested slot and leaves + // class-value types and node-level names intact, even on invalid nodes. + const expected: unknown = JSON.parse(JSON.stringify(module, (_key, value: unknown) => { + const type = value as { kind?: string; className?: string } | null; + return type?.kind === "object" && type.className === "Fenced" ? F64 : value; + })); + sanitizeUnregisteredClassTypes(module, (name) => name === "Registered"); + expect(module).toEqual(expected); + expect(module.functions[0]!.params[0]!.type).toBe(F64); + expect(module.unions![0]!.arms[2]).toBe(kept); + expect(module.unions![0]!.arms[3]).toBe(constructor); + expect(missing).toEqual({ kind: "object", className: "Fenced" }); + sanitizeUnregisteredClassTypes(module, (name) => name === "Registered"); + expect(module).toEqual(expected); +}); + +test("modules without optional tables and registered class types retain their structure", () => { + const loc = { file: "registered.ts", start: 0, end: 1 }; + const type: IrType = { kind: "object", className: "Registered" }; + const module: IrModule = { irVersion: 13, sourceFile: loc.file, entry: "main", functions: [{ + name: "main", returnType: type, params: [], locals: [], body: [], loc, + }] }; + const before = structuredClone(module); + sanitizeUnregisteredClassTypes(module, () => true); + expect(module).toEqual(before); + expect(module.functions[0]!.returnType).toBe(type); +}); diff --git a/packages/compiler/src/frontend/lowering/sanitize-class-types.ts b/packages/compiler/src/frontend/lowering/sanitize-class-types.ts new file mode 100644 index 00000000..d90c5147 --- /dev/null +++ b/packages/compiler/src/frontend/lowering/sanitize-class-types.ts @@ -0,0 +1,68 @@ +import { F64, type IrModule, type IrType } from "../../ir/ir.js"; +import { everyStmtList } from "../../ir/traverse.js"; + +/** Replace inert instance slots left by runtime-fenced class declarations. + * Construction of an unregistered class already traps, but its remaining + * storage types must agree across declarations, signatures and expressions. + * Keep class values and node-level class names for the validator to check. + * Traverse typed IR directly: boxing a whole module into unknown recursively + * copies its records and makes this pass costly in the native compiler. */ +export function sanitizeUnregisteredClassTypes(module: IrModule, hasClass: (name: string) => boolean): void { + const rewrite = (type: IrType): IrType => { + switch (type.kind) { + case "object": + return hasClass(type.className) ? type : F64; + case "array": + case "set": + type.elem = rewrite(type.elem); + break; + case "map": + type.key = rewrite(type.key); + type.value = rewrite(type.value); + break; + case "func": + for (let i = 0; i < type.params.length; i++) type.params[i] = rewrite(type.params[i]!); + type.ret = rewrite(type.ret); + break; + case "promise": + type.inner = rewrite(type.inner); + break; + case "generator": + type.yieldT = rewrite(type.yieldT); + type.retT = rewrite(type.retT); + type.nextT = rewrite(type.nextT); + break; + } + // Record and union references use IDs. Visit each definition below, + // without following those edges through recursive type graphs. + return type; + }; + for (const global of module.globals ?? []) global.type = rewrite(global.type); + for (const cls of module.classes ?? []) { + for (const field of cls.fields) field.type = rewrite(field.type); + for (const capture of cls.localCaptures ?? []) capture.type = rewrite(capture.type); + } + for (const record of module.records ?? []) { + for (const field of record.fields) field.type = rewrite(field.type); + if (record.indexValue) record.indexValue = rewrite(record.indexValue); + } + for (const union of module.unions ?? []) { + for (let i = 0; i < union.arms.length; i++) union.arms[i] = rewrite(union.arms[i]!); + } + for (const fn of module.functions) { + fn.returnType = rewrite(fn.returnType); + for (const param of fn.params) param.type = rewrite(param.type); + for (const local of fn.locals) local.type = rewrite(local.type); + for (const capture of fn.captures ?? []) capture.type = rewrite(capture.type); + for (const capture of fn.classCaptures ?? []) capture.type = rewrite(capture.type); + if (fn.generator) { + fn.generator.yieldT = rewrite(fn.generator.yieldT); + fn.generator.nextT = rewrite(fn.generator.nextT); + // resultType is a record reference; its fields are visited above. + } + everyStmtList(fn.body, { + expr: (expr) => { expr.type = rewrite(expr.type); return true; }, + stmt: () => true, + }); + } +} diff --git a/packages/compiler/src/frontend/pipeline-native.ts b/packages/compiler/src/frontend/pipeline-native.ts index 453d94ec..9110d869 100644 --- a/packages/compiler/src/frontend/pipeline-native.ts +++ b/packages/compiler/src/frontend/pipeline-native.ts @@ -1,6 +1,6 @@ import { runFrontend, type Frontend } from "./pipeline.js"; import { loadProgram } from "./program.js"; -import { FrontendServices } from "./services.js"; +import { FrontendServices, type ComptimeEvaluator } from "./services.js"; import { createNativeTs7Api } from "./ts7/native-api.js"; /** Native callers supply the installed TS7 executable explicitly. The @@ -11,13 +11,15 @@ export function runNativeFrontend( executable: string, npmStatic?: readonly string[] | "auto" | "lib", externalTypes?: Readonly>, + evaluateComptime?: ComptimeEvaluator, + libraryNpmStatic: readonly string[] = [], ): Frontend { - const services = new FrontendServices((options) => createNativeTs7Api({ ...options, executable })); + const services = new FrontendServices((options) => createNativeTs7Api({ ...options, executable }), process.cwd(), evaluateComptime); try { const frontend = runFrontend(entryPath, (path, options) => loadProgram(path, services, { npmStatic: options.npmStatic ?? [], externalTypes: Object.entries(options.externalTypes ?? {}), - }), npmStatic, externalTypes); + }), npmStatic, externalTypes, libraryNpmStatic); return { ...frontend, dispose: () => { diff --git a/packages/compiler/src/frontend/pipeline.ts b/packages/compiler/src/frontend/pipeline.ts index e7db7d24..1fc52fbc 100644 --- a/packages/compiler/src/frontend/pipeline.ts +++ b/packages/compiler/src/frontend/pipeline.ts @@ -59,6 +59,14 @@ export interface Frontend { dispose: () => void; } +/** Hosts own transport lifetimes; shared compiler stages own the frontend. */ +export type FrontendFactory = ( + entryPath: string, + npmStatic?: readonly string[] | "auto" | "lib", + externalTypes?: Readonly>, + libraryNpmStatic?: readonly string[], +) => Frontend; + /** --npm-static=auto (and library mode's mandatory twin): one throwaway * load finds every bare npm import the program's own modules make, then * the eligibility heuristics (npm-static.ts) pick the packages whose diff --git a/packages/compiler/src/frontend/provenance-core.ts b/packages/compiler/src/frontend/provenance-core.ts new file mode 100644 index 00000000..66ea43ca --- /dev/null +++ b/packages/compiler/src/frontend/provenance-core.ts @@ -0,0 +1,452 @@ +/* `--provenance-sources` — the provenance-assisted static compilation + * pipeline (EXPERIMENTAL prototype; the registry doc in + * provenance-registry.ts states the thesis). + * + * For every bare npm specifier the entry's module graph imports, this + * asks the npm registry for the package's PROVENANCE ATTESTATION + * (GET registry.npmjs.org/-/npm/v1/attestations/@ — the + * SLSA predicate names the exact {repository, commit} the published dist + * was built from), fetches that source tree once (content-addressed cache + * under ~/.cache/scriptc/provenance/), locates the package + * directory inside it (monorepos publish from subdirectories), and maps + * the published entry to its TypeScript source (dist/lib/build targets + * rewritten to their src twins). Mapped packages compile as ordinary + * program modules — real types, real statements, the static frontier — + * and everything that cannot be mapped FALLS BACK to the island path + * with a note; a fallback is never a build failure. + * + * PRODUCTION GAPS, deliberately unbuilt in the prototype: no sigstore + * bundle verification (the attestation is trusted as served), no + * dist-tarball ↔ source build reproduction (the behavior differential is + * the honest check today), source-entry mapping is heuristic (exports + * targets rewritten dist→src), and transitive dependency versions + * resolve from the DRIVER's installed tree rather than the package's own + * lockfile. + * + * Offline/test hook: SCRIPTC_PROVENANCE_MANIFEST= pre-seeds + * {"packages": {"": {"dir": "", "commit"?, "repo"?}}} + * — those packages skip the network entirely (harness fixtures ride + * this); unlisted packages still take the live pipeline. */ + +import { execFileSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, readFileSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { builtinModules } from "node:module"; +import { homedir, tmpdir } from "node:os"; +import { dirname, isAbsolute, join, resolve } from "node:path"; +import { sourceImportsOfFile } from "./module-syntax.js"; +import type { SourceFile } from "./ts7/ast-types.js"; +import type { Ts7SourceKind } from "./ts7/source-parser.js"; +import { resolveExports, resolveRelativeModule } from "./resolve.js"; +import { packageNameOfSpecifier as packageNameOf } from "./workspace-registry.js"; +import type { ProvenancePackageSource, ProvenanceSources } from "./provenance-registry.js"; + +export type ProvenanceParser = (path: string, source: string, kind: Ts7SourceKind) => SourceFile; + +const NODE_IMPORT_CONDITIONS = new Set(["import", "node", "default"]); + +const NODE_BUILTINS = new Set(builtinModules); + +/** Hard cap on packages one compile will source-map (prototype guard). */ +const MAX_PACKAGES = 16; + +function isFile(path: string): boolean { + try { + return statSync(path).isFile(); + } catch { + return false; + } +} + +function isDirectory(path: string): boolean { + try { + return statSync(path).isDirectory(); + } catch { + return false; + } +} + +function readJson(path: string): Record | null { + try { + return JSON.parse(readFileSync(path, "utf8")) as Record; + } catch { + return null; + } +} + +/* ── the bare-import prescan ───────────────────────────────────────────── + * The provenance pipeline runs BEFORE the program loads (tsgo needs the + * "paths" mapping at creation), so the bare specifiers come from a light + * parse walk of the entry's RELATIVE import closure — the same specifier + * collection shapes npm.ts scans embedded modules with, over the shared + * native TypeScript syntax service. */ + +function moduleSpecifiersLite(source: string, fileName: string, parseSourceFile: ProvenanceParser): { spec: string; typeOnly: boolean }[] { + return sourceImportsOfFile(parseSourceFile(fileName, source, "ts")); +} + +/** Every bare (non-relative, non-builtin, non-"#") VALUE specifier the + * relative closure of `roots` imports, in first-encounter order. */ +function bareImportsOf(roots: readonly string[], parseSourceFile: ProvenanceParser): string[] { + const seenFiles = new Set(); + const bare: string[] = []; + const bareSeen = new Set(); + const queue = [...roots]; + while (queue.length > 0) { + const file = resolve(queue.shift()!); + if (seenFiles.has(file)) continue; + seenFiles.add(file); + let text: string; + try { + text = readFileSync(file, "utf8"); + } catch { + continue; + } + for (const { spec, typeOnly } of moduleSpecifiersLite(text, file, parseSourceFile)) { + if (typeOnly) continue; + if (spec.startsWith("./") || spec.startsWith("../")) { + const dep = resolveRelativeModule(file, spec); + if (dep !== null && !dep.endsWith(".json") && !dep.endsWith(".d.ts")) queue.push(dep); + continue; + } + if (spec.startsWith("#") || spec.startsWith("node:")) continue; + if (NODE_BUILTINS.has(packageNameOf(spec))) continue; + if (!bareSeen.has(spec)) { + bareSeen.add(spec); + bare.push(spec); + } + } + } + return bare; +} + +/* ── installed-package lookup (name/version/published entry) ──────────── */ + +interface InstalledPackage { + dir: string; + name: string; + version: string; + pkgJson: Record; +} + +/** node_modules/ walking up from `fromDir`, realpath-free (the + * published package.json is all this needs). */ +function findInstalled(fromDir: string, name: string): InstalledPackage | null { + for (let dir = fromDir; ; ) { + const candidate = join(dir, "node_modules", name); + const pkgJson = readJson(join(candidate, "package.json")); + if (pkgJson !== null && typeof pkgJson["version"] === "string") { + return { + dir: candidate, + name: typeof pkgJson["name"] === "string" ? pkgJson["name"] : name, + version: pkgJson["version"], + pkgJson, + }; + } + const parent = dirname(dir); + if (parent === dir) return null; + dir = parent; + } +} + +/* ── attestation → {repo, commit} ──────────────────────────────────────── */ + +interface Attested { + repo: string; + commit: string; +} + +/** GET the npm attestation set and extract the SLSA provenance + * predicate's resolved source dependency. Throws with a one-line reason + * on every failure shape (no attestation, network, malformed). */ +async function fetchAttestation(name: string, version: string): Promise { + const url = `https://registry.npmjs.org/-/npm/v1/attestations/${encodeURIComponent(`${name}@${version}`).replace(/%40/g, "@").replace(/%2F/gi, "/")}`; + const res = await fetch(url, { signal: AbortSignal.timeout(30_000) }); + if (res.status === 404) throw new Error("no provenance attestation published"); + if (!res.ok) throw new Error(`attestation fetch failed (HTTP ${res.status})`); + const body = (await res.json()) as { attestations?: { predicateType?: string; bundle?: { dsseEnvelope?: { payload?: string } } }[] }; + for (const att of body.attestations ?? []) { + if (!att.predicateType?.startsWith("https://slsa.dev/provenance")) continue; + const payload = att.bundle?.dsseEnvelope?.payload; + if (payload === undefined) continue; + const stmt = JSON.parse(Buffer.from(payload, "base64").toString("utf8")) as { + predicate?: { buildDefinition?: { resolvedDependencies?: { uri?: string; digest?: { gitCommit?: string } }[] } }; + }; + for (const dep of stmt.predicate?.buildDefinition?.resolvedDependencies ?? []) { + const commit = dep.digest?.gitCommit; + if (typeof commit === "string" && commit !== "" && typeof dep.uri === "string") { + return { repo: dep.uri, commit }; + } + } + } + throw new Error("attestation set carries no SLSA provenance predicate"); +} + +/* ── source fetch (content-addressed by the attested commit) ──────────── */ + +function cacheRoot(): string { + return process.env["SCRIPTC_PROVENANCE_CACHE"] ?? join(homedir(), ".cache", "scriptc", "provenance"); +} + +/** The cached source tree for an attested commit, fetching it once from + * codeload (github repos only in the prototype). Returns the tree root. */ +async function fetchSourceTree(repo: string, commit: string): Promise { + const dest = join(cacheRoot(), commit); + if (isDirectory(dest)) return dest; + const m = /github\.com\/([^/]+)\/([^/@#]+)/.exec(repo); + if (m === null) throw new Error(`source repository is not a github URL (${repo})`); + const url = `https://codeload.github.com/${m[1]}/${m[2]}/tar.gz/${commit}`; + const res = await fetch(url, { signal: AbortSignal.timeout(120_000) }); + if (!res.ok) throw new Error(`source fetch failed (HTTP ${res.status} for ${url})`); + const bytes = Buffer.from(await res.arrayBuffer()); + mkdirSync(cacheRoot(), { recursive: true }); + const tmp = mkdtempSync(join(tmpdir(), "scriptc-provenance-")); + try { + const tarball = join(tmp, "src.tgz"); + writeFileSync(tarball, bytes); + const extractDir = join(tmp, "tree"); + mkdirSync(extractDir); + execFileSync("tar", ["-xzf", tarball, "-C", extractDir, "--strip-components=1"]); + try { + renameSync(extractDir, dest); + } catch { + // A parallel compile won the rename — its tree is the same content. + if (!isDirectory(dest)) throw new Error("source cache rename failed"); + } + } finally { + rmSync(tmp, { recursive: true, force: true }); + } + return dest; +} + +/* ── package dir + source-entry mapping ────────────────────────────────── */ + +/** The directory inside `tree` whose package.json names `name`: the root, + * then the conventional monorepo layouts, then a bounded scan. */ +function locatePackageDir(tree: string, name: string): string | null { + const nameOf = (dir: string): string | null => { + const pkg = readJson(join(dir, "package.json")); + return pkg !== null && typeof pkg["name"] === "string" ? pkg["name"] : null; + }; + if (nameOf(tree) === name) return tree; + const bare = name.startsWith("@") ? name.split("/")[1]! : name; + const candidates = [ + join(tree, "packages", bare), + join(tree, "packages", name), + join(tree, "packages", name.replace("@", "").replace("/", "-")), + join(tree, "packages", `babel-${bare}`), + ]; + for (const c of candidates) { + if (nameOf(c) === name) return c; + } + // Bounded breadth-first scan (depth ≤ 3, node_modules/.git skipped). + const queue: { dir: string; depth: number }[] = [{ dir: tree, depth: 0 }]; + while (queue.length > 0) { + const { dir, depth } = queue.shift()!; + if (depth > 0 && nameOf(dir) === name) return dir; + if (depth >= 3) continue; + let entries: string[]; + try { + entries = readdirSync(dir).filter((entry) => isDirectory(join(dir, entry))).sort(); + } catch { + continue; + } + for (const e of entries) { + if (e === "node_modules" || e.startsWith(".")) continue; + queue.push({ dir: join(dir, e), depth: depth + 1 }); + } + } + return null; +} + +/** The published dist target for `subpath` — "exports" with the import + * condition, else module/main/types fields (root subpath only). */ +function publishedTargetOf(pkgJson: Record, subpath: string): string | null { + if (pkgJson["exports"] !== undefined) { + return resolveExports(pkgJson["exports"], subpath, NODE_IMPORT_CONDITIONS); + } + if (subpath !== ".") return subpath; + for (const field of ["module", "main", "types"]) { + const v = pkgJson[field]; + if (typeof v === "string" && v !== "") return v; + } + return "index.js"; +} + +/** dist target → source file, heuristically: the built path's leading + * dist/lib/build segment rewrites to src (or drops), an esm/cjs/dts flavor + * segment is skipped for a shared TypeScript source tree, extensions + * rewrite to TypeScript twins, and the root entry falls back to the + * conventional src/index.ts homes. First existing candidate wins. */ +function mapEntryToSource(pkgDir: string, target: string, subpath: string): string | null { + const rel = target.replace(/^\.\//, ""); + const stems = new Set([rel]); + const distRe = /^(dist|lib|build|out|output|dist-node|dist-src)\//; + if (distRe.test(rel)) { + const withoutDist = rel.replace(distRe, ""); + if (/^(esm|cjs|dts)\//.test(withoutDist)) { + stems.add(`src/${withoutDist.replace(/^(esm|cjs|dts)\//, "")}`); + } + stems.add(rel.replace(distRe, "src/")); + stems.add(withoutDist); + } else { + stems.add(`src/${rel}`); + } + const candidates: string[] = []; + for (const stem of stems) { + const base = stem.replace(/\.d\.(ts|mts|cts)$/, ".$1").replace(/\.(js|mjs|cjs)$/, ""); + if (/\.(ts|tsx|mts|cts)$/.test(base)) candidates.push(base); + else { + candidates.push(`${base}.ts`, `${base}.mts`, `${base}.cts`, `${base}.tsx`); + candidates.push(join(base, "index.ts")); + } + } + if (subpath === ".") { + candidates.push("src/index.ts", "src/index.mts", "index.ts", "src/index.tsx"); + } + for (const c of candidates) { + const abs = join(pkgDir, c); + if (isFile(abs)) return abs; + } + return null; +} + +/* ── the pipeline ─────────────────────────────────────────────────────── */ + +interface ManifestEntry { + dir: string; + commit?: string; + repo?: string; +} + +function readManifest(): Map { + const path = process.env["SCRIPTC_PROVENANCE_MANIFEST"]; + const out = new Map(); + if (path === undefined || path === "") return out; + const manifest = readJson(resolve(path)); + const packages = manifest?.["packages"]; + if (packages === null || typeof packages !== "object") return out; + for (const [name, raw] of Object.entries(packages as Record)) { + if (raw === null || typeof raw !== "object") continue; + const e = raw as { dir?: unknown; commit?: unknown; repo?: unknown }; + if (typeof e.dir !== "string") continue; + const dir = isAbsolute(e.dir) ? e.dir : resolve(dirname(resolve(path)), e.dir); + out.set(name, { + dir, + ...(typeof e.commit === "string" ? { commit: e.commit } : {}), + ...(typeof e.repo === "string" ? { repo: e.repo } : {}), + }); + } + return out; +} + +/** Resolves provenance sources for everything the entry's module graph + * imports (one transitive round per newly-mapped tree: source imports of + * OTHER packages try the pipeline too). Never throws for a package + * failure — those become notes and the package keeps its island path. */ +export async function resolveProvenanceSourcesWithParser(entryPath: string, parseSourceFile: ProvenanceParser): Promise { + const entry = resolve(entryPath); + const manifest = readManifest(); + const packages: ProvenancePackageSource[] = []; + const notes: string[] = []; + /** package name → mapped package (or null after a noted failure). */ + const processed = new Map(); + + /** All specifiers seen so far, grouped by package name. */ + const specifiersByPackage = new Map>(); + const enqueue = (specs: readonly string[]): string[] => { + const newNames: string[] = []; + for (const spec of specs) { + const name = packageNameOf(spec); + let set = specifiersByPackage.get(name); + if (set === undefined) { + specifiersByPackage.set(name, (set = new Set())); + newNames.push(name); + } + set.add(spec); + } + return newNames; + }; + + const mapOne = async (name: string): Promise => { + if (processed.has(name)) return; + if (processed.size >= MAX_PACKAGES) { + processed.set(name, null); + notes.push(`${name}: skipped — provenance package limit (${MAX_PACKAGES}) reached; island path used`); + return; + } + processed.set(name, null); // claimed; overwritten on success + const installed = findInstalled(dirname(entry), name); + if (installed === null) { + notes.push(`${name}: not installed under the entry's node_modules; island path used`); + return; + } + let dir: string; + let repo: string; + let commit: string; + const seeded = manifest.get(name); + try { + if (seeded !== undefined) { + dir = seeded.dir; + repo = seeded.repo ?? "(manifest)"; + commit = seeded.commit ?? "(manifest)"; + if (!isDirectory(dir)) throw new Error(`manifest dir does not exist (${dir})`); + } else { + const attested = await fetchAttestation(installed.name, installed.version); + repo = attested.repo; + commit = attested.commit; + const tree = await fetchSourceTree(repo, commit); + const located = locatePackageDir(tree, installed.name); + if (located === null) { + throw new Error(`package directory not found inside the attested source tree (${tree})`); + } + dir = located; + } + const entries: Record = {}; + for (const spec of specifiersByPackage.get(name) ?? new Set()) { + const parts = spec.split("/"); + const nameLen = spec.startsWith("@") ? 2 : 1; + const subpath = parts.length === nameLen ? "." : `./${parts.slice(nameLen).join("/")}`; + const target = publishedTargetOf(installed.pkgJson, subpath); + const source = target === null ? null : mapEntryToSource(dir, target, subpath); + if (source === null) { + notes.push( + `${name}@${installed.version}: no source mapping for '${spec}' (published target: ${target ?? "unexported"}); island path used`, + ); + continue; + } + entries[spec] = source; + } + if (Object.keys(entries).length === 0) return; + const srcPkg = readJson(join(dir, "package.json")); + const sourceVersion = typeof srcPkg?.["version"] === "string" ? srcPkg["version"] : undefined; + const pkg: ProvenancePackageSource = { + name: installed.name, + version: installed.version, + ...(sourceVersion !== undefined && sourceVersion !== installed.version ? { sourceVersion } : {}), + repo, + commit, + dir, + entries, + }; + if (pkg.sourceVersion !== undefined) { + notes.push( + `${name}: source tree's package.json says ${pkg.sourceVersion}, installed is ${installed.version} (release tooling that bumps at publish) — the behavior differential is the check`, + ); + } + packages.push(pkg); + processed.set(name, pkg); + // One transitive round: the mapped source's own bare imports try + // the pipeline too (versions resolve from the DRIVER's tree — a + // prototype heuristic; production wants the package's lockfile). + const inner = enqueue(bareImportsOf(Object.values(entries), parseSourceFile)); + for (const n of inner) await mapOne(n); + } catch (e) { + notes.push(`${name}@${installed.version}: ${e instanceof Error ? e.message : String(e)}; island path used`); + } + }; + + for (const name of enqueue(bareImportsOf([entry], parseSourceFile))) { + await mapOne(name); + } + return { packages, notes }; +} diff --git a/packages/compiler/src/frontend/provenance.ts b/packages/compiler/src/frontend/provenance.ts index 73c50e05..8defcaf8 100644 --- a/packages/compiler/src/frontend/provenance.ts +++ b/packages/compiler/src/frontend/provenance.ts @@ -1,453 +1,7 @@ -/* `--provenance-sources` — the provenance-assisted static compilation - * pipeline (EXPERIMENTAL prototype; the registry doc in - * provenance-registry.ts states the thesis). - * - * For every bare npm specifier the entry's module graph imports, this - * asks the npm registry for the package's PROVENANCE ATTESTATION - * (GET registry.npmjs.org/-/npm/v1/attestations/@ — the - * SLSA predicate names the exact {repository, commit} the published dist - * was built from), fetches that source tree once (content-addressed cache - * under ~/.cache/scriptc/provenance/), locates the package - * directory inside it (monorepos publish from subdirectories), and maps - * the published entry to its TypeScript source (dist/lib/build targets - * rewritten to their src twins). Mapped packages compile as ordinary - * program modules — real types, real statements, the static frontier — - * and everything that cannot be mapped FALLS BACK to the island path - * with a note; a fallback is never a build failure. - * - * PRODUCTION GAPS, deliberately unbuilt in the prototype: no sigstore - * bundle verification (the attestation is trusted as served), no - * dist-tarball ↔ source build reproduction (the behavior differential is - * the honest check today), source-entry mapping is heuristic (exports - * targets rewritten dist→src), and transitive dependency versions - * resolve from the DRIVER's installed tree rather than the package's own - * lockfile. - * - * Offline/test hook: SCRIPTC_PROVENANCE_MANIFEST= pre-seeds - * {"packages": {"": {"dir": "", "commit"?, "repo"?}}} - * — those packages skip the network entirely (harness fixtures ride - * this); unlisted packages still take the live pipeline. */ - -import { execFile } from "node:child_process"; -import { readFileSync, readdirSync, statSync } from "node:fs"; -import { mkdir, mkdtemp, rename, rm, writeFile } from "node:fs/promises"; -import { builtinModules } from "node:module"; -import { homedir, tmpdir } from "node:os"; -import { dirname, isAbsolute, join, resolve } from "node:path"; -import { promisify } from "node:util"; -import { sourceImportsOfFile } from "./module-syntax.js"; import { parseSourceFile } from "./ts7/source-parser-node.js"; -import { resolveExports, resolveRelativeModule } from "./resolve.js"; -import { packageNameOfSpecifier as packageNameOf } from "./workspace-registry.js"; -import type { ProvenancePackageSource, ProvenanceSources } from "./provenance-registry.js"; +import { resolveProvenanceSourcesWithParser } from "./provenance-core.js"; +import type { ProvenanceSources } from "./provenance-registry.js"; -const NODE_IMPORT_CONDITIONS = new Set(["import", "node", "default"]); - -const execFileAsync = promisify(execFile); - -const NODE_BUILTINS = new Set(builtinModules); - -/** Hard cap on packages one compile will source-map (prototype guard). */ -const MAX_PACKAGES = 16; - -function isFile(path: string): boolean { - try { - return statSync(path).isFile(); - } catch { - return false; - } -} - -function isDirectory(path: string): boolean { - try { - return statSync(path).isDirectory(); - } catch { - return false; - } -} - -function readJson(path: string): Record | null { - try { - return JSON.parse(readFileSync(path, "utf8")) as Record; - } catch { - return null; - } -} - -/* ── the bare-import prescan ───────────────────────────────────────────── - * The provenance pipeline runs BEFORE the program loads (tsgo needs the - * "paths" mapping at creation), so the bare specifiers come from a light - * parse walk of the entry's RELATIVE import closure — the same specifier - * collection shapes npm.ts scans embedded modules with, over the shared - * native TypeScript syntax service. */ - -function moduleSpecifiersLite(source: string, fileName: string): { spec: string; typeOnly: boolean }[] { - return sourceImportsOfFile(parseSourceFile(fileName, source, "ts")); -} - -/** Every bare (non-relative, non-builtin, non-"#") VALUE specifier the - * relative closure of `roots` imports, in first-encounter order. */ -function bareImportsOf(roots: readonly string[]): string[] { - const seenFiles = new Set(); - const bare: string[] = []; - const bareSeen = new Set(); - const queue = [...roots]; - while (queue.length > 0) { - const file = resolve(queue.shift()!); - if (seenFiles.has(file)) continue; - seenFiles.add(file); - let text: string; - try { - text = readFileSync(file, "utf8"); - } catch { - continue; - } - for (const { spec, typeOnly } of moduleSpecifiersLite(text, file)) { - if (typeOnly) continue; - if (spec.startsWith("./") || spec.startsWith("../")) { - const dep = resolveRelativeModule(file, spec); - if (dep !== null && !dep.endsWith(".json") && !dep.endsWith(".d.ts")) queue.push(dep); - continue; - } - if (spec.startsWith("#") || spec.startsWith("node:")) continue; - if (NODE_BUILTINS.has(packageNameOf(spec))) continue; - if (!bareSeen.has(spec)) { - bareSeen.add(spec); - bare.push(spec); - } - } - } - return bare; -} - -/* ── installed-package lookup (name/version/published entry) ──────────── */ - -interface InstalledPackage { - dir: string; - name: string; - version: string; - pkgJson: Record; -} - -/** node_modules/ walking up from `fromDir`, realpath-free (the - * published package.json is all this needs). */ -function findInstalled(fromDir: string, name: string): InstalledPackage | null { - for (let dir = fromDir; ; ) { - const candidate = join(dir, "node_modules", name); - const pkgJson = readJson(join(candidate, "package.json")); - if (pkgJson !== null && typeof pkgJson["version"] === "string") { - return { - dir: candidate, - name: typeof pkgJson["name"] === "string" ? pkgJson["name"] : name, - version: pkgJson["version"], - pkgJson, - }; - } - const parent = dirname(dir); - if (parent === dir) return null; - dir = parent; - } -} - -/* ── attestation → {repo, commit} ──────────────────────────────────────── */ - -interface Attested { - repo: string; - commit: string; -} - -/** GET the npm attestation set and extract the SLSA provenance - * predicate's resolved source dependency. Throws with a one-line reason - * on every failure shape (no attestation, network, malformed). */ -async function fetchAttestation(name: string, version: string): Promise { - const url = `https://registry.npmjs.org/-/npm/v1/attestations/${encodeURIComponent(`${name}@${version}`).replace(/%40/g, "@").replace(/%2F/gi, "/")}`; - const res = await fetch(url, { signal: AbortSignal.timeout(30_000) }); - if (res.status === 404) throw new Error("no provenance attestation published"); - if (!res.ok) throw new Error(`attestation fetch failed (HTTP ${res.status})`); - const body = (await res.json()) as { attestations?: { predicateType?: string; bundle?: { dsseEnvelope?: { payload?: string } } }[] }; - for (const att of body.attestations ?? []) { - if (!att.predicateType?.startsWith("https://slsa.dev/provenance")) continue; - const payload = att.bundle?.dsseEnvelope?.payload; - if (payload === undefined) continue; - const stmt = JSON.parse(Buffer.from(payload, "base64").toString("utf8")) as { - predicate?: { buildDefinition?: { resolvedDependencies?: { uri?: string; digest?: { gitCommit?: string } }[] } }; - }; - for (const dep of stmt.predicate?.buildDefinition?.resolvedDependencies ?? []) { - const commit = dep.digest?.gitCommit; - if (typeof commit === "string" && commit !== "" && typeof dep.uri === "string") { - return { repo: dep.uri, commit }; - } - } - } - throw new Error("attestation set carries no SLSA provenance predicate"); -} - -/* ── source fetch (content-addressed by the attested commit) ──────────── */ - -function cacheRoot(): string { - return process.env["SCRIPTC_PROVENANCE_CACHE"] ?? join(homedir(), ".cache", "scriptc", "provenance"); -} - -/** The cached source tree for an attested commit, fetching it once from - * codeload (github repos only in the prototype). Returns the tree root. */ -async function fetchSourceTree(repo: string, commit: string): Promise { - const dest = join(cacheRoot(), commit); - if (isDirectory(dest)) return dest; - const m = /github\.com\/([^/]+)\/([^/@#]+)/.exec(repo); - if (m === null) throw new Error(`source repository is not a github URL (${repo})`); - const url = `https://codeload.github.com/${m[1]}/${m[2]}/tar.gz/${commit}`; - const res = await fetch(url, { signal: AbortSignal.timeout(120_000) }); - if (!res.ok) throw new Error(`source fetch failed (HTTP ${res.status} for ${url})`); - const bytes = Buffer.from(await res.arrayBuffer()); - await mkdir(cacheRoot(), { recursive: true }); - const tmp = await mkdtemp(join(tmpdir(), "scriptc-provenance-")); - try { - const tarball = join(tmp, "src.tgz"); - await writeFile(tarball, bytes); - const extractDir = join(tmp, "tree"); - await mkdir(extractDir); - await execFileAsync("tar", ["-xzf", tarball, "-C", extractDir, "--strip-components=1"]); - try { - await rename(extractDir, dest); - } catch { - // A parallel compile won the rename — its tree is the same content. - if (!isDirectory(dest)) throw new Error("source cache rename failed"); - } - } finally { - await rm(tmp, { recursive: true, force: true }); - } - return dest; -} - -/* ── package dir + source-entry mapping ────────────────────────────────── */ - -/** The directory inside `tree` whose package.json names `name`: the root, - * then the conventional monorepo layouts, then a bounded scan. */ -function locatePackageDir(tree: string, name: string): string | null { - const nameOf = (dir: string): string | null => { - const pkg = readJson(join(dir, "package.json")); - return pkg !== null && typeof pkg["name"] === "string" ? pkg["name"] : null; - }; - if (nameOf(tree) === name) return tree; - const bare = name.startsWith("@") ? name.split("/")[1]! : name; - const candidates = [ - join(tree, "packages", bare), - join(tree, "packages", name), - join(tree, "packages", name.replace("@", "").replace("/", "-")), - join(tree, "packages", `babel-${bare}`), - ]; - for (const c of candidates) { - if (nameOf(c) === name) return c; - } - // Bounded breadth-first scan (depth ≤ 3, node_modules/.git skipped). - const queue: { dir: string; depth: number }[] = [{ dir: tree, depth: 0 }]; - while (queue.length > 0) { - const { dir, depth } = queue.shift()!; - if (depth > 0 && nameOf(dir) === name) return dir; - if (depth >= 3) continue; - let entries: string[]; - try { - entries = readdirSync(dir).filter((entry) => isDirectory(join(dir, entry))).sort(); - } catch { - continue; - } - for (const e of entries) { - if (e === "node_modules" || e.startsWith(".")) continue; - queue.push({ dir: join(dir, e), depth: depth + 1 }); - } - } - return null; -} - -/** The published dist target for `subpath` — "exports" with the import - * condition, else module/main/types fields (root subpath only). */ -function publishedTargetOf(pkgJson: Record, subpath: string): string | null { - if (pkgJson["exports"] !== undefined) { - return resolveExports(pkgJson["exports"], subpath, NODE_IMPORT_CONDITIONS); - } - if (subpath !== ".") return subpath; - for (const field of ["module", "main", "types"]) { - const v = pkgJson[field]; - if (typeof v === "string" && v !== "") return v; - } - return "index.js"; -} - -/** dist target → source file, heuristically: the built path's leading - * dist/lib/build segment rewrites to src (or drops), an esm/cjs/dts flavor - * segment is skipped for a shared TypeScript source tree, extensions - * rewrite to TypeScript twins, and the root entry falls back to the - * conventional src/index.ts homes. First existing candidate wins. */ -function mapEntryToSource(pkgDir: string, target: string, subpath: string): string | null { - const rel = target.replace(/^\.\//, ""); - const stems = new Set([rel]); - const distRe = /^(dist|lib|build|out|output|dist-node|dist-src)\//; - if (distRe.test(rel)) { - const withoutDist = rel.replace(distRe, ""); - if (/^(esm|cjs|dts)\//.test(withoutDist)) { - stems.add(`src/${withoutDist.replace(/^(esm|cjs|dts)\//, "")}`); - } - stems.add(rel.replace(distRe, "src/")); - stems.add(withoutDist); - } else { - stems.add(`src/${rel}`); - } - const candidates: string[] = []; - for (const stem of stems) { - const base = stem.replace(/\.d\.(ts|mts|cts)$/, ".$1").replace(/\.(js|mjs|cjs)$/, ""); - if (/\.(ts|tsx|mts|cts)$/.test(base)) candidates.push(base); - else { - candidates.push(`${base}.ts`, `${base}.mts`, `${base}.cts`, `${base}.tsx`); - candidates.push(join(base, "index.ts")); - } - } - if (subpath === ".") { - candidates.push("src/index.ts", "src/index.mts", "index.ts", "src/index.tsx"); - } - for (const c of candidates) { - const abs = join(pkgDir, c); - if (isFile(abs)) return abs; - } - return null; -} - -/* ── the pipeline ─────────────────────────────────────────────────────── */ - -interface ManifestEntry { - dir: string; - commit?: string; - repo?: string; -} - -function readManifest(): Map { - const path = process.env["SCRIPTC_PROVENANCE_MANIFEST"]; - const out = new Map(); - if (path === undefined || path === "") return out; - const manifest = readJson(resolve(path)); - const packages = manifest?.["packages"]; - if (packages === null || typeof packages !== "object") return out; - for (const [name, raw] of Object.entries(packages as Record)) { - if (raw === null || typeof raw !== "object") continue; - const e = raw as { dir?: unknown; commit?: unknown; repo?: unknown }; - if (typeof e.dir !== "string") continue; - const dir = isAbsolute(e.dir) ? e.dir : resolve(dirname(resolve(path)), e.dir); - out.set(name, { - dir, - ...(typeof e.commit === "string" ? { commit: e.commit } : {}), - ...(typeof e.repo === "string" ? { repo: e.repo } : {}), - }); - } - return out; -} - -/** Resolves provenance sources for everything the entry's module graph - * imports (one transitive round per newly-mapped tree: source imports of - * OTHER packages try the pipeline too). Never throws for a package - * failure — those become notes and the package keeps its island path. */ -export async function resolveProvenanceSources(entryPath: string): Promise { - const entry = resolve(entryPath); - const manifest = readManifest(); - const packages: ProvenancePackageSource[] = []; - const notes: string[] = []; - /** package name → mapped package (or null after a noted failure). */ - const processed = new Map(); - - /** All specifiers seen so far, grouped by package name. */ - const specifiersByPackage = new Map>(); - const enqueue = (specs: readonly string[]): string[] => { - const newNames: string[] = []; - for (const spec of specs) { - const name = packageNameOf(spec); - let set = specifiersByPackage.get(name); - if (set === undefined) { - specifiersByPackage.set(name, (set = new Set())); - newNames.push(name); - } - set.add(spec); - } - return newNames; - }; - - const mapOne = async (name: string): Promise => { - if (processed.has(name)) return; - if (processed.size >= MAX_PACKAGES) { - processed.set(name, null); - notes.push(`${name}: skipped — provenance package limit (${MAX_PACKAGES}) reached; island path used`); - return; - } - processed.set(name, null); // claimed; overwritten on success - const installed = findInstalled(dirname(entry), name); - if (installed === null) { - notes.push(`${name}: not installed under the entry's node_modules; island path used`); - return; - } - let dir: string; - let repo: string; - let commit: string; - const seeded = manifest.get(name); - try { - if (seeded !== undefined) { - dir = seeded.dir; - repo = seeded.repo ?? "(manifest)"; - commit = seeded.commit ?? "(manifest)"; - if (!isDirectory(dir)) throw new Error(`manifest dir does not exist (${dir})`); - } else { - const attested = await fetchAttestation(installed.name, installed.version); - repo = attested.repo; - commit = attested.commit; - const tree = await fetchSourceTree(repo, commit); - const located = locatePackageDir(tree, installed.name); - if (located === null) { - throw new Error(`package directory not found inside the attested source tree (${tree})`); - } - dir = located; - } - const entries: Record = {}; - for (const spec of specifiersByPackage.get(name) ?? []) { - const parts = spec.split("/"); - const nameLen = spec.startsWith("@") ? 2 : 1; - const subpath = parts.length === nameLen ? "." : `./${parts.slice(nameLen).join("/")}`; - const target = publishedTargetOf(installed.pkgJson, subpath); - const source = target === null ? null : mapEntryToSource(dir, target, subpath); - if (source === null) { - notes.push( - `${name}@${installed.version}: no source mapping for '${spec}' (published target: ${target ?? "unexported"}); island path used`, - ); - continue; - } - entries[spec] = source; - } - if (Object.keys(entries).length === 0) return; - const srcPkg = readJson(join(dir, "package.json")); - const sourceVersion = typeof srcPkg?.["version"] === "string" ? srcPkg["version"] : undefined; - const pkg: ProvenancePackageSource = { - name: installed.name, - version: installed.version, - ...(sourceVersion !== undefined && sourceVersion !== installed.version ? { sourceVersion } : {}), - repo, - commit, - dir, - entries, - }; - if (pkg.sourceVersion !== undefined) { - notes.push( - `${name}: source tree's package.json says ${pkg.sourceVersion}, installed is ${installed.version} (release tooling that bumps at publish) — the behavior differential is the check`, - ); - } - packages.push(pkg); - processed.set(name, pkg); - // One transitive round: the mapped source's own bare imports try - // the pipeline too (versions resolve from the DRIVER's tree — a - // prototype heuristic; production wants the package's lockfile). - const inner = enqueue(bareImportsOf(Object.values(entries))); - for (const n of inner) await mapOne(n); - } catch (e) { - notes.push(`${name}@${installed.version}: ${e instanceof Error ? e.message : String(e)}; island path used`); - } - }; - - for (const name of enqueue(bareImportsOf([entry]))) { - await mapOne(name); - } - return { packages, notes }; +export function resolveProvenanceSources(entryPath: string): Promise { + return resolveProvenanceSourcesWithParser(entryPath, parseSourceFile); } diff --git a/packages/compiler/src/frontend/type-mapper.ts b/packages/compiler/src/frontend/type-mapper.ts index df21c0eb..c4a44e72 100644 --- a/packages/compiler/src/frontend/type-mapper.ts +++ b/packages/compiler/src/frontend/type-mapper.ts @@ -1041,17 +1041,25 @@ function mapTypeInner(type: ts.Type, ctx: TypeMapperCtx): IrType | null { // user aliases with the same names on the normal structural path. { const parseArgsSym = widened.getAliasSymbol() ?? widened.getSymbol(); - if ( - parseArgsSym && - PARSE_ARGS_DYN_TYPES.has(parseArgsSym.name) && - checker.declarationsOf(parseArgsSym).some( - (d) => - ctx.isStdlibFile(d.getSourceFile()) && - isDeclaredInAmbientModule(d as ts.Declaration, "util"), - ) - ) { - return DYN; - } + const belongs = (declaration: ts.Node): boolean => { + if (!ctx.isStdlibFile(declaration.getSourceFile())) return false; + let family = false; + for (let node: ts.Node | undefined = declaration; node; node = node.parent) { + if ((ts.isTypeAliasDeclaration(node) || ts.isInterfaceDeclaration(node)) && + PARSE_ARGS_DYN_TYPES.has(node.name.text)) family = true; + if (ts.isModuleDeclaration(node) && ts.isStringLiteral(node.name)) { + return family && (node.name.text === "util" || node.name.text === "node:util"); + } + } + return false; + }; + if (parseArgsSym && PARSE_ARGS_DYN_TYPES.has(parseArgsSym.name) && + checker.declarationsOf(parseArgsSym).some(belongs)) return DYN; + // ReturnType and instantiated conditional types can erase the alias. + // Require every member to retain the util declaration provenance so + // unrelated records with similar property names keep their own layout. + const members = checker.getPropertiesOfType(widened); + if (members.length > 0 && members.every((member) => checker.declarationsOf(member).some(belongs))) return DYN; } // The lib's BOXED wrapper interfaces used as TYPES (`const n: Number = // 5`): every value such a slot can hold IS the primitive — `new diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index b16cdc59..c0fecd01 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -1,3 +1,9 @@ +import { prepareExecutableModule } from "./executable/prepare.js"; +import { prepareLibrary, libraryLocalizeSymbols, libraryWasmExports, libraryWasmRefusal } from "./library/prepare.js"; +import { analyzeWithFrontend } from "./frontend/analysis.js"; +import { llvmRefusalDiag, targetRefusalDiag } from "./backend/target-diagnostics.js"; +import type { CompileOptions, CompileSourceOptions, CompileRequestOptions, CompileSourceResult, CompileResult, CompileExecutableResult, CompileRequestResult, CompileLibraryOptions, CompileLibraryResult, AnalyzeOptions, AnalyzeResult } from "./compile-types.js"; +export type { CompileOutputKind, CompileBaseOptions, CompileOptions, CompileSourceOptions, CompileRequestOptions, CompileArtifact, CompileFailure, CompileSourceResult, CompileResult, CompileExecutableResult, CompileRequestResult, CompileLibraryOptions, CompileLibraryResult, AnalyzeOptions, AnalyzeResult } from "./compile-types.js"; import { compilePackedLibrary } from "./backend/library-pack.js"; import { InternalCompilerError } from "./errors.js"; import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises"; @@ -18,7 +24,7 @@ import { emitNativeArtifact, NativeCodegenError } from "./backend/native-codegen import { privateSiblingPath } from "./backend/build-cache.js"; import { nativeCodegenTarget, nativeCodegenTargetRefusal } from "./backend/targets.js"; import { windowsSubsystemLinkerArgs, type WindowsSubsystem } from "./backend/targets.js"; -import { createNativeLinkInfo, type NativeLinkInfo } from "./backend/native-link-info.js"; +import { createNativeLinkInfo } from "./backend/native-link-info.js"; import { RuntimePackError } from "./backend/runtime-pack.js"; import { createNativeLinkPlan } from "./backend/link-plan.js"; import { @@ -29,35 +35,23 @@ import { } from "./backend/linker.js"; import { splitLlvmLibraryProgram, splitLlvmProgram } from "./backend/llvm/split.js"; import { emitLibraryIdentityLines, replaceLibraryIdentity, stripLibraryIdentity } from "./backend/library-identity-markers.js"; -import { checkerPanicDiag, ffiNativeBuildDiag, libAsyncExportDiag, libAsyncSurfaceDiag, libExportUnresolvedDiag, libGenericExportDiag, libIntBoundaryDiag, libNpmIneligibleDiag, libSidecarDiag, libUnmappableSignatureDiag, iceDiag, isCheckerPanic, LIB_INBOUND_BYTES_TRAP_CODE, LIB_RUNTIME_TRAP_CODES, nativeCodegenDiag, type ScrDiagnostic } from "./diagnostics/diagnostic.js"; -import { checkLibraryIntegerSlots, classSeed, hasIntSlots, numberCarrierKind, type FnIntSlots, type IntSlotConfig } from "./library/int-infer.js"; -import { loadLibraryProfile, profileRemediation, profileTeaching, type LibraryProfile } from "./library/library-profile.js"; -import { clearFenceEvalCaches, decorateLibraryRefusals, evaluateLibraryFences } from "./library/fence-eval.js"; -import { assembleTrapTeaching } from "./library/trap-teaching.js"; +import { ffiNativeBuildDiag, iceDiag, nativeCodegenDiag, type ScrDiagnostic } from "./diagnostics/diagnostic.js"; +import { loadLibraryProfile, type LibraryProfile } from "./library/library-profile.js"; +import { clearFenceEvalCaches, decorateLibraryRefusals } from "./library/fence-eval.js"; import { - buildSidecar, canonicalModuleGraph, - canonicalPath, clearSidecarCaches, compilerReleaseVersion, libraryIdentityHashes, updateSidecarIdentity, - type SidecarIntegerSlotFacts, - type SidecarIrRecordPattern, - type SidecarIrTypePattern, } from "./library/sidecar.js"; -import { validateSidecar } from "./library/sidecar-validate.js"; -import type { EntryExportInfo } from "./frontend/lib-exports.js"; -import type { ContractFacts } from "./frontend/lib-contract.js"; -import { moduleRuntimeFeatures, moduleLibAsyncSurface, moduleLibNondeterministicSurface, moduleEmbedsBuiltin, moduleUsesFetch, type IrFfiImport, type IrLibSection, type IrModule, type IrRecordShape, type IrType, type SrcLoc } from "./ir/ir.js"; +import { moduleRuntimeFeatures, type IrModule, type SrcLoc } from "./ir/ir.js"; import { serializeModule } from "./ir/serialize.js"; import { validateModule } from "./ir/validate.js"; import { loadProgram } from "./frontend/program-node.js"; -import { runFrontend } from "./frontend/pipeline.js"; +import { runFrontend, type FrontendFactory } from "./frontend/pipeline.js"; import { provenanceSources } from "./frontend/provenance-registry.js"; import { clearResolveCaches } from "./frontend/resolve.js"; -import type { LowerResult } from "./frontend/lowering/lowerer.js"; -import type { CoverageInput } from "./coverage/report.js"; import { loadFfiProfile, type FfiProfile } from "./ffi/ffi-manifest.js"; import { executableLinkFeatures } from "./backend/executable-features.js"; import { FrontendInputTracker, trackedReadFile } from "./frontend/input-tracker.js"; @@ -176,234 +170,6 @@ export { } from "./frontend/provenance-registry.js"; export * as ir from "./ir/ir.js"; -export type CompileOutputKind = "ir" | "llvm" | "asm" | "obj" | "exe"; - -export interface CompileBaseOptions { - /** Primary artifact path. The CLI supplies the output-kind default. */ - outPath: string; - /** Where generated intermediates and compatibility side artifacts land. */ - outDir: string; - /** @deprecated This option no longer controls output cleanup; sibling artifacts are retained. */ - defaultOutputPath?: boolean; - /** Compatibility-only additive IR side artifact for executable builds. - * The CLI's deprecated --emit-ir flag supplies this option. */ - emitIr?: boolean; - sanitize?: boolean; - /** Embed the dynamic-island engine (--dynamic). Off = the static default: - * island constructs are diagnostics and nothing about codegen or linking - * changes. */ - dynamic?: boolean; - /** LLVM is the production code generator. */ - backend?: "llvm"; - /** Native optimization posture. Release is the shipped -O2 default; dev - * uses -O0, source line tables, and stable multi-TU object caching for - * large LLVM programs. Darwin executables include an adjacent .dSYM. */ - optimization?: "release" | "dev"; - /** Remove symbol/debug payload from an executable at link time. */ - strip?: boolean; - /** Windows PE executable subsystem. Console is the default; GUI suppresses - * automatic console-window creation. Only valid for Windows executables. */ - windowsSubsystem?: WindowsSubsystem; - /** --npm-static: package names whose shipped, unminified JS compiles - * STATICALLY as program modules (inference types the bodies; statements - * the lowering cannot prove become runtime fences). "auto" opts in every - * directly-imported package passing the eligibility heuristics (own - * .d.ts, unminified JS, no build-transform markers). A package whose - * preflight refuses marks itself an offender and falls back to the - * island (--dynamic) or the requires-dynamic diagnostic (static builds) - * — never a silent misbuild. Off by default: nothing changes without - * the flag. */ - npmStatic?: readonly string[] | "auto"; - /** Outbound native FFI manifest. Its signature-only TypeScript bindings - * lower to direct C ABI calls. Source outputs retain those declarations; - * archive/system-library inputs join only an executable link. */ - ffiProfilePath?: string; - /** Attach the machine-readable external link recipe to an object result. - * Valid only with outputKind "obj"; it never invokes a linker. */ - nativeLinkInfo?: boolean; -} - -/** Executable compile options. This remains the compatibility type for the - * historical compile() API, whose omitted output kind means executable. */ -export interface CompileOptions extends CompileBaseOptions { - outputKind?: "exe"; - /** Internal validation lane retained for helper-object artifact tests. - * Supported ordinary LLVM executable builds select this path automatically. */ - nativeProgramObject?: boolean; -} - -/** Source-artifact compile options, discriminated by the required kind. */ -export interface CompileSourceOptions extends CompileBaseOptions { - outputKind: Exclude; -} - -/** Internal/dynamic request shape for callers that select the kind at runtime. - * Statically executable/source callers should prefer the narrower interfaces. */ -export interface CompileRequestOptions extends CompileBaseOptions { - outputKind?: CompileOutputKind; - /** Internal validation lane for executable requests. */ - nativeProgramObject?: boolean; -} - -export type CompileArtifact = - | { kind: "ir"; path: string } - | { kind: "llvm"; path: string } - | { kind: "asm"; path: string } - | { kind: "obj"; path: string; nativeLinkInfo?: NativeLinkInfo } - | { - kind: "exe"; - path: string; - translationUnitPath: string; - backend: "llvm"; - - }; - -export type CompileFailure = { - ok: false; - diagnostics: ScrDiagnostic[]; - sourceTexts: Map; -}; - -export type CompileSourceResult = - | { ok: true; artifact: Extract } - | CompileFailure; - -/** Historical executable result shape retained for source compatibility. */ -export type CompileResult = - | { - ok: true; - binaryPath: string; - llvmPath: string; - irPath?: string; - backend: "llvm"; - - } - | CompileFailure; - -export type CompileExecutableResult = - /** The generated LLVM source is retained beside the executable. */ - | (Extract & { - artifact: Extract; - }) - | CompileFailure; - -/** Result union for callers that choose outputKind dynamically. */ -export type CompileRequestResult = CompileSourceResult | CompileExecutableResult; - -/** The LLVM backend's tier refusal as a diagnostic. SC3xxx = backend - * coverage (the program is fine — this backend doesn't compile it yet); - * the parenthesized kind tag is machine-readable for the differential - * harness's histogram. */ -function llvmRefusalDiag(err: LlvmUnsupportedError, entryPath: string): ScrDiagnostic { - return { - code: "SC3001", - message: err.message, - loc: err.loc ?? { file: entryPath, start: 0, end: 0 }, - }; -} - -/** A valid program surface that the selected execution target cannot host. - * SC3xxx stays the backend/target-coverage family: source semantics are - * valid, but this target deliberately refuses them instead of emitting a - * binary that traps later. */ -function targetRefusalDiag(target: string, surface: string, loc: SrcLoc): ScrDiagnostic { - return { - code: "SC3002", - message: `${target} target does not support ${surface}`, - loc, - }; -} - -/** APIs that require host capabilities absent from portable WASI Preview 1. - * These are target diagnostics, not backend-tier gaps: the same language IR - * (including async, generators, and the dynamic island) is otherwise valid. - * Keep the fine-grained walk first so diagnostics point at the API use; the - * embedded-module checks are the entry-anchored safety net for island code. */ -function moduleWasiUnavailableSurface(mod: IrModule): { surface: string; loc: SrcLoc } | null { - const entryLoc: SrcLoc = { file: mod.sourceFile, start: 0, end: 0 }; - const prefixes: readonly (readonly [string, string])[] = [ - ["cp.", "child processes (WASI Preview 1 has no process-spawning API)"], - ["child.", "child processes (WASI Preview 1 has no process-spawning API)"], - ["spawnRes.", "child processes (WASI Preview 1 has no process-spawning API)"], - ["net.", "network sockets (WASI Preview 1 has no socket API)"], - ["http.", "network sockets (WASI Preview 1 has no socket API)"], - ["https.", "network sockets (WASI Preview 1 has no socket API)"], - ["http2.", "network sockets (WASI Preview 1 has no socket API)"], - ["h2.", "network sockets (WASI Preview 1 has no socket API)"], - ["dgram.", "network sockets (WASI Preview 1 has no socket API)"], - ["dns.", "network sockets (WASI Preview 1 has no socket API)"], - ["tls.", "network sockets (WASI Preview 1 has no socket API)"], - ["fetch.", "network-backed fetch (WASI Preview 1 has no socket API)"], - ["fs.watch", "filesystem watching (WASI Preview 1 has no notification API)"], - ["watcher.", "filesystem watching (WASI Preview 1 has no notification API)"], - ]; - const kinds: ReadonlyMap = new Map([ - ["child", "child processes (WASI Preview 1 has no process-spawning API)"], - ["spawnRes", "child processes (WASI Preview 1 has no process-spawning API)"], - ["childStream", "child processes (WASI Preview 1 has no process-spawning API)"], - ["childWriter", "child processes (WASI Preview 1 has no process-spawning API)"], - ["netServer", "network sockets (WASI Preview 1 has no socket API)"], - ["netSocket", "network sockets (WASI Preview 1 has no socket API)"], - ["http2Session", "network sockets (WASI Preview 1 has no socket API)"], - ["http2Stream", "network sockets (WASI Preview 1 has no socket API)"], - ["dgramSocket", "network sockets (WASI Preview 1 has no socket API)"], - ["fsWatcher", "filesystem watching (WASI Preview 1 has no notification API)"], - ["httpReq", "network sockets (WASI Preview 1 has no socket API)"], - ["httpRes", "network sockets (WASI Preview 1 has no socket API)"], - ["httpClientReq", "network sockets (WASI Preview 1 has no socket API)"], - ["secureCtx", "network sockets (WASI Preview 1 has no socket API)"], - ]); - let found: { surface: string; loc: SrcLoc } | null = null; - const visit = (value: unknown, inheritedLoc: SrcLoc): void => { - if (found !== null || value === null || typeof value !== "object") return; - if (Array.isArray(value)) { - for (const item of value) visit(item, inheritedLoc); - return; - } - const node = value as { kind?: unknown; fn?: unknown; loc?: SrcLoc }; - const loc = node.loc ?? inheritedLoc; - if (typeof node.kind === "string") { - const kindSurface = kinds.get(node.kind); - if (kindSurface !== undefined) { - found = { surface: kindSurface, loc }; - return; - } - if (node.kind === "libCall" && typeof node.fn === "string") { - if (node.fn === "process.kill" || node.fn === "process.killNum" || - node.fn === "process.onSignal" || node.fn === "process.offSignal") { - found = { surface: "OS signals (WASI Preview 1 has no signal API)", loc }; - return; - } - if (node.fn === "os.networkInterfaces") { - found = { surface: "network-interface enumeration (WASI Preview 1 has no interface API)", loc }; - return; - } - for (const [prefix, surface] of prefixes) { - if (node.fn.startsWith(prefix)) { - found = { surface, loc }; - return; - } - } - } - } - for (const key of Object.keys(value)) { - visit((value as Record)[key], loc); - } - }; - visit(mod, entryLoc); - if (found !== null) return found; - - if (moduleUsesFetch(mod)) { - return { surface: "network-backed fetch (WASI Preview 1 has no socket API)", loc: entryLoc }; - } - for (const builtin of ["node:http", "node:https", "node:net", "node:tls"] as const) { - if (moduleEmbedsBuiltin(mod, builtin)) { - return { surface: `${builtin} networking (WASI Preview 1 has no socket API)`, loc: entryLoc }; - } - } - return null; -} - /** Clang may print every warning from the generated/runtime translation * units before the actionable linker failure. Keep the source diagnostic * precise by starting at the first portable linker marker; if the driver @@ -421,11 +187,6 @@ function ffiNativeBuildDetail(err: CcCompileError): string { ); } -export interface AnalyzeResult { - coverage: CoverageInput; - sourceTexts: Map; -} - /** Platform semantics depend on the output target, without compiler discovery. */ export function buildTargetPlatform(env: NodeJS.ProcessEnv = process.env): string { return configuredTargetPlatform(env); @@ -438,109 +199,13 @@ export function sourceTargetPlatform(env: NodeJS.ProcessEnv = process.env): stri return configuredTargetPlatform(env); } -export interface AnalyzeOptions { - /** Analyze as a --dynamic build (island constructs lower instead of - * producing requires-dynamic diagnostics). */ - dynamic?: boolean; - /** --npm-static (see CompileOptions.npmStatic): the analysis compiles - * opted-in packages' JS as program modules and the coverage report - * carries each package's static/fallback status. */ - npmStatic?: readonly string[] | "auto"; - /** Analyze with the outbound native bindings from this FFI manifest. */ - ffiProfilePath?: string; - /** Coverage-only external host type surfaces: exact bare module - * specifier → local declaration file. The checker uses the declarations - * to analyze project code, but imported runtime values remain explicit - * SC1010 blockers rather than being counted as executable. */ - externalTypes?: Readonly>; -} - -/** Analysis without codegen: how much of the program compiles statically. - * Unlike compile(), lowering diagnostics are data here, not failure. */ export function analyze(entryPath: string, opts: AnalyzeOptions = {}): AnalyzeResult { - let ffi: FfiProfile | null = null; - if (opts.ffiProfilePath !== undefined) { - const loaded = loadFfiProfile(opts.ffiProfilePath); - if (!loaded.ok) { - return { - coverage: { - file: entryPath, - dynamic: opts.dynamic ?? false, - stats: { statementsTotal: 0, statementsFailed: 0, statementsIsland: 0, functionsSkipped: 0 }, - diagnostics: loaded.diagnostics, - preflightFailed: true, - }, - sourceTexts: new Map(), - }; - } - ffi = loaded.profile; - } - const fe = runFrontend(entryPath, loadProgram, opts.npmStatic, opts.externalTypes); - try { - const emptyStats = { statementsTotal: 0, statementsFailed: 0, statementsIsland: 0, functionsSkipped: 0 }; - - const preflight = fe.preflight; - // Import-FORM fences don't stop the analysis: the module graph is still - // computable (a fenced import contributes no edges), the imported - // bindings poison at their use sites, and the fences join the blockers - // list beside statement-level ones — the report shows a statement - // percentage instead of stopping at the import lines. Everything else — - // tsc errors, config incompatibilities, circular imports — still stops - // at preflight (no trustworthy program to lower). Builds are unchanged: - // compile() fails on every preflight diagnostic exactly as before. - const IMPORT_FENCES = new Set(["SC1010", "SC1012", "SC1013", "SC1014", "SC1015"]); - if (preflight.some((d) => !IMPORT_FENCES.has(d.code))) { - return { - coverage: { - file: entryPath, - dynamic: opts.dynamic ?? false, - stats: emptyStats, - diagnostics: preflight, - ...(fe.npmStatic.length > 0 ? { npmStatic: fe.npmStatic } : {}), - preflightFailed: true, - }, - sourceTexts: fe.sourceTexts(), - }; - } - // Coverage is whole-program by design: builds stop at what the entry - // reaches, but the analysis additionally lowers the unreached remainder - // (throwaway) so the report covers everything the source declares — with - // the unreached share in its own group. - const lowered = fe.lower({ - dynamic: opts.dynamic ?? false, - coverage: true, - targetPlatform: buildTargetPlatform(), - ...(ffi !== null ? { ffiImports: ffi.functions } : {}), - }); - const provenance = provenanceSources(); - return { - coverage: { - file: entryPath, - dynamic: opts.dynamic ?? false, - stats: lowered.stats, - // The import fences report as blockers alongside the statement-level - // ones (use sites of the fenced bindings emit matching diagnostics, - // which the report groups with these). - diagnostics: [...preflight, ...lowered.diagnostics], - ...(lowered.runtimeFences.length > 0 ? { runtimeFences: lowered.runtimeFences } : {}), - ...(lowered.unreached ? { unreached: lowered.unreached } : {}), - ...(lowered.npmBuiltins ? { npmBuiltins: lowered.npmBuiltins } : {}), - ...(lowered.npmLazyTraps ? { npmLazyTraps: lowered.npmLazyTraps } : {}), - ...(fe.npmStatic.length > 0 ? { npmStatic: fe.npmStatic } : {}), - // --provenance-sources: the per-package attribution inputs (the - // report aggregates statsByFile under each package's source dir). - ...(provenance !== null ? { provenance } : {}), - ...(lowered.statsByFile ? { statsByFile: lowered.statsByFile } : {}), - ...(lowered.provenanceElided ? { provenanceElided: lowered.provenanceElided } : {}), - preflightFailed: false, - }, - sourceTexts: fe.sourceTexts(), - }; - } finally { - fe.dispose(); - } + return analyzeWithFrontend(entryPath, opts, buildTargetPlatform(), nodeFrontend); } +const nodeFrontend: FrontendFactory = (entry, npmStatic, externalTypes, libraryNpmStatic) => + runFrontend(entry, loadProgram, npmStatic, externalTypes, libraryNpmStatic); + /** The whole pipeline: load → preflight → lower → validate → emit LLVM → link. */ function clearCompileSessionCaches(): void { clearResolveCaches(); @@ -755,58 +420,9 @@ async function prepareExecutableInput( entryPath: string, opts: CompileRequestOptions, ffi: FfiProfile | null, buildPlatform: string, ): Promise { const outputKind = opts.outputKind ?? "exe"; - const fe = runFrontend(entryPath, loadProgram, opts.npmStatic); - let lowered: LowerResult; - let sourceTexts: Map; - // The frontend (and its tsgo server) is released as soon as lowering - // ends — clang and the link never hold it open. - try { - const fail = (diagnostics: ScrDiagnostic[]): CompileFailure => ({ - ok: false, - diagnostics, - sourceTexts: fe.sourceTexts(), - }); - - if (fe.preflight.length > 0) return fail(fe.preflight); - - try { - lowered = fe.lower({ - dynamic: opts.dynamic ?? false, - targetPlatform: buildPlatform, - ...(ffi !== null ? { ffiImports: ffi.functions } : {}), - }); - } catch (e) { - // The last-resort panic fence: an upstream tsgo panic that crossed a - // checker call no statement/collection fence wrapped still becomes a - // clean failed compile (anchored at the entry), never a crashed CLI. - if (!isCheckerPanic(e)) throw e; - return fail([ - checkerPanicDiag(e.message.split("\n", 1)[0]!, { file: entryPath, start: 0, end: 0 }), - ]); - } - if (lowered.module === null) return fail(lowered.diagnostics); - - const validation = validateModule(lowered.module); - if (validation.length > 0) { - return fail(validation.map((v) => iceDiag(v.message, v.loc))); - } - if (buildPlatform === "wasi") { - const entryLoc: SrcLoc = { file: entryPath, start: 0, end: 0 }; - if (opts.sanitize) { - return fail([targetRefusalDiag("wasm32-wasi", "--sanitize", entryLoc)]); - } - if (ffi !== null) { - return fail([targetRefusalDiag("wasm32-wasi", "native FFI manifests", entryLoc)]); - } - const unavailable = moduleWasiUnavailableSurface(lowered.module); - if (unavailable !== null) { - return fail([targetRefusalDiag("wasm32-wasi", unavailable.surface, unavailable.loc)]); - } - } - sourceTexts = fe.sourceTexts(); - } finally { - fe.dispose(); - } + const prepared = prepareExecutableModule(entryPath, opts, ffi, buildPlatform, nodeFrontend); + if (!prepared.ok) return prepared; + const { mod, sourceTexts } = prepared; const stem = basename(entryPath).replace(/\.(ts|mts|cts|js|mjs|cjs)$/, ""); const defaultSourcePaths = { @@ -819,14 +435,14 @@ async function prepareExecutableInput( if (outputKind === "ir") { await mkdir(dirname(opts.outPath), { recursive: true }); - await writeFile(opts.outPath, serializeModule(lowered.module)); + await writeFile(opts.outPath, serializeModule(mod)); return { ok: true, artifact: { kind: "ir", path: opts.outPath } }; } if (outputKind === "llvm" || outputKind === "asm" || outputKind === "obj") { let llvm: string; try { - llvm = emitLlvmModule(lowered.module, { + llvm = emitLlvmModule(mod, { targetTriple: process.env["SCRIPTC_TARGET"] ?? "", ...debugOptions, pointerBits: buildPlatform === "wasi" ? 32 : 64, @@ -873,7 +489,7 @@ async function prepareExecutableInput( programObject: opts.outPath, target, features: executableNativeFeatures( - lowered.module, + mod, "llvm", opts.dynamic ?? false, opts.optimization ?? "release", @@ -892,7 +508,7 @@ async function prepareExecutableInput( const backend = "llvm" as const; let llvmSource: string; try { - llvmSource = emitLlvmModule(lowered.module, { + llvmSource = emitLlvmModule(mod, { targetTriple: process.env["SCRIPTC_TARGET"] ?? "", ...debugOptions, pointerBits: buildPlatform === "wasi" ? 32 : 64, @@ -907,11 +523,11 @@ async function prepareExecutableInput( let irPath: string | undefined; if (opts.emitIr) { irPath = defaultSourcePaths.ir; - await writeFile(irPath, serializeModule(lowered.module)); + await writeFile(irPath, serializeModule(mod)); } const nativeFeatures = executableNativeFeatures( - lowered.module, + mod, backend, opts.dynamic ?? false, opts.optimization ?? "release", @@ -1354,397 +970,6 @@ async function compileTracked( * emission; there is no fallback concept on this path (an out-of-tier * program under emission "llvm" is SC3001, fail-loudly). */ -export interface CompileLibraryOptions { - profilePath: string; - /** Where the archive and the kept program TU land. */ - outDir: string; - /** Artifact path. Default: .lib.a, or .wasm for wasm32-wasi. */ - outPath?: string; - emitIr?: boolean; - sanitize?: boolean; -} - -export type CompileLibraryResult = - /** `sidecarPath` is present exactly when the profile declares a - * `sidecar` section: the contract JSON written beside the archive by - * the same invocation (ask 2). */ - | { ok: true; archivePath: string; llvmPath: string; backend: "llvm"; irPath?: string; sidecarPath?: string } - | { ok: false; diagnostics: ScrDiagnostic[]; sourceTexts: Map }; - -/** The marshalling-class fit over IR types (design §4.2 + the ratified - * integer plumbing classes): number is every f64-backed class, bool/string - * map directly, bytes is the u8 element kind. */ -function libClassFits(cls: string, t: IrType): boolean { - switch (cls) { - case "bool": - return t.kind === "bool"; - case "string": - return t.kind === "string"; - case "bytes": - return t.kind === "bytes" && t.elem === "u8"; - default: // f64 and the u8/u32/i32 plumbing classes - return t.kind === "f64"; - } -} - -/** Resolve the profile's export map against the entry module — SC4002/ - * SC4004/SC4007 from the declaration facts, SC4003 from the lowered IR - * signatures — and land the library section on the module. */ -function resolveLibrarySection( - profile: LibraryProfile, - entryInfo: Map, - mod: IrModule, - entryPath: string, -): { lib: IrLibSection } | { diagnostics: ScrDiagnostic[] } { - const diagnostics: ScrDiagnostic[] = []; - const entryLoc = { file: entryPath, start: 0, end: 0 }; - const fnByName = new Map(mod.functions.map((f) => [f.name, f])); - const exports: IrLibSection["exports"] = []; - for (const e of profile.exports) { - const info = entryInfo.get(e.export); - if (info === undefined) { - diagnostics.push( - libExportUnresolvedDiag(e.export, "the entry module has no exported function declaration by that name", entryLoc), - ); - continue; - } - if (info.generic) { - diagnostics.push(libGenericExportDiag(e.export, info.loc)); - continue; - } - if (info.async || info.generator) { - diagnostics.push(libAsyncExportDiag(e.export, info.async ? "async" : "generator", info.loc)); - continue; - } - const fn = fnByName.get(e.export); - if (fn === undefined) { - diagnostics.push( - libExportUnresolvedDiag(e.export, "the export did not lower to a compiled function", info.loc), - ); - continue; - } - if (fn.params.length !== e.params.length) { - diagnostics.push( - libUnmappableSignatureDiag( - e.export, - "signature", - `has ${fn.params.length} parameter(s) but the profile declares ${e.params.length} marshalling class(es)`, - info.loc, - ), - ); - continue; - } - let bad = false; - e.params.forEach((cls, i) => { - if (!libClassFits(cls, fn.params[i]!.type)) { - bad = true; - diagnostics.push( - libUnmappableSignatureDiag( - e.export, - `parameter ${i + 1} ('${fn.params[i]!.name}')`, - `has IR type '${fn.params[i]!.type.kind}', which does not fit the declared marshalling class '${cls}'`, - info.loc, - ), - ); - } - }); - if (e.returns === "void" ? fn.returnType.kind !== "void" : !libClassFits(e.returns, fn.returnType)) { - bad = true; - diagnostics.push( - libUnmappableSignatureDiag( - e.export, - "the return", - `has IR type '${fn.returnType.kind}', which does not fit the declared marshalling class '${e.returns}'`, - info.loc, - ), - ); - } - if (!bad) { - const resolvedExport: IrLibSection["exports"][number] = { - symbol: e.symbol, - fnName: e.export, - params: e.params, - returns: e.returns, - }; - if (e.params.includes("bytes")) { - // The wrapper's one host-contract trap (an inbound bytes length - // past the marshalling class's range) is assembled HERE, once, as - // the structured trap-teaching message: the profile's teaching for - // SC4012 (or the mode's default text), the code, the trapping - // export's C symbol exactly as the host linked it, and the - // profile's remediation when supplied — so the backend emits the - // same bytes and the sink sees one canonical message. - resolvedExport.inboundBytesTrap = assembleTrapTeaching( - profileTeaching(profile, LIB_INBOUND_BYTES_TRAP_CODE) ?? - "scriptc: library inbound bytes length out of range\n", - LIB_INBOUND_BYTES_TRAP_CODE, - e.symbol, - profileRemediation(profile, LIB_INBOUND_BYTES_TRAP_CODE), - ); - } - if (e.params.includes("i64") || e.params.includes("u64")) { - // The sibling host-contract trap for inbound declared-integer - // parameters (ask 4): a value past ±(2^53−1) cannot ride f64 - // exactly, and silent rounding is a coercion the author never - // wrote. Same code (SC4012 — one host-contract story), same - // assembly-once discipline. - resolvedExport.inboundIntTrap = assembleTrapTeaching( - profileTeaching(profile, LIB_INBOUND_BYTES_TRAP_CODE) ?? - "scriptc: library inbound integer parameter out of range\n", - LIB_INBOUND_BYTES_TRAP_CODE, - e.symbol, - profileRemediation(profile, LIB_INBOUND_BYTES_TRAP_CODE), - ); - } - exports.push(resolvedExport); - } - } - if (diagnostics.length > 0) return { diagnostics }; - // The runtime detected-trap overlay rows: one per family code the profile - // declares teaching or remediation text for, in the registry family's - // order. LLVM emits these rows as the program TU's overlay table, - // which the runtime uses to assemble the sink message. (SC4012 stays compile-time - // assembled into the wrapper's message above and never reaches the - // funnel's assembly path.) - const trapOverlays: IrLibSection["trapOverlays"] = []; - for (const code of LIB_RUNTIME_TRAP_CODES) { - const teaching = profileTeaching(profile, code); - const remediation = profileRemediation(profile, code); - if (teaching !== undefined || remediation !== undefined) { - trapOverlays.push({ - code, - ...(teaching !== undefined ? { teaching } : {}), - ...(remediation !== undefined ? { remediation } : {}), - }); - } - } - return { - lib: { - profileName: profile.name, - prefix: profile.prefix, - initSymbol: profile.initSymbol, - sinkRegisterSymbol: profile.sinkRegisterSymbol, - collectSymbol: profile.collectSymbol, - resultResetSymbol: profile.resultResetSymbol, - threadInstances: profile.instancePerThread, - // Host-callback channels: declaration order is the runtime slot - // assignment, and the unregistered-call trap text is assembled HERE, - // once, for consistent constant bytes (a DETECTED - // trap: the funnel classifies the "scriptc: library callback " - // prefix as SC4025 and names the entry the host called — the entry - // is runtime knowledge, so no compile-time SC4012-style assembly - // can carry it). Both fields stay absent on callback-free profiles - // (the byte-identity guarantee). - ...(profile.callbacks.length > 0 - ? { - callbackRegisterSymbol: profile.callbackRegisterSymbol!, - callbacks: profile.callbacks.map((cb, i) => ({ - name: cb.name, - slot: i, - params: [...cb.params], - returns: cb.returns, - unregisteredTrap: `scriptc: library callback '${cb.name}' invoked before registration\n`, - })), - } - : {}), - exports, - trapOverlays, - }, - }; -} - -/** The export map's integer-slot obligations (ask 4): i64/u64 params and - * returns become declared boundary slots keyed `exports..params[i]` - * / `exports..return`; the u8/u32/i32 plumbing classes contribute - * their proven inbound shapes as parameter seeds (the wrapper's coercion - * contract), tightening the intraprocedural analysis at zero declaration - * cost. Sidecar-declared slots (record fields, msg arms, helper params - * and returns) merge into the same config at sidecar build. */ -function libraryIntSlotConfig(profile: LibraryProfile): IntSlotConfig { - const cfg: IntSlotConfig = { fns: new Map(), records: new Map() }; - for (const e of profile.exports) { - const params = e.params.map((c) => (c === "i64" || c === "u64" ? c : null)); - const ret = e.returns === "i64" || e.returns === "u64" ? e.returns : null; - const paramSeeds = e.params.map((c) => (c === "u8" || c === "u32" || c === "i32" ? classSeed(c) : null)); - if (params.every((p) => p === null) && ret === null && paramSeeds.every((s) => s === null)) continue; - const slots: FnIntSlots = { - fnName: e.export, - params, - paramPaths: e.params.map((c, i) => (c === "i64" || c === "u64" ? `exports.${e.export}.params[${i}]` : null)), - ret, - retPath: ret !== null ? `exports.${e.export}.return` : null, - paramSeeds, - }; - cfg.fns.set(e.export, slots); - } - return cfg; -} - -/** Match the sidecar syntax's exact structural type projection against the - * frontend's interned IR registries. The pattern deliberately mirrors - * ShapeRegistry's identity: every field name and recursively mapped field - * type participates. Tagged payload records additionally accept omission - * of their `kind` field because the lowering may carry that discriminant - * only in the surrounding union tag. */ -function sidecarRecordMatcher( - mod: IrModule, -): (pattern: SidecarIrRecordPattern, shape: IrRecordShape) => boolean { - const records = new Map((mod.records ?? []).map((shape) => [shape.id, shape])); - const unions = new Map((mod.unions ?? []).map((union) => [union.id, union])); - - const recordMatches = ( - pattern: SidecarIrRecordPattern, - shape: IrRecordShape, - ): boolean => { - if (shape.tuple === true || shape.indexValue !== undefined) return false; - const variants = [pattern.fields]; - if (pattern.kindMayBeOmitted === true) { - variants.push(pattern.fields.filter((field) => field.name !== "kind")); - } - return variants.some( - (fields) => - fields.length === shape.fields.length && - fields.every((field) => { - const actual = shape.fields.find((candidate) => candidate.name === field.name); - return actual !== undefined && typeMatches(field.type, actual.type); - }), - ); - }; - - const unionMatches = ( - patterns: SidecarIrTypePattern[], - actual: IrType[], - ): boolean => { - if (patterns.length !== actual.length) return false; - const used = new Set(); - const visit = (index: number): boolean => { - if (index === patterns.length) return true; - for (let i = 0; i < actual.length; i++) { - if (used.has(i) || !typeMatches(patterns[index]!, actual[i]!)) continue; - used.add(i); - if (visit(index + 1)) return true; - used.delete(i); - } - return false; - }; - return visit(0); - }; - - const typeMatches = ( - pattern: SidecarIrTypePattern, - actual: IrType, - ): boolean => { - switch (pattern.kind) { - case "f64": - case "string": - case "bool": - case "nullT": - case "undefinedT": - case "dyn": - return actual.kind === pattern.kind; - case "bytes": - return actual.kind === "bytes" && actual.elem === pattern.elem; - case "array": - return actual.kind === "array" && typeMatches(pattern.elem, actual.elem); - case "record": { - if (actual.kind !== "record") return false; - const shape = records.get(actual.shapeId); - return shape !== undefined && recordMatches(pattern, shape); - } - case "union": { - if (actual.kind !== "union") return false; - const union = unions.get(actual.unionId); - return union !== undefined && unionMatches(pattern.arms, union.arms); - } - } - }; - - return (pattern, shape) => recordMatches(pattern, shape); -} - -/** Merge the sidecar-resolved integer slots (ask 4) into the inference - * config: helper slots key by function name and IR parameter index (the - * projection already shifted past the model receiver); record-field - * slots map onto every interned IR shape whose complete structural field - * signature matches the projected record's. Shapes intern structurally, - * so a same-shaped second type shares the obligation. DECLARED paths with - * the same class coalesce while retaining every source path for verdicts; - * differing classes refuse because one lowered field cannot seed or check - * two distinct class contracts without arm provenance. A - * record fact that matches no shape binds nothing: no compiled code - * constructs the type (the contract surface — init/update/subscriptions - * and every helper — is force-lowered whenever integer slots are - * declared, so this is genuine vacuity, not dead-stripping). */ -function mergeSidecarIntSlots( - cfg: IntSlotConfig, - facts: SidecarIntegerSlotFacts, - mod: IrModule, -): { ok: true; config: IntSlotConfig } | { ok: false; diagnostic: ScrDiagnostic } { - const recordMatches = sidecarRecordMatcher(mod); - for (const h of facts.helpers) { - const fn = mod.functions.find((f) => f.name === h.fnName); - const arity = Math.max(fn?.params.length ?? 0, (h.index ?? 0) + 1); - let slots = cfg.fns.get(h.fnName); - if (slots === undefined) { - slots = { - fnName: h.fnName, - params: new Array(arity).fill(null), - paramPaths: new Array(arity).fill(null), - ret: null, - retPath: null, - paramSeeds: new Array(arity).fill(null), - }; - cfg.fns.set(h.fnName, slots); - } - if (h.kind === "param") { - const i = h.index!; - while (slots.params.length <= i) { - slots.params.push(null); - slots.paramPaths.push(null); - slots.paramSeeds.push(null); - } - slots.params[i] = h.cls; - slots.paramPaths[i] = h.path; - } else { - slots.ret = h.cls; - slots.retPath = h.path; - } - } - for (const r of facts.records) { - for (const shape of mod.records ?? []) { - if (!recordMatches(r.shape, shape)) continue; - const target = shape.fields.find((f) => f.name === r.targetField); - if (target === undefined || numberCarrierKind(target.type, mod) === null) continue; - let m = cfg.records.get(shape.id); - if (m === undefined) { - m = new Map(); - cfg.records.set(shape.id, m); - } - const existing = m.get(r.targetField); - if (existing !== undefined && existing.cls !== r.cls) { - const paths = [ - ...existing.paths.map((path) => `'${path}' (${existing.cls})`), - `'${r.path}' (${r.cls})`, - ]; - return { - ok: false, - diagnostic: libSidecarDiag( - `integer slots ${paths.join(" and ")} collapse to the same lowered record field '${r.targetField}' — their proof obligations cannot be kept distinct`, - r.loc, - "kind-tagged union arms and structurally identical records may share one lowered shape — same-class declarations coalesce, but differing classes require distinct structural shapes or at most one classified slot", - ), - }; - } - if (existing === undefined) { - m.set(r.targetField, { cls: r.cls, paths: [r.path] }); - } else if (!existing.paths.includes(r.path)) { - existing.paths.push(r.path); - } - } - } - return { ok: true, config: cfg }; -} - function libraryNativeFeatures( mod: IrModule, backend: "llvm", @@ -1767,22 +992,6 @@ function libraryNativeFeatures( }; } -function libraryLocalizeSymbols(profile: LibraryProfile): string[] | undefined { - return profile.localizeRuntime - ? [ - profile.initSymbol, - profile.sinkRegisterSymbol, - ...(profile.collectSymbol !== null ? [profile.collectSymbol] : []), - ...(profile.resultResetSymbol !== null ? [profile.resultResetSymbol] : []), - ...(profile.callbackRegisterSymbol !== null ? [profile.callbackRegisterSymbol] : []), - ...(profile.sidecar !== null - ? [profile.sidecar.buildIdSymbol, profile.sidecar.abiVersionSymbol] - : []), - ...profile.exports.map((entry) => entry.symbol), - ] - : undefined; -} - async function compileLibraryNative( profile: LibraryProfile, llvmPath: string, @@ -1794,13 +1003,7 @@ async function compileLibraryNative( if (wasmTarget?.platform === "wasi") { await compilePackedLibrary({ cPath: llvmPath, outPath: archivePath, optimization: profile.optimization, ...features, - }, wasmTarget, [ - profile.initSymbol, "scriptc_alloc", "scriptc_free", - ...(profile.collectSymbol === null ? [] : [profile.collectSymbol]), - ...(profile.resultResetSymbol === null ? [] : [profile.resultResetSymbol]), - ...(profile.sidecar === null ? [] : [profile.sidecar.buildIdSymbol, profile.sidecar.abiVersionSymbol]), - ...profile.exports.map((entry) => entry.symbol), - ]); + }, wasmTarget, libraryWasmExports(profile)); return; } const localizeSymbols = libraryLocalizeSymbols(profile); @@ -2002,11 +1205,6 @@ async function compileLibraryTracked( } if (directory === profileDir || dirname(directory) === directory) break; } - const archivePath = opts.outPath ?? join( - opts.outDir, - `${basename(entryPath).replace(/\.(ts|mts|cts|js|mjs|cjs)$/, "")}${buildTargetPlatform() === "wasi" ? ".wasm" : ".lib.a"}`, - ); - // Mobile-target admission first — a pure env/host check, so a refused // pairing never reaches toolchain discovery. iOS targets (device and // simulator) build on darwin hosts only: the Apple SDK sysroot and the @@ -2028,16 +1226,13 @@ async function compileLibraryTracked( } const buildPlatform = buildTargetPlatform(); + const archivePath = opts.outPath ?? join( + opts.outDir, + `${basename(entryPath).replace(/\.(ts|mts|cts|js|mjs|cjs)$/, "")}${buildPlatform === "wasi" ? ".wasm" : ".lib.a"}`, + ); if (buildPlatform === "wasi") { - const reserved = new Set(["scriptc_alloc", "scriptc_free", "memory", "_initialize", "_start"]); - const symbols = [profile.initSymbol, profile.sinkRegisterSymbol, profile.collectSymbol, profile.resultResetSymbol, profile.callbackRegisterSymbol, ...profile.exports.map((entry) => entry.symbol), profile.sidecar?.buildIdSymbol, profile.sidecar?.abiVersionSymbol]; - const surface = opts.sanitize ? "sanitized library builds" - : profile.instancePerThread ? "thread-instanced libraries (instantiate separate Wasm instances instead)" - : profile.localizeRuntime ? "runtime localization (Wasm instances already isolate their runtime)" - : symbols.some((symbol) => symbol != null && reserved.has(symbol)) ? "library symbols reserved by the Wasm embedding ABI" - : profile.callbacks.some((cb) => cb.name === "panic") ? "a callback named 'panic' (reserved by the Wasm embedding ABI)" - : null; - if (surface !== null) return { ok: false, diagnostics: [targetRefusalDiag("wasm32-wasi", surface, { file: entryPath, start: 0, end: 0 })], sourceTexts: new Map() }; + const refusal = libraryWasmRefusal(profile, opts.sanitize ?? false); + if (refusal !== null) return { ok: false, diagnostics: [refusal], sourceTexts: new Map() }; } // Multi-instance library mode (abi.localize_runtime) localizes per @@ -2151,222 +1346,13 @@ async function compileLibraryTracked( } timing("semantic-cache-miss"); - // Bare npm specifiers in a library graph take the STATIC-OR-REFUSE - // posture: "lib" runs the same auto-detection and eligibility bar as - // the executable lane's --npm-static (own .d.ts, unminified shipped JS, - // no build-transform markers), automatically — the library path has no - // island/dynamic tier to offer. Explicit profile npm_static entries may - // also attempt source inference; any failed attempt remains a refusal. - const fe = runFrontend(entryPath, loadProgram, "lib", undefined, profile.npmStatic); - timing("frontend-load", { - entry_bytes: fe.entryText().length, - source_files: fe.sourceTexts().size, - }); - let lowered: LowerResult; - let sourceTexts: Map; - let entryInfo: Map; - let contractFacts: ContractFacts | null; - try { - // Every library refusal leaves through the ask-5 teaching decoration: - // profile text attaches by code, manifest id, or fence coverage as the - // attributed note (the SC4004/SC4005 rider generalized). - const fail = (diagnostics: ScrDiagnostic[]): CompileLibraryResult => ({ - ok: false, - diagnostics: decorateLibraryRefusals(diagnostics, profile), - sourceTexts: fe.sourceTexts(), - }); - // The npm verdicts FIRST: whatever the shared frontend would have - // served from the island — an eligibility miss, an untyped install, a - // preflight offender inside a package's files, a dropped inferred - // surface — refuses here with the package and the specific bar it - // missed. Checked before the general preflight, whose diagnostics for - // these same imports speak executable-lane teachings (SC1010/SC0001 at - // the unresolvable edge); the library answer is this one. - const npmRefused = fe.npmStatic.filter((s) => s.status === "fallback"); - if (npmRefused.length > 0) { - return fail( - npmRefused.map((s) => - libNpmIneligibleDiag( - s.package, - // The one shared offender reason that narrates the executable - // lane's fallback loses that clause here — no island exists on - // this path to serve anything. - (s.detail ?? "its static compilation was refused").replace("; the island serves the package", ""), - fe.npmImportSites.get(s.package) ?? { file: entryPath, start: 0, end: 0 }, - ), - ), - ); - } - if (fe.preflight.length > 0) return fail(fe.preflight); - contractFacts = profile.sidecar !== null ? fe.entryContract() : null; - // Ask 4, contract-surface reachability: when the sidecar declares ANY - // integer slot, the designated init/update/subscriptions exports and - // every contract helper (model-first exported function) seed lowering - // too. They are attested surface — a declared record-field or msg-arm - // class obligates EVERY write those bodies perform, and a declared - // helper param is checked at their internal call sites — so the - // attestation must cover COMPILED bodies, never a dead-stripped - // vacuity (the bug this closes: a model-slot declaration whose only - // writers were dead-stripped attested without any proof). - const contractSurfaceRoots: string[] = []; - if (profile.sidecar !== null && profile.sidecar.integerSlots.length > 0) { - const sc = profile.sidecar; - const fnNames = new Set(contractFacts!.functions.filter((f) => !f.generic).map((f) => f.name)); - for (const name of [sc.initExport, sc.updateExport, sc.subscriptionsExport]) { - if (fnNames.has(name)) contractSurfaceRoots.push(name); - } - for (const fn of contractFacts!.functions) { - if (fn.generic) continue; - const first = fn.params[0]; - if (first !== undefined && first.shape !== null && first.shape.k === "ref" && first.shape.name === sc.model) { - contractSurfaceRoots.push(fn.name); - } - } - } - // The profile's host-callback channels ride the FFI import machinery: - // each channel is a signature-only ambient binding whose direct calls - // lower to ffiCall nodes (the classes are a subset of the FFI's), and - // `libraryCallbacks` flips the recognition to the library flavor — - // SC4024 diagnostics, unused channels legal, undeclared references - // refused with the callback teaching. The library lane never loads a - // native-FFI manifest, so the channel set owns the surface outright. - const cbImports: IrFfiImport[] = profile.callbacks.map((cb) => ({ - name: cb.name, - symbol: cb.name, - params: [...cb.params], - returns: cb.returns, - })); - try { - lowered = fe.lower({ - dynamic: false, - targetPlatform: buildPlatform, - ...(cbImports.length > 0 ? { ffiImports: cbImports, libraryCallbacks: true } : {}), - // The profile-mapped exports are called from OUTSIDE the graph: - // they seed reachability beside the entry's top level (an - // executable build would dead-strip an uncalled export). A helper - // with a declared integer slot (ask 4) seeds too: its attestation - // must cover a COMPILED body, never a dead-stripped vacuity — the - // sidecar advertises the slot's class, so the proof must exist. - libRoots: [ - ...new Set([ - ...profile.exports.map((e) => e.export), - ...(profile.sidecar?.integerSlots ?? []) - .map((s) => /^helpers\.([^.]+)\.(?:params\[\d+\]|return)$/.exec(s.slot)?.[1]) - .filter((n): n is string => n !== undefined), - ...contractSurfaceRoots, - ]), - ], - }); - timing("lower", { - lib_roots: profile.exports.length + contractSurfaceRoots.length, - }); - } catch (e) { - if (!isCheckerPanic(e)) throw e; - return fail([checkerPanicDiag(e.message.split("\n", 1)[0]!, { file: entryPath, start: 0, end: 0 })]); - } - if (lowered.module === null) return fail(lowered.diagnostics); - entryInfo = fe.entryExports(); - sourceTexts = fe.sourceTexts(); - } finally { - fe.dispose(); - } - const mod = lowered.module!; - timing("frontend-dispose"); - + const prepared = prepareLibrary(profile, opts.profilePath, compilerReleaseVersion(), buildPlatform, nodeFrontend, timing); + if (!prepared.ok) return prepared; + const { mod, sourceTexts, sidecarJson } = prepared; const fail = (diagnostics: ScrDiagnostic[]): CompileLibraryResult => ({ - ok: false, - diagnostics: decorateLibraryRefusals(diagnostics, profile), - sourceTexts, + ok: false, diagnostics: decorateLibraryRefusals(diagnostics, profile), sourceTexts, }); - // Export resolution first (SC4002/SC4003/SC4004/SC4007 anchor at the - // mapped declaration — a mapped async export reports as SC4004, not the - // graph-wide gate), then the async_free requirement (ratified, SC4005), - // then the profile's determinism fences (ask 5, SC4008) over the same - // compiled graph the attestation scan reads: all refused before anything - // is emitted, so the narrowed library link set below is structural fact. - const resolved = resolveLibrarySection(profile, entryInfo, mod, entryPath); - if ("diagnostics" in resolved) return fail(resolved.diagnostics); - const asyncSurface = moduleLibAsyncSurface(mod); - if (asyncSurface !== null) { - return fail([libAsyncSurfaceDiag(asyncSurface.surface, asyncSurface.loc)]); - } - const fenced = evaluateLibraryFences(mod, profile); - if (fenced.length > 0) return fail(fenced); - mod.lib = resolved.lib; - if (buildPlatform === "wasi") { - const unavailable = moduleWasiUnavailableSurface(mod); - if (unavailable !== null) return fail([targetRefusalDiag("wasm32-wasi", unavailable.surface, unavailable.loc)]); - } - - // Ask 4's declared integer slots: the export map's i64/u64 classes - // seed the config here; sidecar-declared slots (record fields, msg - // arms, helper params/returns) merge in after the projection resolves - // them below. - let intCfg = libraryIntSlotConfig(profile); - - // The ask-2 contract sidecar rides the same invocation. Identity first - // (schema §2's worked build_id definition over compiler version, profile - // bytes, and the sorted canonical module graph; source_hash per the - // profile's "module-graph" contract) — the u64 lands on the IR so native - // archive assembly emits the identity getters from the ONE value the - // sidecar records (V12's coherence by construction), then the projection into - // the schema (declaration orders from the AST) and the V1–V14 - // self-check before anything is written. - let sidecarJson: string | null = null; - if (profile.sidecar !== null) { - const rootDir = dirname(resolve(opts.profilePath)); - const modules = canonicalModuleGraph(rootDir, sourceTexts); - const { buildId, sourceHash } = libraryIdentityHashes(compilerReleaseVersion(), profile.profileBytes, modules); - mod.lib.identity = { - buildIdSymbol: profile.sidecar.buildIdSymbol, - abiVersionSymbol: profile.sidecar.abiVersionSymbol, - buildId, - abiVersion: profile.sidecar.abiVersion, - }; - const built = buildSidecar({ - profile, - facts: contractFacts!, - compilerVersion: compilerReleaseVersion(), - entry: canonicalPath(rootDir, entryPath), - buildId, - sourceHash, - deterministic: moduleLibNondeterministicSurface(mod) === null, - }); - if (!built.ok) return fail(built.diagnostics); - const violations = validateSidecar(built.doc); - if (violations.length > 0) { - // The projection above refuses every user-caused shape; a rule - // violation surviving to here is an emitter bug. - return fail(violations.map((v) => iceDiag(`sidecar self-check failed — ${v}`, { file: entryPath, start: 0, end: 0 }))); - } - sidecarJson = built.json; - const merged = mergeSidecarIntSlots(intCfg, built.integerSlotFacts, mod); - if (!merged.ok) return fail([merged.diagnostic]); - intCfg = merged.config; - } - timing("contract-sidecar", { source_files: sourceTexts.size }); - - // Ask 4: the integer-boundary inference — every value that can reach a - // profile-declared i64/u64 slot must PROVE representability, wholeness, - // and range, or the build refuses with the failed obligation, the - // observed evidence, and the author's fix (SC4021/SC4022/SC4023). Runs - // only when at least one integer slot is declared; the sidecar (already - // built above, written only on success) may then attest the classes — - // §5's invariant that an attested integer class means the proof was - // discharged holds because no artifact leaves this function otherwise. - if (hasIntSlots(intCfg)) { - const refusals = checkLibraryIntegerSlots(mod, intCfg).filter((v) => v.outcome === "refuse"); - if (refusals.length > 0) { - return fail(refusals.map((v) => libIntBoundaryDiag(v.path, v.cls, v.obligation!, v.detail!, v.fix!, v.loc))); - } - } - timing("integer-proof"); - - const validation = validateModule(mod); - if (validation.length > 0) return fail(validation.map((v) => iceDiag(v.message, v.loc))); - timing("ir-validate"); - await mkdir(opts.outDir, { recursive: true }); const stem = basename(entryPath).replace(/\.(ts|mts|cts|js|mjs|cjs)$/, ""); const llvmPath = join(opts.outDir, `${stem}.lib.ll`); diff --git a/packages/compiler/src/ir/ir.ts b/packages/compiler/src/ir/ir.ts index 52bbbc53..b9cd5bcb 100644 --- a/packages/compiler/src/ir/ir.ts +++ b/packages/compiler/src/ir/ir.ts @@ -5702,7 +5702,7 @@ export type IrExpr = * undefined arm of its union (an optional field) is DROPPED from the * output — Node's rule for undefined-valued properties. The value is * BORROWED; the result string is owned (+1). Never throws. */ - | { kind: "jsonStringify"; value: IrExpr; type: IrType; loc: SrcLoc } + | { kind: "jsonStringify"; value: IrExpr; indent?: string; type: IrType; loc: SrcLoc } /** The dynamic-boundary check — a CHECKED cast `dynValue as T`: validate * the dyn value's JSON dyn against `type` (a non-dyn, JSON-representable * IR type) and BUILD the typed value (+1), or THROW a catchable diff --git a/packages/compiler/src/library/cache-primitives.ts b/packages/compiler/src/library/cache-primitives.ts index c38b59f3..4ce73b37 100644 --- a/packages/compiler/src/library/cache-primitives.ts +++ b/packages/compiler/src/library/cache-primitives.ts @@ -77,7 +77,7 @@ export function frontendOutputExclusions( if (dirname(directory) === directory) break; } } - return { outputPaths: outputArtifacts, outputDirectories }; + return { outputPaths: outputArtifacts, outputDirectories: [...outputDirectories] }; } export async function readCachedFile(path: string, expected: string): Promise { diff --git a/packages/compiler/src/library/fence-eval.ts b/packages/compiler/src/library/fence-eval.ts index 01345efd..1843d84a 100644 --- a/packages/compiler/src/library/fence-eval.ts +++ b/packages/compiler/src/library/fence-eval.ts @@ -439,14 +439,14 @@ export function evaluateLibraryFences(mod: IrModule, profile: FenceProfileView): /* ── the generalized teaching rider ────────────────────────────────────── */ function surfaceMatchesDiag( - surface: { code?: string; name: string; kind: SurfaceEntryKind }, + surface: { code?: string | undefined; name: string; kind: SurfaceEntryKind }, diag: ScrDiagnostic, ): boolean { if (surface.code === undefined || surface.code !== diag.code) return false; // A diagnostic-fence entry IS the code family. Method refusals use a // receiver description instead of the manifest's prototype spelling. if (surface.kind === "diagnostic-fence" || diag.message.includes(surface.name)) return true; - for (const [prefix, receiver] of [["number.prototype.", "numbers"], ["string.prototype.", "strings"]] as const) { + for (const [prefix, receiver] of [["number.prototype.", "numbers"], ["string.prototype.", "strings"]] as [string, string][]) { if (surface.name.startsWith(prefix)) { const method = surface.name.slice(prefix.length); return diag.message.includes(`'.${method}()' on ${receiver}`); @@ -469,14 +469,14 @@ function teachingForRefusal(profile: FenceProfileView, diag: ScrDiagnostic): str for (const [key, text] of Object.entries(profile.teachings)) { if (!key.includes(".")) continue; const entry = fenceTaxonomy().byId.get(key); - if (entry !== undefined && surfaceMatchesDiag(entry, diag)) return text; + if (entry !== undefined && surfaceMatchesDiag({ code: entry.code, name: entry.name, kind: entry.kind }, diag)) return text; } // 3. A fence covering the refused surface: the teaching rides the // surface's own refusal (the non-static half of fence coverage). for (const fence of profile.fences) { if (fence.teaching === undefined) continue; for (const s of fence.surfaces) { - if (surfaceMatchesDiag(s, diag)) return fence.teaching; + if (surfaceMatchesDiag({ code: s.code, name: s.name, kind: s.kind }, diag)) return fence.teaching; } } return undefined; diff --git a/packages/compiler/src/library/int-infer.ts b/packages/compiler/src/library/int-infer.ts index 794975b5..6d15c79c 100644 --- a/packages/compiler/src/library/int-infer.ts +++ b/packages/compiler/src/library/int-infer.ts @@ -613,8 +613,7 @@ function globalEffectsOf(mod: IrModule): GlobalEffects { default: break; } - for (const key of Object.keys(e) as (keyof typeof e)[]) { - const v = e[key] as unknown; + for (const [key, v] of Object.entries(e as unknown as Record)) { if (Array.isArray(v)) { for (const item of v) { if (item !== null && typeof item === "object" && "kind" in (item as object)) { @@ -646,7 +645,7 @@ function globalEffectsOf(mod: IrModule): GlobalEffects { default: break; } - for (const v of Object.values(s) as unknown[]) { + for (const v of Object.values(s as unknown as Record)) { if (Array.isArray(v)) { for (const item of v) { if (item !== null && typeof item === "object" && typeof (item as { kind?: unknown }).kind === "string") { @@ -684,7 +683,7 @@ function globalEffectsOf(mod: IrModule): GlobalEffects { unknown.add(name); changed = true; } - for (const g of writes.get(callee) ?? []) { + for (const g of writes.get(callee) ?? new Set()) { if (!w.has(g)) { w.add(g); changed = true; @@ -936,7 +935,8 @@ class FnAnalyzer { // could select. Multiple classified fields intentionally emit // independent obligations (their classes and paths may differ). if (s.overflowOnly !== true) { - for (const slot of this.cfg.records.get(s.shapeId)?.values() ?? []) { + const recordSlots = this.cfg.records.get(s.shapeId); + for (const slot of recordSlots === undefined ? [] : [...recordSlots.values()]) { this.emitRecordSlot(v, slot, s.loc); } } @@ -1024,7 +1024,7 @@ class FnAnalyzer { const assigned = new Set(); const visitStmt = (s: IrStmt): void => { if (s.kind === "assign" || s.kind === "varDecl") assigned.add(s.localId); - for (const v of Object.values(s) as unknown[]) { + for (const v of Object.values(s as unknown as Record)) { if (Array.isArray(v)) { for (const item of v) { if (item !== null && typeof item === "object" && typeof (item as { kind?: unknown }).kind === "string") { @@ -1046,7 +1046,7 @@ class FnAnalyzer { const visitExpr = (e: IrExpr): void => { if (e.kind === "assignExpr" || e.kind === "incDec") assigned.add(e.localId); if (e.kind === "seqExpr") e.stmts.forEach(visitStmt); - for (const v of Object.values(e) as unknown[]) { + for (const v of Object.values(e as unknown as Record)) { if (Array.isArray(v)) { for (const item of v) { if (item !== null && typeof item === "object" && typeof (item as { kind?: unknown }).kind === "string") { @@ -1579,7 +1579,8 @@ class FnAnalyzer { }); } this.havocCall(e.callee, env); - if (slots?.ret != null) return classSeed(slots.ret); + const returnSlot = slots?.ret; + if (returnSlot != null) return classSeed(returnSlot); return { ...TOP }; } case "unionWrap": { @@ -1716,7 +1717,7 @@ function typeContainsFunc(t: unknown): boolean { * order for the shapes we don't model precisely). */ function childExprs(e: IrExpr): IrExpr[] { const out: IrExpr[] = []; - for (const [key, v] of Object.entries(e)) { + for (const [key, v] of Object.entries(e as unknown as Record)) { if (key === "type") continue; if (Array.isArray(v)) { for (const item of v) { diff --git a/packages/compiler/src/library/library-profile.ts b/packages/compiler/src/library/library-profile.ts index 1f67ad6e..ac16856e 100644 --- a/packages/compiler/src/library/library-profile.ts +++ b/packages/compiler/src/library/library-profile.ts @@ -442,7 +442,7 @@ class ProfileError extends Error { } function req(v: unknown, path: string, kind: "string" | "number" | "boolean"): T { - if (typeof v !== kind) { + if (!(kind === "string" ? typeof v === "string" : kind === "number" ? typeof v === "number" : typeof v === "boolean")) { throw new ProfileError(`'${path}' must be a ${kind}${v === undefined ? " (missing)" : ""}`); } return v as T; @@ -513,7 +513,7 @@ export function loadLibraryProfile( if (!Array.isArray(npmStaticRaw) || npmStaticRaw.some((name) => typeof name !== "string" || !/^(?:@[a-z0-9_.-]+\/)?[a-z0-9_][a-z0-9_.-]*$/.test(name))) { throw new ProfileError("'npm_static' must be an array of npm package names (not subpath specifiers)"); } - const npmStatic = [...new Set(npmStaticRaw)] as string[]; + const npmStatic = [...new Set(npmStaticRaw as string[])]; if (name === "") throw new ProfileError("'name' must be a non-empty identity string"); const entryRel = req(p["entry"], "entry", "string"); if (entryRel === "") throw new ProfileError("'entry' must name the profile's one entry module"); @@ -868,11 +868,12 @@ export function loadLibraryProfile( if (prefix === "") throw new ProfileError(`'${path}.prefix' must be a non-empty manifest id prefix`); decl.prefix = prefix; } - for (const rider of ["teaching", "remediation"] as const) { + for (const rider of ["teaching", "remediation"] as ("teaching" | "remediation")[]) { if (ff[rider] === undefined) continue; const text = req(ff[rider], `${path}.${rider}`, "string"); checkRiderText(text, `${path}.${rider}`); - decl[rider] = text; + if (rider === "teaching") decl.teaching = text; + else decl.remediation = text; } return decl; }); diff --git a/packages/compiler/src/library/prepare.ts b/packages/compiler/src/library/prepare.ts new file mode 100644 index 00000000..b40e5b0b --- /dev/null +++ b/packages/compiler/src/library/prepare.ts @@ -0,0 +1,665 @@ +import { dirname, resolve } from "node:path"; +import type { CompileFailure } from "../compile-types.js"; +import { moduleWasiUnavailableSurface, targetRefusalDiag } from "../backend/target-diagnostics.js"; +import { checkerPanicDiag, libAsyncExportDiag, libAsyncSurfaceDiag, libExportUnresolvedDiag, libGenericExportDiag, libIntBoundaryDiag, libNpmIneligibleDiag, libSidecarDiag, libUnmappableSignatureDiag, iceDiag, isCheckerPanic, LIB_INBOUND_BYTES_TRAP_CODE, LIB_RUNTIME_TRAP_CODES, type ScrDiagnostic } from "../diagnostics/diagnostic.js"; +import { checkLibraryIntegerSlots, classSeed, hasIntSlots, numberCarrierKind, type FnIntSlots, type IntSlotConfig } from "./int-infer.js"; +import { profileRemediation, profileTeaching, type LibraryProfile } from "./library-profile.js"; +import { decorateLibraryRefusals, evaluateLibraryFences } from "./fence-eval.js"; +import { assembleTrapTeaching } from "./trap-teaching.js"; +import { buildSidecar, canonicalModuleGraph, canonicalPath, libraryIdentityHashes, type SidecarIntegerSlotFacts, type SidecarIrRecordPattern, type SidecarIrTypePattern } from "./sidecar.js"; +import { validateSidecar } from "./sidecar-validate.js"; +import type { EntryExportInfo } from "../frontend/lib-exports.js"; +import type { ContractFacts } from "../frontend/lib-contract.js"; +import type { LowerResult } from "../frontend/lowering/lowerer.js"; +import type { FrontendFactory } from "../frontend/pipeline.js"; +import { moduleLibAsyncSurface, moduleLibNondeterministicSurface, type IrFfiImport, type IrLibSection, type IrModule, type IrRecordShape, type IrType } from "../ir/ir.js"; +import { validateModule } from "../ir/validate.js"; + +/** The marshalling-class fit over IR types (design §4.2 + the ratified + * integer plumbing classes): number is every f64-backed class, bool/string + * map directly, bytes is the u8 element kind. */ +function libClassFits(cls: string, t: IrType): boolean { + switch (cls) { + case "bool": + return t.kind === "bool"; + case "string": + return t.kind === "string"; + case "bytes": + return t.kind === "bytes" && t.elem === "u8"; + default: // f64 and the u8/u32/i32 plumbing classes + return t.kind === "f64"; + } +} + +/** Resolve the profile's export map against the entry module — SC4002/ + * SC4004/SC4007 from the declaration facts, SC4003 from the lowered IR + * signatures — and land the library section on the module. */ +function resolveLibrarySection( + profile: LibraryProfile, + entryInfo: Map, + mod: IrModule, + entryPath: string, +): { lib: IrLibSection } | { diagnostics: ScrDiagnostic[] } { + const diagnostics: ScrDiagnostic[] = []; + const entryLoc = { file: entryPath, start: 0, end: 0 }; + const fnByName = new Map(mod.functions.map((f) => [f.name, f])); + const exports: IrLibSection["exports"] = []; + for (const e of profile.exports) { + const info = entryInfo.get(e.export); + if (info === undefined) { + diagnostics.push( + libExportUnresolvedDiag(e.export, "the entry module has no exported function declaration by that name", entryLoc), + ); + continue; + } + if (info.generic) { + diagnostics.push(libGenericExportDiag(e.export, info.loc)); + continue; + } + if (info.async || info.generator) { + diagnostics.push(libAsyncExportDiag(e.export, info.async ? "async" : "generator", info.loc)); + continue; + } + const fn = fnByName.get(e.export); + if (fn === undefined) { + diagnostics.push( + libExportUnresolvedDiag(e.export, "the export did not lower to a compiled function", info.loc), + ); + continue; + } + if (fn.params.length !== e.params.length) { + diagnostics.push( + libUnmappableSignatureDiag( + e.export, + "signature", + `has ${fn.params.length} parameter(s) but the profile declares ${e.params.length} marshalling class(es)`, + info.loc, + ), + ); + continue; + } + let bad = false; + e.params.forEach((cls, i) => { + if (!libClassFits(cls, fn.params[i]!.type)) { + bad = true; + diagnostics.push( + libUnmappableSignatureDiag( + e.export, + `parameter ${i + 1} ('${fn.params[i]!.name}')`, + `has IR type '${fn.params[i]!.type.kind}', which does not fit the declared marshalling class '${cls}'`, + info.loc, + ), + ); + } + }); + if (e.returns === "void" ? fn.returnType.kind !== "void" : !libClassFits(e.returns, fn.returnType)) { + bad = true; + diagnostics.push( + libUnmappableSignatureDiag( + e.export, + "the return", + `has IR type '${fn.returnType.kind}', which does not fit the declared marshalling class '${e.returns}'`, + info.loc, + ), + ); + } + if (!bad) { + const resolvedExport: IrLibSection["exports"][number] = { + symbol: e.symbol, + fnName: e.export, + params: e.params, + returns: e.returns, + }; + if (e.params.includes("bytes")) { + // The wrapper's one host-contract trap (an inbound bytes length + // past the marshalling class's range) is assembled HERE, once, as + // the structured trap-teaching message: the profile's teaching for + // SC4012 (or the mode's default text), the code, the trapping + // export's C symbol exactly as the host linked it, and the + // profile's remediation when supplied — so the backend emits the + // same bytes and the sink sees one canonical message. + resolvedExport.inboundBytesTrap = assembleTrapTeaching( + profileTeaching(profile, LIB_INBOUND_BYTES_TRAP_CODE) ?? + "scriptc: library inbound bytes length out of range\n", + LIB_INBOUND_BYTES_TRAP_CODE, + e.symbol, + profileRemediation(profile, LIB_INBOUND_BYTES_TRAP_CODE), + ); + } + if (e.params.includes("i64") || e.params.includes("u64")) { + // The sibling host-contract trap for inbound declared-integer + // parameters (ask 4): a value past ±(2^53−1) cannot ride f64 + // exactly, and silent rounding is a coercion the author never + // wrote. Same code (SC4012 — one host-contract story), same + // assembly-once discipline. + resolvedExport.inboundIntTrap = assembleTrapTeaching( + profileTeaching(profile, LIB_INBOUND_BYTES_TRAP_CODE) ?? + "scriptc: library inbound integer parameter out of range\n", + LIB_INBOUND_BYTES_TRAP_CODE, + e.symbol, + profileRemediation(profile, LIB_INBOUND_BYTES_TRAP_CODE), + ); + } + exports.push(resolvedExport); + } + } + if (diagnostics.length > 0) return { diagnostics }; + // The runtime detected-trap overlay rows: one per family code the profile + // declares teaching or remediation text for, in the registry family's + // order. LLVM emits these rows as the program TU's overlay table, + // which the runtime uses to assemble the sink message. (SC4012 stays compile-time + // assembled into the wrapper's message above and never reaches the + // funnel's assembly path.) + const trapOverlays: IrLibSection["trapOverlays"] = []; + for (const code of LIB_RUNTIME_TRAP_CODES) { + const teaching = profileTeaching(profile, code); + const remediation = profileRemediation(profile, code); + if (teaching !== undefined || remediation !== undefined) { + trapOverlays.push({ + code, + ...(teaching !== undefined ? { teaching } : {}), + ...(remediation !== undefined ? { remediation } : {}), + }); + } + } + const lib: IrLibSection = { + profileName: profile.name, + prefix: profile.prefix, + initSymbol: profile.initSymbol, + sinkRegisterSymbol: profile.sinkRegisterSymbol, + collectSymbol: profile.collectSymbol, + resultResetSymbol: profile.resultResetSymbol, + threadInstances: profile.instancePerThread, + // Host-callback channels: declaration order is the runtime slot + // assignment, and the unregistered-call trap text is assembled HERE, + // once, for consistent constant bytes (a DETECTED + // trap: the funnel classifies the "scriptc: library callback " + // prefix as SC4025 and names the entry the host called — the entry + // is runtime knowledge, so no compile-time SC4012-style assembly + // can carry it). Both fields stay absent on callback-free profiles + // (the byte-identity guarantee). + + exports, + trapOverlays, + }; + if (profile.callbacks.length > 0) { + lib.callbackRegisterSymbol = profile.callbackRegisterSymbol!; + lib.callbacks = profile.callbacks.map((cb, slot) => ({ + name: cb.name, slot, params: [...cb.params], returns: cb.returns, + unregisteredTrap: `scriptc: library callback '${cb.name}' invoked before registration\n`, + })); + } + return { lib }; +} + +/** The export map's integer-slot obligations (ask 4): i64/u64 params and + * returns become declared boundary slots keyed `exports..params[i]` + * / `exports..return`; the u8/u32/i32 plumbing classes contribute + * their proven inbound shapes as parameter seeds (the wrapper's coercion + * contract), tightening the intraprocedural analysis at zero declaration + * cost. Sidecar-declared slots (record fields, msg arms, helper params + * and returns) merge into the same config at sidecar build. */ +function libraryIntSlotConfig(profile: LibraryProfile): IntSlotConfig { + const cfg: IntSlotConfig = { fns: new Map(), records: new Map() }; + for (const e of profile.exports) { + const params = e.params.map((c) => (c === "i64" || c === "u64" ? c : null)); + const ret = e.returns === "i64" || e.returns === "u64" ? e.returns : null; + const paramSeeds = e.params.map((c) => (c === "u8" || c === "u32" || c === "i32" ? classSeed(c) : null)); + if (params.every((p) => p === null) && ret === null && paramSeeds.every((s) => s === null)) continue; + const slots: FnIntSlots = { + fnName: e.export, + params, + paramPaths: e.params.map((c, i) => (c === "i64" || c === "u64" ? `exports.${e.export}.params[${i}]` : null)), + ret, + retPath: ret !== null ? `exports.${e.export}.return` : null, + paramSeeds, + }; + cfg.fns.set(e.export, slots); + } + return cfg; +} + +/** Match the sidecar syntax's exact structural type projection against the + * frontend's interned IR registries. The pattern deliberately mirrors + * ShapeRegistry's identity: every field name and recursively mapped field + * type participates. Tagged payload records additionally accept omission + * of their `kind` field because the lowering may carry that discriminant + * only in the surrounding union tag. */ +function sidecarRecordMatcher( + mod: IrModule, +): (pattern: SidecarIrRecordPattern, shape: IrRecordShape) => boolean { + const records = new Map((mod.records ?? []).map((shape) => [shape.id, shape])); + const unions = new Map((mod.unions ?? []).map((union) => [union.id, union])); + + const recordMatches = ( + pattern: SidecarIrRecordPattern, + shape: IrRecordShape, + ): boolean => { + if (shape.tuple === true || shape.indexValue !== undefined) return false; + const variants = [pattern.fields]; + if (pattern.kindMayBeOmitted === true) { + variants.push(pattern.fields.filter((field) => field.name !== "kind")); + } + return variants.some( + (fields) => + fields.length === shape.fields.length && + fields.every((field) => { + const actual = shape.fields.find((candidate) => candidate.name === field.name); + return actual !== undefined && typeMatches(field.type, actual.type); + }), + ); + }; + + const unionMatches = ( + patterns: SidecarIrTypePattern[], + actual: IrType[], + ): boolean => { + if (patterns.length !== actual.length) return false; + const used = new Set(); + const visit = (index: number): boolean => { + if (index === patterns.length) return true; + for (let i = 0; i < actual.length; i++) { + if (used.has(i) || !typeMatches(patterns[index]!, actual[i]!)) continue; + used.add(i); + if (visit(index + 1)) return true; + used.delete(i); + } + return false; + }; + return visit(0); + }; + + const typeMatches = ( + pattern: SidecarIrTypePattern, + actual: IrType, + ): boolean => { + switch (pattern.kind) { + case "f64": + case "string": + case "bool": + case "nullT": + case "undefinedT": + case "dyn": + return actual.kind === pattern.kind; + case "bytes": + return actual.kind === "bytes" && actual.elem === pattern.elem; + case "array": + return actual.kind === "array" && typeMatches(pattern.elem, actual.elem); + case "record": { + if (actual.kind !== "record") return false; + const shape = records.get(actual.shapeId); + return shape !== undefined && recordMatches(pattern, shape); + } + case "union": { + if (actual.kind !== "union") return false; + const union = unions.get(actual.unionId); + return union !== undefined && unionMatches(pattern.arms, union.arms); + } + } + }; + + return (pattern, shape) => recordMatches(pattern, shape); +} + +/** Merge the sidecar-resolved integer slots (ask 4) into the inference + * config: helper slots key by function name and IR parameter index (the + * projection already shifted past the model receiver); record-field + * slots map onto every interned IR shape whose complete structural field + * signature matches the projected record's. Shapes intern structurally, + * so a same-shaped second type shares the obligation. DECLARED paths with + * the same class coalesce while retaining every source path for verdicts; + * differing classes refuse because one lowered field cannot seed or check + * two distinct class contracts without arm provenance. A + * record fact that matches no shape binds nothing: no compiled code + * constructs the type (the contract surface — init/update/subscriptions + * and every helper — is force-lowered whenever integer slots are + * declared, so this is genuine vacuity, not dead-stripping). */ +function mergeSidecarIntSlots( + cfg: IntSlotConfig, + facts: SidecarIntegerSlotFacts, + mod: IrModule, +): { ok: true; config: IntSlotConfig } | { ok: false; diagnostic: ScrDiagnostic } { + const recordMatches = sidecarRecordMatcher(mod); + for (const h of facts.helpers) { + const fn = mod.functions.find((f) => f.name === h.fnName); + const arity = Math.max(fn?.params.length ?? 0, (h.index ?? 0) + 1); + let slots = cfg.fns.get(h.fnName); + if (slots === undefined) { + slots = { + fnName: h.fnName, + params: new Array(arity).fill(null), + paramPaths: new Array(arity).fill(null), + ret: null, + retPath: null, + paramSeeds: new Array(arity).fill(null), + }; + cfg.fns.set(h.fnName, slots); + } + if (h.kind === "param") { + const i = h.index!; + while (slots.params.length <= i) { + slots.params.push(null); + slots.paramPaths.push(null); + slots.paramSeeds.push(null); + } + slots.params[i] = h.cls; + slots.paramPaths[i] = h.path; + } else { + slots.ret = h.cls; + slots.retPath = h.path; + } + } + for (const r of facts.records) { + for (const shape of mod.records ?? []) { + if (!recordMatches(r.shape, shape)) continue; + const target = shape.fields.find((f) => f.name === r.targetField); + if (target === undefined || numberCarrierKind(target.type, mod) === null) continue; + let m = cfg.records.get(shape.id); + if (m === undefined) { + m = new Map(); + cfg.records.set(shape.id, m); + } + const existing = m.get(r.targetField); + if (existing !== undefined && existing.cls !== r.cls) { + const paths = [ + ...existing.paths.map((path) => `'${path}' (${existing.cls})`), + `'${r.path}' (${r.cls})`, + ]; + return { + ok: false, + diagnostic: libSidecarDiag( + `integer slots ${paths.join(" and ")} collapse to the same lowered record field '${r.targetField}' — their proof obligations cannot be kept distinct`, + r.loc, + "kind-tagged union arms and structurally identical records may share one lowered shape — same-class declarations coalesce, but differing classes require distinct structural shapes or at most one classified slot", + ), + }; + } + if (existing === undefined) { + m.set(r.targetField, { cls: r.cls, paths: [r.path] }); + } else if (!existing.paths.includes(r.path)) { + existing.paths.push(r.path); + } + } + } + return { ok: true, config: cfg }; +} + +export interface PreparedLibrary { + ok: true; + mod: IrModule; + sourceTexts: Map; + sidecarJson: string | null; +} + +export function prepareLibrary( + profile: LibraryProfile, + profilePath: string, + compilerVersion: string, + buildPlatform: string, + createFrontend: FrontendFactory, + timing: (phase: string, detail?: Record) => void = () => {}, +): PreparedLibrary | CompileFailure { + const entryPath = profile.entry; + // Bare npm specifiers in a library graph take the STATIC-OR-REFUSE + // posture: "lib" runs the same auto-detection and eligibility bar as + // the executable lane's --npm-static (own .d.ts, unminified shipped JS, + // no build-transform markers), automatically — the library path has no + // island/dynamic tier to offer. Explicit profile npm_static entries may + // also attempt source inference; any failed attempt remains a refusal. + const fe = createFrontend(entryPath, "lib", undefined, profile.npmStatic); + timing("frontend-load", { + entry_bytes: fe.entryText().length, + source_files: fe.sourceTexts().size, + }); + let lowered: LowerResult; + let sourceTexts: Map; + let entryInfo: Map; + let contractFacts: ContractFacts | null; + try { + // Every library refusal leaves through the ask-5 teaching decoration: + // profile text attaches by code, manifest id, or fence coverage as the + // attributed note (the SC4004/SC4005 rider generalized). + const fail = (diagnostics: ScrDiagnostic[]): CompileFailure => ({ + ok: false, + diagnostics: decorateLibraryRefusals(diagnostics, profile), + sourceTexts: fe.sourceTexts(), + }); + // The npm verdicts FIRST: whatever the shared frontend would have + // served from the island — an eligibility miss, an untyped install, a + // preflight offender inside a package's files, a dropped inferred + // surface — refuses here with the package and the specific bar it + // missed. Checked before the general preflight, whose diagnostics for + // these same imports speak executable-lane teachings (SC1010/SC0001 at + // the unresolvable edge); the library answer is this one. + const npmRefused = fe.npmStatic.filter((s) => s.status === "fallback"); + if (npmRefused.length > 0) { + return fail( + npmRefused.map((s) => + libNpmIneligibleDiag( + s.package, + // The one shared offender reason that narrates the executable + // lane's fallback loses that clause here — no island exists on + // this path to serve anything. + (s.detail ?? "its static compilation was refused").replace("; the island serves the package", ""), + fe.npmImportSites.get(s.package) ?? { file: entryPath, start: 0, end: 0 }, + ), + ), + ); + } + if (fe.preflight.length > 0) return fail(fe.preflight); + contractFacts = profile.sidecar !== null ? fe.entryContract() : null; + // Ask 4, contract-surface reachability: when the sidecar declares ANY + // integer slot, the designated init/update/subscriptions exports and + // every contract helper (model-first exported function) seed lowering + // too. They are attested surface — a declared record-field or msg-arm + // class obligates EVERY write those bodies perform, and a declared + // helper param is checked at their internal call sites — so the + // attestation must cover COMPILED bodies, never a dead-stripped + // vacuity (the bug this closes: a model-slot declaration whose only + // writers were dead-stripped attested without any proof). + const contractSurfaceRoots: string[] = []; + if (profile.sidecar !== null && profile.sidecar.integerSlots.length > 0) { + const sc = profile.sidecar; + const fnNames = new Set(contractFacts!.functions.filter((f) => !f.generic).map((f) => f.name)); + for (const name of [sc.initExport, sc.updateExport, sc.subscriptionsExport]) { + if (fnNames.has(name)) contractSurfaceRoots.push(name); + } + for (const fn of contractFacts!.functions) { + if (fn.generic) continue; + const first = fn.params[0]; + if (first !== undefined && first.shape !== null && first.shape.k === "ref" && first.shape.name === sc.model) { + contractSurfaceRoots.push(fn.name); + } + } + } + // The profile's host-callback channels ride the FFI import machinery: + // each channel is a signature-only ambient binding whose direct calls + // lower to ffiCall nodes (the classes are a subset of the FFI's), and + // `libraryCallbacks` flips the recognition to the library flavor — + // SC4024 diagnostics, unused channels legal, undeclared references + // refused with the callback teaching. The library lane never loads a + // native-FFI manifest, so the channel set owns the surface outright. + const cbImports: IrFfiImport[] = profile.callbacks.map((cb) => ({ + name: cb.name, + symbol: cb.name, + params: [...cb.params], + returns: cb.returns, + })); + const integerSlotRoots: string[] = []; + for (const slot of profile.sidecar?.integerSlots ?? []) { + const name = /^helpers\.([^.]+)\.(?:params\[\d+\]|return)$/.exec(slot.slot)?.[1]; + if (name !== undefined) integerSlotRoots.push(name); + } + try { + lowered = fe.lower({ + dynamic: false, + targetPlatform: buildPlatform, + ...(cbImports.length > 0 ? { ffiImports: cbImports } : {}), + ...(cbImports.length > 0 ? { libraryCallbacks: true } : {}), + // The profile-mapped exports are called from OUTSIDE the graph: + // they seed reachability beside the entry's top level (an + // executable build would dead-strip an uncalled export). A helper + // with a declared integer slot (ask 4) seeds too: its attestation + // must cover a COMPILED body, never a dead-stripped vacuity — the + // sidecar advertises the slot's class, so the proof must exist. + libRoots: [ + ...new Set([ + ...profile.exports.map((e) => e.export), + ...integerSlotRoots, + ...contractSurfaceRoots, + ]), + ], + }); + timing("lower", { + lib_roots: profile.exports.length + contractSurfaceRoots.length, + }); + } catch (e) { + if (!isCheckerPanic(e)) throw e; + return fail([checkerPanicDiag(e.message.split("\n", 1)[0]!, { file: entryPath, start: 0, end: 0 })]); + } + if (lowered.module === null) return fail(lowered.diagnostics); + entryInfo = fe.entryExports(); + sourceTexts = fe.sourceTexts(); + } finally { + fe.dispose(); + } + const mod = lowered.module!; + timing("frontend-dispose"); + + const fail = (diagnostics: ScrDiagnostic[]): CompileFailure => ({ + ok: false, + diagnostics: decorateLibraryRefusals(diagnostics, profile), + sourceTexts, + }); + + // Export resolution first (SC4002/SC4003/SC4004/SC4007 anchor at the + // mapped declaration — a mapped async export reports as SC4004, not the + // graph-wide gate), then the async_free requirement (ratified, SC4005), + // then the profile's determinism fences (ask 5, SC4008) over the same + // compiled graph the attestation scan reads: all refused before anything + // is emitted, so the narrowed library link set below is structural fact. + const resolved = resolveLibrarySection(profile, entryInfo, mod, entryPath); + if ("diagnostics" in resolved) return fail(resolved.diagnostics); + const asyncSurface = moduleLibAsyncSurface(mod); + if (asyncSurface !== null) { + return fail([libAsyncSurfaceDiag(asyncSurface.surface, asyncSurface.loc)]); + } + const fenced = evaluateLibraryFences(mod, profile); + if (fenced.length > 0) return fail(fenced); + mod.lib = resolved.lib; + if (buildPlatform === "wasi") { + const unavailable = moduleWasiUnavailableSurface(mod); + if (unavailable !== null) return fail([targetRefusalDiag("wasm32-wasi", unavailable.surface, unavailable.loc)]); + } + + // Ask 4's declared integer slots: the export map's i64/u64 classes + // seed the config here; sidecar-declared slots (record fields, msg + // arms, helper params/returns) merge in after the projection resolves + // them below. + let intCfg = libraryIntSlotConfig(profile); + + // The ask-2 contract sidecar rides the same invocation. Identity first + // (schema §2's worked build_id definition over compiler version, profile + // bytes, and the sorted canonical module graph; source_hash per the + // profile's "module-graph" contract) — the u64 lands on the IR so native + // archive assembly emits the identity getters from the ONE value the + // sidecar records (V12's coherence by construction), then the projection into + // the schema (declaration orders from the AST) and the V1–V14 + // self-check before anything is written. + let sidecarJson: string | null = null; + if (profile.sidecar !== null) { + const rootDir = dirname(resolve(profilePath)); + const modules = canonicalModuleGraph(rootDir, sourceTexts); + const { buildId, sourceHash } = libraryIdentityHashes(compilerVersion, profile.profileBytes, modules); + mod.lib.identity = { + buildIdSymbol: profile.sidecar.buildIdSymbol, + abiVersionSymbol: profile.sidecar.abiVersionSymbol, + buildId, + abiVersion: profile.sidecar.abiVersion, + }; + const built = buildSidecar({ + profile, + facts: contractFacts!, + compilerVersion: compilerVersion, + entry: canonicalPath(rootDir, entryPath), + buildId, + sourceHash, + deterministic: moduleLibNondeterministicSurface(mod) === null, + }); + if (!built.ok) return fail(built.diagnostics); + // Validate the serialized contract, including its omitted optional + // properties, through the same checked-value path used by consumers. + const violations = validateSidecar(JSON.parse(built.json)); + if (violations.length > 0) { + // The projection above refuses every user-caused shape; a rule + // violation surviving to here is an emitter bug. + return fail(violations.map((v) => iceDiag(`sidecar self-check failed — ${v}`, { file: entryPath, start: 0, end: 0 }))); + } + sidecarJson = built.json; + const merged = mergeSidecarIntSlots(intCfg, built.integerSlotFacts, mod); + if (!merged.ok) return fail([merged.diagnostic]); + intCfg = merged.config; + } + timing("contract-sidecar", { source_files: sourceTexts.size }); + + // Ask 4: the integer-boundary inference — every value that can reach a + // profile-declared i64/u64 slot must PROVE representability, wholeness, + // and range, or the build refuses with the failed obligation, the + // observed evidence, and the author's fix (SC4021/SC4022/SC4023). Runs + // only when at least one integer slot is declared; the sidecar (already + // built above, written only on success) may then attest the classes — + // §5's invariant that an attested integer class means the proof was + // discharged holds because no artifact leaves this function otherwise. + if (hasIntSlots(intCfg)) { + const refusals = checkLibraryIntegerSlots(mod, intCfg).filter((v) => v.outcome === "refuse"); + if (refusals.length > 0) { + return fail(refusals.map((v) => libIntBoundaryDiag(v.path, v.cls, v.obligation!, v.detail!, v.fix!, v.loc))); + } + } + timing("integer-proof"); + + const validation = validateModule(mod); + if (validation.length > 0) return fail(validation.map((v) => iceDiag(v.message, v.loc))); + timing("ir-validate"); + + return { ok: true, mod, sourceTexts, sidecarJson }; +} + +export function libraryLocalizeSymbols(profile: LibraryProfile): string[] | undefined { + return profile.localizeRuntime + ? [ + profile.initSymbol, + profile.sinkRegisterSymbol, + ...(profile.collectSymbol !== null ? [profile.collectSymbol] : []), + ...(profile.resultResetSymbol !== null ? [profile.resultResetSymbol] : []), + ...(profile.callbackRegisterSymbol !== null ? [profile.callbackRegisterSymbol] : []), + ...(profile.sidecar !== null + ? [profile.sidecar.buildIdSymbol, profile.sidecar.abiVersionSymbol] + : []), + ...profile.exports.map((entry) => entry.symbol), + ] + : undefined; +} + +export function libraryWasmExports(profile: LibraryProfile): string[] { + return [ + profile.initSymbol, "scriptc_alloc", "scriptc_free", + ...(profile.collectSymbol === null ? [] : [profile.collectSymbol]), + ...(profile.resultResetSymbol === null ? [] : [profile.resultResetSymbol]), + ...(profile.sidecar === null ? [] : [profile.sidecar.buildIdSymbol, profile.sidecar.abiVersionSymbol]), + ...profile.exports.map((entry) => entry.symbol), + ]; +} + +export function libraryWasmRefusal(profile: LibraryProfile, sanitize: boolean): ScrDiagnostic | null { + const reserved = new Set(["scriptc_alloc", "scriptc_free", "memory", "_initialize", "_start"]); + const symbols = [profile.initSymbol, profile.sinkRegisterSymbol, profile.collectSymbol, profile.resultResetSymbol, + profile.callbackRegisterSymbol, ...profile.exports.map((entry) => entry.symbol), profile.sidecar?.buildIdSymbol, profile.sidecar?.abiVersionSymbol]; + const surface = sanitize ? "sanitized library builds" + : profile.instancePerThread ? "thread-instanced libraries (instantiate separate Wasm instances instead)" + : profile.localizeRuntime ? "runtime localization (Wasm instances already isolate their runtime)" + : symbols.some((symbol) => symbol != null && reserved.has(symbol)) ? "library symbols reserved by the Wasm embedding ABI" + : profile.callbacks.some((cb) => cb.name === "panic") ? "a callback named 'panic' (reserved by the Wasm embedding ABI)" + : null; + return surface === null ? null : targetRefusalDiag("wasm32-wasi", surface, { file: profile.entry, start: 0, end: 0 }); +} diff --git a/packages/compiler/src/library/sidecar-validate.ts b/packages/compiler/src/library/sidecar-validate.ts index bac7e6b0..f3b7567a 100644 --- a/packages/compiler/src/library/sidecar-validate.ts +++ b/packages/compiler/src/library/sidecar-validate.ts @@ -14,7 +14,7 @@ type Dict = Record; -const TOP_LEVEL_ORDER = [ +const TOP_LEVEL_ORDER: readonly string[] = [ "format", "wire_version", "abi_version", @@ -35,11 +35,11 @@ const TOP_LEVEL_ORDER = [ "integer_slots", "deterministic", "async_free", -] as const; +]; const HASH_RE = /^[0-9a-f]{16}$/; const TYPEREF_KINDS = new Set(["bool", "f64", "i64", "bytes", "void", "optional", "slice", "node", "value", "enum", "union"]); -const FUNCTION_CHANNELS = ["command_msg", "frame_msg", "key_msg", "pinch_msg"] as const; +const FUNCTION_CHANNELS = ["command_msg", "frame_msg", "key_msg", "pinch_msg"]; function isDict(v: unknown): v is Dict { return v !== null && typeof v === "object" && !Array.isArray(v); @@ -58,12 +58,12 @@ export function validateSidecar(doc: unknown): string[] { /* ── V1: required fields, format, top-level key order ─────────────── */ const keys = Object.keys(doc); for (const k of TOP_LEVEL_ORDER) { - if (!(k in doc)) bad("V1", `required field '${k}' is missing`); + if (!(Object.hasOwn(doc, k))) bad("V1", `required field '${k}' is missing`); } for (const k of keys) { if (!(TOP_LEVEL_ORDER as readonly string[]).includes(k)) bad("V1", `unknown top-level field '${k}' (emit only format-1 fields)`); } - const present = TOP_LEVEL_ORDER.filter((k) => k in doc); + const present = TOP_LEVEL_ORDER.filter((k) => Object.hasOwn(doc, k)); const actual = keys.filter((k) => (TOP_LEVEL_ORDER as readonly string[]).includes(k)); if (present.join(",") !== actual.join(",")) { bad("V1", "top-level keys are not emitted in the schema's §1 order"); @@ -348,7 +348,7 @@ export function validateSidecar(doc: unknown): string[] { return; } visiting.add(name); - for (const next of edges.get(name) ?? []) visit(next, [...path, name]); + for (const next of edges.get(name) ?? new Set()) visit(next, [...path, name]); visiting.delete(name); done.add(name); }; @@ -405,7 +405,7 @@ export function validateSidecar(doc: unknown): string[] { bad("V9", `channels.${ch} is ${v} but the suffix '${ch}' is ${v ? "absent from" : "present in"} abi.exports`); } } - for (const ch of ["appearance_msg", "chrome_msg"] as const) { + for (const ch of ["appearance_msg", "chrome_msg"]) { const v = channels[ch]; if (v === null) continue; if (typeof v !== "string") { @@ -450,7 +450,7 @@ export function validateSidecar(doc: unknown): string[] { while (grew) { grew = false; for (const name of [...reachable]) { - for (const next of edges.get(name) ?? []) { + for (const next of edges.get(name) ?? new Set()) { if (!reachable.has(next)) { reachable.add(next); grew = true; diff --git a/packages/compiler/src/library/sidecar.ts b/packages/compiler/src/library/sidecar.ts index 4859cedc..a69d7b0e 100644 --- a/packages/compiler/src/library/sidecar.ts +++ b/packages/compiler/src/library/sidecar.ts @@ -145,6 +145,10 @@ export interface SidecarDoc { /* ── identity hashing (schema §2 + the "module-graph" source contract) ─── */ let releaseVersion: string | null = null; +let installedReleaseVersion: string | null = null; + +/** Installed native clients read their release identity from the distribution. */ +export function setCompilerReleaseVersion(version: string): void { installedReleaseVersion = version; } /** The package version is stable within one compilation, but a long-lived * source/worktree process may observe a release stamp between compilations. */ @@ -157,6 +161,7 @@ export function clearSidecarCaches(): void { * module lives two levels below the package root in src/ and dist/ * alike). build_id input 1 and the sidecar's `compiler_version`. */ export function compilerReleaseVersion(): string { + if (installedReleaseVersion !== null) return installedReleaseVersion; if (releaseVersion === null) { const pkgPath = join(dirname(fileURLToPath(import.meta.url)), "..", "..", "package.json"); releaseVersion = (JSON.parse(readFileSync(pkgPath, "utf8")) as { version: string }).version; @@ -806,7 +811,7 @@ class Projector { if (shape.k === "object") return shape.fields; if (shape.k === "array") return this.inlineRecordFields(shape.elem); if (shape.k === "union") { - const present = shape.parts.filter((part) => part.k !== "absent"); + const present = shape.parts.filter((part): boolean => part.k !== "absent"); return present.length === 1 && present.length !== shape.parts.length ? this.inlineRecordFields(present[0]!) : null; @@ -1008,7 +1013,7 @@ class Projector { case "array": return { kind: "slice", elem: this.shapeRef(shape.elem, container, member, loc, synthesizedContext) }; case "union": { - const present = shape.parts.filter((p) => p.k !== "absent"); + const present = shape.parts.filter((p): boolean => p.k !== "absent"); const absents = shape.parts.length - present.length; if (absents > 0 && present.length === 1) { const inner = this.shapeRef(present[0]!, container, member, loc, synthesizedContext); @@ -1486,8 +1491,9 @@ export function buildSidecar(input: SidecarBuildInput): SidecarBuildResult { } const r = fn.returns; if (r !== null && r.k === "ref" && r.name === config.model) return false; - if (r !== null && r.k === "tuple" && r.elems.length === 2 && r.elems[0]!.k === "ref" && (r.elems[0] as { name: string }).name === config.model) { - return true; + if (r !== null && r.k === "tuple" && r.elems.length === 2) { + const first = r.elems[0]!; + if (first.k === "ref" && first.name === config.model) return true; } throw new SidecarError( `${which} export '${exportName}' must declare its return as '${config.model}' (bare state) or a two-element tuple '[${config.model}, ...]' (state plus an effect value)`, @@ -1523,7 +1529,7 @@ export function buildSidecar(input: SidecarBuildInput): SidecarBuildResult { const exports = abiExportSuffixes(profile); const exportSet = new Set(exports); const namedChannel = (constName: "appearanceMsg" | "chromeMsg"): string | null => { - const c = facts[constName]; + const c = constName === "appearanceMsg" ? facts.appearanceMsg : facts.chromeMsg; if (c === null) return null; const payload = armByName.get(c.value); if (payload === undefined) { diff --git a/packages/compiler/src/native/arguments.test.ts b/packages/compiler/src/native/arguments.test.ts deleted file mode 100644 index e9aa2846..00000000 --- a/packages/compiler/src/native/arguments.test.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { expect, test } from "vitest"; -import { parseNativeArguments } from "./arguments.js"; - -test("native build arguments preserve paths and select explicit output modes", () => { - expect(parseNativeArguments([ - "build", "source with spaces.ts", "-o", "result with spaces", "--backend=llvm", "--emit=obj", - "--toolchain", "/installed/compiler.json", "--dev", "--strip", "--keep-llvm", "--ffi=bindings.json", "--npm-static=one,@scope/two", - ], "default.json")).toEqual({ - help: false, toolchainPath: "/installed/compiler.json", - build: { - entryPath: "source with spaces.ts", outputPath: "result with spaces", backend: "llvm", outputKind: "obj", - optimization: "dev", strip: true, keepLlvm: true, ffiProfilePath: "bindings.json", npmStatic: ["one", "@scope/two"], - }, - }); - expect(parseNativeArguments(["--out=result", "--npm-static", "auto", "--", "-entry.ts"], "default.json")).toMatchObject({ - toolchainPath: "default.json", build: { entryPath: "-entry.ts", outputKind: "exe", backend: "llvm", npmStatic: "auto" }, - }); - expect(parseNativeArguments(["--help"], "default.json").help).toBe(true); -}); - -test("native build rejects unknown, missing and ambiguous arguments", () => { - for (const [args, message] of [ - [[], "entry source"], - [["main.ts"], "output path"], - [["main.ts", "-o"], "requires a value"], - [["main.ts", "-o", "--dev"], "requires a value"], - [["main.ts", "-o=x", "extra.ts"], "unexpected argument"], - [["main.ts", "-o=x", "--dynamic"], "unknown native compiler option"], - [["main.ts", "-o=x", "--backend=other"], "LLVM is the only backend"], - [["main.ts", "-o=x", "--backend=c"], "LLVM is the only backend"], - [["main.ts", "-o=x", "--emit=c"], "--emit must"], - [["main.ts", "-o=x", "--emit=other"], "--emit must"], - ] as const) expect(() => parseNativeArguments(args, "toolchain.json")).toThrow(message); -}); diff --git a/packages/compiler/src/native/arguments.ts b/packages/compiler/src/native/arguments.ts deleted file mode 100644 index d2455e6a..00000000 --- a/packages/compiler/src/native/arguments.ts +++ /dev/null @@ -1,79 +0,0 @@ -import type { NativeBuildOptions } from "./driver.js"; - -export interface NativeArguments { - help: boolean; - toolchainPath: string; - build: NativeBuildOptions; -} - -export function parseNativeArguments(args: readonly string[], defaultToolchain: string): NativeArguments { - let toolchainPath = defaultToolchain; - let entryPath = ""; - let outputPath = ""; - let backend = "llvm" as const; - let outputKind: "exe" | "obj" | "llvm" = "exe"; - let optimization: "release" | "dev" = "release"; - let strip = false; - let keepLlvm = false; - let help = false; - let ffiProfilePath: string | undefined; - let npmStatic: string[] | "auto" | undefined; - let positional = false; - for (let index = 0; index < args.length; index++) { - const arg = args[index]!; - if (!positional && (arg === "--help" || arg === "-h")) { help = true; continue; } - if (!positional && arg === "--") { positional = true; continue; } - if (!positional && arg === "--dev") { optimization = "dev"; continue; } - if (!positional && arg === "--strip") { strip = true; continue; } - if (!positional && arg === "--keep-llvm") { keepLlvm = true; continue; } - if (!positional && arg.startsWith("-")) { - const equals = arg.indexOf("="); - const name = equals < 0 ? arg : arg.slice(0, equals); - if (!["-o", "--out", "--toolchain", "--backend", "--emit", "--ffi", "--npm-static"].includes(name)) { - throw new Error(`unknown native compiler option: ${name}`); - } - const value = equals < 0 ? args[++index] : arg.slice(equals + 1); - if (value === undefined || value === "" || (equals < 0 && value.startsWith("--"))) throw new Error(`${name} requires a value`); - if (name === "-o" || name === "--out") outputPath = value; - else if (name === "--toolchain") toolchainPath = value; - else if (name === "--ffi") ffiProfilePath = value; - else if (name === "--npm-static") npmStatic = value === "auto" ? "auto" : value.split(","); - else if (name === "--backend") { - if (value !== "llvm") throw new Error("LLVM is the only backend"); - backend = value; - } else if (name === "--emit") { - if (value !== "exe" && value !== "obj" && value !== "llvm") throw new Error("--emit must be exe, obj, or llvm"); - outputKind = value; - } - continue; - } - if (!positional && index === 0 && arg === "build") continue; - if (entryPath !== "") throw new Error(`unexpected argument: ${arg}`); - entryPath = arg; - } - if (!help && entryPath === "") throw new Error("a TypeScript entry source is required"); - if (!help && outputPath === "") throw new Error("an output path is required (-o )"); - return { - help, toolchainPath, - build: { - entryPath, outputPath, backend, outputKind, optimization, strip, - ...(keepLlvm ? { keepLlvm: true } : {}), - ...(ffiProfilePath === undefined ? {} : { ffiProfilePath }), - ...(npmStatic === undefined ? {} : { npmStatic }), - }, - }; -} - -export const NATIVE_HELP = `Usage: scriptc-native build -o [options] - - --backend Code generation backend (default: llvm) - --emit Output artifact (default: exe) - --dev Disable optimization and include debug information - --strip Strip executable symbols - --keep-llvm Keep the generated LLVM at .ll - --ffi Native FFI bindings and link inputs - --npm-static Compile comma-separated packages, or auto - --toolchain Native toolchain manifest (default: beside executable) - --help Show this help - -The native compiler builds static programs; unsupported statements are errors.`; diff --git a/packages/compiler/src/native/cache.test.ts b/packages/compiler/src/native/cache.test.ts new file mode 100644 index 00000000..bfe9513a --- /dev/null +++ b/packages/compiler/src/native/cache.test.ts @@ -0,0 +1,69 @@ +import { mkdtempSync, readFileSync, rmSync, utimesSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, test, vi } from "vitest"; +import { contentDigest, NativeCache, openNativeCache } from "./cache.js"; + +const directories: string[] = []; +function directory(): string { + const path = mkdtempSync(join(tmpdir(), "scriptc-native-cache-test-")); + directories.push(path); + return path; +} +afterEach(() => { + vi.unstubAllEnvs(); + for (const path of directories.splice(0)) rmSync(path, { recursive: true, force: true }); +}); + +test("cache hits require an intact payload and digest", () => { + const cache = new NativeCache(directory()); + const key = contentDigest("program identity"); + const content = Buffer.from([0, 10, 128, 255]); + cache.write("object", key, content); + expect(cache.read("object", key)).toEqual(content); + const path = join(cache.root, "object", key); + writeFileSync(path, Buffer.from([0, 10, 128, 254])); + expect(cache.read("object", key)).toBeNull(); + cache.write("object", key, content); + rmSync(path + ".sha256"); + expect(cache.read("object", key)).toBeNull(); +}); + +test("a failed cache publication does not fail the build or replace unrelated files", () => { + const cache = new NativeCache(directory()); + const occupied = join(cache.root, "object"); + writeFileSync(occupied, "keep"); + expect(() => cache.write("object", contentDigest("key"), "payload")).not.toThrow(); + expect(readFileSync(occupied, "utf8")).toBe("keep"); +}); + +test("eviction removes oldest complete entries and leaves unrelated paths alone", () => { + const cache = new NativeCache(directory()); + const oldKey = contentDigest("old"); + const newKey = contentDigest("new"); + cache.write("frontend", oldKey, "a".repeat(700)); + cache.write("object", newKey, "b".repeat(700)); + utimesSync(join(cache.root, "frontend", oldKey), new Date(0), new Date(0)); + const unrelated = join(cache.root, "object", "unrelated"); + writeFileSync(unrelated, "keep"); + vi.stubEnv("SCRIPTC_CACHE_MAX_MB", String(1000 / (1024 * 1024))); + cache.prune(); + expect(cache.read("frontend", oldKey)).toBeNull(); + expect(cache.read("object", newKey)?.length).toBe(700); + expect(readFileSync(unrelated, "utf8")).toBe("keep"); +}); + +test("private-directory admission and cache disable options fail closed", () => { + vi.stubEnv("SCRIPTC_NO_CACHE", "0"); + vi.stubEnv("SCRIPTC_CACHE_DIR", directory()); + const inspect = vi.fn(() => false); + expect(openNativeCache(inspect)).toBeNull(); + expect(inspect).toHaveBeenCalledWith(process.env["SCRIPTC_CACHE_DIR"], false); + const rootOnly = vi.fn().mockReturnValueOnce(true).mockReturnValueOnce(false); + expect(openNativeCache(rootOnly)).toBeNull(); + expect(openNativeCache(() => true)).toBeInstanceOf(NativeCache); + vi.stubEnv("SCRIPTC_NO_CACHE", "1"); + const disabled = vi.fn(() => true); + expect(openNativeCache(disabled)).toBeNull(); + expect(disabled).not.toHaveBeenCalled(); +}); diff --git a/packages/compiler/src/native/cache.ts b/packages/compiler/src/native/cache.ts new file mode 100644 index 00000000..5452fd0d --- /dev/null +++ b/packages/compiler/src/native/cache.ts @@ -0,0 +1,92 @@ +import { createHash } from "node:crypto"; +import { mkdirSync, mkdtempSync, readFileSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { resolveBuildCacheRoot } from "../backend/cache-root.js"; + +export function contentDigest(value: string | Uint8Array): string { + const hash = createHash("sha256"); + if (typeof value === "string") hash.update(value); + else hash.update(value); + return hash.digest("hex"); +} + +/** A compiler cache contains private source and executable data. The host + * checks ownership and permissions before this class accepts a root. */ +export class NativeCache { + readonly root: string; + + constructor(root: string) { + this.root = join(root, "native-v1"); + mkdirSync(this.root, { recursive: true, mode: 0o700 }); + } + + read(family: string, key: string): Buffer | null { + try { + const path = join(this.root, family, key); + const bytes = readFileSync(path); + if (contentDigest(bytes) !== readFileSync(path + ".sha256", "utf8").trim()) return null; + return bytes; + } catch { return null; } + } + + write(family: string, key: string, bytes: string | Uint8Array): void { + let stage: string | null = null; + try { + const directory = join(this.root, family); + mkdirSync(directory, { recursive: true, mode: 0o700 }); + stage = mkdtempSync(join(directory, ".write-")); + writeFileSync(join(stage, "payload"), bytes); + writeFileSync(join(stage, "digest"), contentDigest(bytes) + "\n"); + const path = join(directory, key); + // Concurrent writers publish identical content under the same key; + // a reader between the two renames simply treats it as a miss. + renameSync(join(stage, "payload"), path); + renameSync(join(stage, "digest"), path + ".sha256"); + } catch { /* Cache failures never change a build's result. */ } + finally { + try { if (stage !== null) rmSync(stage, { recursive: true, force: true }); } catch { /* Cleanup is best effort too. */ } + } + } + + prune(): void { + try { + const configured = Number(process.env["SCRIPTC_CACHE_MAX_MB"] ?? "4096"); + const limit = (Number.isFinite(configured) && configured >= 0 ? configured : 4096) * 1024 * 1024; + const files: { path: string; size: number; time: number }[] = []; + let total = 0; + for (const family of ["frontend", "object", "sanitizer", "executable", "binary", "dsym"]) { + const directory = join(this.root, family); + let names: string[]; + try { names = readdirSync(directory); } catch { continue; } + for (const name of names) { + if (!/^[0-9a-f]{64}$/.test(name)) continue; + const path = join(directory, name); + const info = statSync(path); + if (!info.isFile()) continue; + files.push({ path, size: info.size, time: info.mtimeMs }); + total += info.size; + } + } + if (total <= limit) return; + files.sort((a, b) => a.time - b.time); + for (const file of files) { + if (total <= limit) break; + rmSync(file.path, { force: true }); + rmSync(file.path + ".sha256", { force: true }); + total -= file.size; + } + } catch { /* Eviction is best effort. */ } + } +} + +export function openNativeCache(isPrivate: (path: string, harden: boolean) => boolean): NativeCache | null { + const root = resolveBuildCacheRoot(); + if (root === null) return null; + try { + mkdirSync(root, { recursive: true, mode: 0o700 }); + if (!isPrivate(root, process.env["SCRIPTC_CACHE_DIR"] === undefined)) return null; + const cache = new NativeCache(root); + if (!isPrivate(cache.root, false)) return null; + return cache; + } catch { return null; } +} diff --git a/packages/compiler/src/native/cli.ts b/packages/compiler/src/native/cli.ts new file mode 100644 index 00000000..185a88a6 --- /dev/null +++ b/packages/compiler/src/native/cli.ts @@ -0,0 +1,100 @@ +import { spawnSync } from "node:child_process"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { runCli } from "../cli/command.js"; +import type { NativeCacheWarmProfile } from "../cli/host.js"; +import { selectNativeRuntimePack, stageNativeRuntimeSelection } from "../backend/runtime-pack-native.js"; +import { setDeclarationRoot } from "../frontend/dts-paths.js"; +import { FrontendServices } from "../frontend/services.js"; +import { createNativeTs7Api } from "../frontend/ts7/native-api.js"; +import { resolveProvenanceSourcesWithParser } from "../frontend/provenance-core.js"; +import { setCompilerReleaseVersion } from "../library/sidecar.js"; +import { NativeCompiler } from "./compiler.js"; +import { openNativeCache } from "./cache.js"; +import { loadNativeToolchain } from "./toolchain.js"; +import { buildSanitizedRuntime } from "./sanitizer.js"; +import { runCompilerTask } from "./task.js"; + +declare function compilerNativePrivateDirectory(path: string, harden: boolean): boolean; + +async function main(): Promise { + const manifestPath = process.env["SCRIPTC_TOOLCHAIN"] ?? process.execPath + ".json"; + let compiler: NativeCompiler | null = null; + let cache: ReturnType = null; + const getCompiler = (): NativeCompiler => { + if (compiler !== null) return compiler; + const toolchain = loadNativeToolchain(manifestPath, process.env); + if (toolchain.declarationsRoot !== undefined) setDeclarationRoot(toolchain.declarationsRoot); + setCompilerReleaseVersion(toolchain.compilerVersion); + cache = openNativeCache((path, harden) => compilerNativePrivateDirectory(path, harden)); + compiler = new NativeCompiler(toolchain, cache); + return compiler; + }; + return runCli(process.argv.slice(2), { + version: () => loadNativeToolchain(manifestPath).compilerVersion, + sourceTargetPlatform: () => getCompiler().toolchain.target.platform, + analyze: (entry, options) => runCompilerTask(() => getCompiler().analyze(entry, options)), + compile: (entry, options) => runCompilerTask(() => getCompiler().compile(entry, options)), + compileLibrary: (options) => runCompilerTask(() => getCompiler().compileLibrary(options)), + resolveProvenanceSources: async (entry) => { + const toolchain = getCompiler().toolchain; + const services = new FrontendServices((options) => createNativeTs7Api({ ...options, executable: toolchain.ts7Executable })); + try { return await resolveProvenanceSourcesWithParser(entry, (path, source, kind) => services.parse(path, source, kind)); } + finally { services.close(); } + }, + warmNativeCaches: async (options) => { + const toolchain = getCompiler().toolchain; + if (cache === null) throw new Error("the native build cache is disabled or unavailable"); + if (!toolchain.target.supports.exe) throw new Error(`native cache warming requires an executable target; ${toolchain.target.name} supports library archives`); + const profiles: { profile: NativeCacheWarmProfile; elapsedMs: number }[] = []; + for (const profile of [...new Set(options.profiles ?? ["runtime", "tls", "dynamic"] as NativeCacheWarmProfile[])]) { + const start = performance.now(); + const stage = mkdtempSync(join(tmpdir(), "scriptc-warm-")); + try { + const pack = selectNativeRuntimePack(toolchain.runtimePackRoot, toolchain.target, toolchain.compilerVersion, { + dynamic: profile === "dynamic", fetch: profile === "tls", regex: false, + copying: false, textDecoderLegacy: false, fileHandle: false, netIsland: false, + zlib: false, assert: false, inspect: false, dynInvoke: false, dc: false, + dynAsync: false, events: false, emitter: false, symbol: false, bigint: false, + searchParams: false, qs: false, parseArgs: false, stream: false, net: false, + http: false, http2: false, dgram: false, watch: false, foreignFfi: false, + nodeTest: false, tls: false, tlsCa: false, + }, options.optimization ?? "release"); + if (options.sanitize) buildSanitizedRuntime(toolchain, pack, stage, "executable", cache); + else stageNativeRuntimeSelection(pack, stage); + } finally { rmSync(stage, { recursive: true, force: true }); } + profiles.push({ profile, elapsedMs: performance.now() - start }); + } + return { cacheRoot: cache.root, profiles }; + }, + run: async (binary) => { + const toolchain = getCompiler().toolchain; + let executable = binary; + const args: string[] = []; + if (toolchain.target.platform === "wasi") { + if (toolchain.wasiNodeRunner === undefined) throw new Error("the WASI runner is missing from this installation"); + executable = "node"; + args.push("--no-warnings", toolchain.wasiNodeRunner, binary); + } + const result = spawnSync(executable, args, { stdio: "inherit" }); + if (result.error) { + if (toolchain.target.platform === "wasi") { + throw new Error(`could not start the Node.js WASI host: ${result.error.message}; running WASI modules requires Node.js 24 or newer on PATH; use scriptc build to compile without Node`); + } + throw new Error(result.error.message); + } + if (result.signal) { + process.stderr.write(`scriptc: program killed by ${result.signal}\n`); + return 1; + } + return result.status ?? 0; + }, + }); +} + +try { process.exitCode = await main(); } +catch (error) { + process.stderr.write(`scriptc: ${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 1; +} diff --git a/packages/compiler/src/native/driver.test.ts b/packages/compiler/src/native/compiler.test.ts similarity index 83% rename from packages/compiler/src/native/driver.test.ts rename to packages/compiler/src/native/compiler.test.ts index b2cf39b9..4efd39cb 100644 --- a/packages/compiler/src/native/driver.test.ts +++ b/packages/compiler/src/native/compiler.test.ts @@ -2,7 +2,8 @@ import { mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "n import { join } from "node:path"; import { tmpdir } from "node:os"; import { afterEach, expect, test, vi } from "vitest"; -import { buildNative, type NativeToolchain } from "./driver.js"; +import { NativeCompiler } from "./compiler.js"; +import type { NativeToolchain } from "./toolchain.js"; import { runNativeFrontend } from "../frontend/pipeline-native.js"; import { emitNativeObject } from "../backend/native-tools.js"; import { MACOS_ARM64_TARGET } from "../backend/targets.js"; @@ -54,18 +55,19 @@ test.each([false, true])("object generation releases frontend resources and pres helperExecutable: "unused", helperPackageRoot: "unused", runtimePackRoot: "unused", linker: "unused", linkerArgs: [], dsymutil: "unused", }; - const build = () => buildNative({ - entryPath: entry, outputPath: output, backend: "llvm", outputKind: "obj", - optimization: "release", strip: true, keepLlvm: true, - }, toolchain); + const build = () => new NativeCompiler(toolchain).compile(entry, { + outDir: directory, outPath: output, backend: "llvm", outputKind: "obj", + optimization: "release", strip: true, + }); if (fail) { - expect(build).toThrow("codegen failed"); + expect(build()).toMatchObject({ ok: false, diagnostics: [expect.objectContaining({ message: "codegen failed" })] }); expect(readFileSync(output, "utf8")).toBe("previous object"); expect(readFileSync(llvmPath, "utf8")).toBe("previous LLVM"); } else { - expect(build()).toMatchObject({ outputPath: output, llvmPath }); + expect(build()).toMatchObject({ ok: true, artifact: { kind: "obj", path: output } }); expect(readFileSync(output, "utf8")).toBe("new object"); - expect(readFileSync(llvmPath, "utf8")).toBe(emitted); + expect(emitted).toContain("define i32 @main"); + expect(readFileSync(llvmPath, "utf8")).toBe("previous LLVM"); } expect(dispose).toHaveBeenCalledTimes(1); expect(readdirSync(directory).sort()).toEqual(["output.o", "output.o.ll"]); diff --git a/packages/compiler/src/native/compiler.ts b/packages/compiler/src/native/compiler.ts new file mode 100644 index 00000000..44154f9a --- /dev/null +++ b/packages/compiler/src/native/compiler.ts @@ -0,0 +1,286 @@ +import { mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs"; +import { basename, dirname, join, resolve } from "node:path"; +import type { AnalyzeOptions, AnalyzeResult, CompileFailure, CompileLibraryOptions, CompileLibraryResult, CompileRequestOptions, CompileRequestResult } from "../compile-types.js"; +import { LlvmUnsupportedError } from "../backend/llvm/emitter.js"; +import { executableLinkInputs } from "../backend/link-plan-core.js"; +import { formatNativeLinkInfo } from "../backend/native-link-info-core.js"; +import { emitNativeObject, requireNativeArtifact, runNativeTool, verifyNativeHelper } from "../backend/native-tools.js"; +import { selectNativeRuntimePack, stageNativeRuntimeSelection } from "../backend/runtime-pack-native.js"; +import { RuntimePackError } from "../backend/runtime-pack-core.js"; +import { NativeCodegenError } from "../backend/native-codegen-core.js"; +import { llvmRefusalDiag } from "../backend/target-diagnostics.js"; +import { nativeCodegenDiag } from "../diagnostics/diagnostic.js"; +import { loadFfiProfile, type FfiProfile } from "../ffi/ffi-manifest.js"; +import { analyzeWithFrontend } from "../frontend/analysis.js"; +import type { FrontendFactory } from "../frontend/pipeline.js"; +import { runNativeFrontend } from "../frontend/pipeline-native.js"; +import { loadLibraryProfile } from "../library/library-profile.js"; +import { libraryLocalizeSymbols, libraryWasmExports, libraryWasmRefusal } from "../library/prepare.js"; +import { clearFenceEvalCaches } from "../library/fence-eval.js"; +import { archiveNativeLibrary, localizeNativeLibrary, linkNativeWasmLibrary } from "./library.js"; +import type { NativeToolchain } from "./toolchain.js"; +import { evaluateNativeComptime } from "./comptime.js"; +import { contentDigest, type NativeCache } from "./cache.js"; +import { splitLlvmProgram, splitLlvmLibraryProgram } from "../backend/llvm/split.js"; +import { prepareNativeExecutable, prepareNativeLibrary } from "./prepare.js"; +import { buildSanitizedRuntime, sanitizerDriver, sanitizerFlags } from "./sanitizer.js"; +import { openNativeExecutableCache } from "./executable-cache.js"; +import { nativeFileIdentity } from "./file-identity.js"; + +function failure(error: unknown, entry: string, sources: Map): CompileFailure { + return { ok: false, diagnostics: [nativeCodegenDiag("SC3004", error instanceof Error ? error.message : String(error), entry)], sourceTexts: sources }; +} + +/** Shared frontend semantics with native processes for code generation and linking. */ +export class NativeCompiler { + private readonly frontend: FrontendFactory; + + constructor(readonly toolchain: NativeToolchain, private readonly cache: NativeCache | null = null) { + const evaluator = toolchain.comptimeExecutable; + this.frontend = (entry, npmStatic, externalTypes, libraryNpmStatic) => runNativeFrontend(entry, toolchain.ts7Executable, npmStatic, externalTypes, + evaluator === undefined ? undefined : (source, timeout) => evaluateNativeComptime(source, timeout, toolchain.ts7Executable, evaluator), libraryNpmStatic); + } + + analyze(entry: string, options: AnalyzeOptions): AnalyzeResult { + return analyzeWithFrontend(resolve(entry), options, this.toolchain.target.platform, this.frontend); + } + + private emitObject(input: string, output: string, source: string, optimization: "release" | "dev", outputKind: "obj" | "asm"): void { + const toolchain = this.toolchain; + const helperOptions = { + executable: toolchain.helperExecutable, packageRoot: toolchain.helperPackageRoot, + compilerVersion: toolchain.compilerVersion, target: toolchain.target, helper: toolchain.helper, + }; + let key: string | null = null; + let identity: string | null = null; + if (this.cache !== null && process.env["SCRIPTC_LLVM_HELPER"] === undefined) { + try { + identity = nativeFileIdentity(toolchain.helperExecutable); + key = contentDigest(JSON.stringify({ schema: 1, llvm: contentDigest(readFileSync(input)), outputKind, optimization, source, + target: toolchain.target, helper: identity, + identity: readFileSync(join(toolchain.helperPackageRoot, "package.json"), "utf8"), version: toolchain.compilerVersion })); + const bytes = this.cache.read("object", key); + if (bytes !== null) { + verifyNativeHelper(helperOptions); + if (nativeFileIdentity(toolchain.helperExecutable) === identity) { + writeFileSync(output, bytes); + return; + } + key = null; + } + } catch { key = null; } + } + emitNativeObject({ + executable: toolchain.helperExecutable, packageRoot: toolchain.helperPackageRoot, + compilerVersion: toolchain.compilerVersion, target: toolchain.target, helper: toolchain.helper, + inputPath: input, outputPath: output, sourcePath: source, optimization, outputKind, + }); + if (this.cache !== null && key !== null) { + try { + if (nativeFileIdentity(toolchain.helperExecutable) === identity) this.cache.write("object", key, readFileSync(output)); + } catch { /* An updated helper or unavailable cache only prevents reuse. */ } + } + } + + private emitProgramObject(input: string, output: string, source: string, optimization: "release" | "dev", stage: string, library: boolean): void { + const llvm = optimization === "dev" && this.toolchain.target.platform !== "wasi" ? readFileSync(input, "utf8") : null; + const split = llvm === null ? null : library ? splitLlvmLibraryProgram(llvm) : splitLlvmProgram(llvm); + if (split === null) { this.emitObject(input, output, source, optimization, "obj"); return; } + const objects: string[] = []; + for (const [index, shard] of split.shards.entries()) { + const shardInput = join(stage, `shard-${index}.ll`); + const shardOutput = join(stage, `shard-${index}` + this.toolchain.target.outputSuffixes.obj); + writeFileSync(shardInput, shard.source); + this.emitObject(shardInput, shardOutput, source + "." + shard.name, optimization, "obj"); + objects.push(shardOutput); + } + const merged = localizeNativeLibrary(this.toolchain, stage, objects, [], split.publicSymbols); + renameSync(merged, output); + } + + compile(entry: string, options: CompileRequestOptions): CompileRequestResult { + const toolchain = this.toolchain; + const target = toolchain.target; + const output = resolve(options.outPath); + const outputKind = options.outputKind ?? "exe"; + const optimization = options.optimization ?? "release"; + entry = resolve(entry); + let sourceTexts = new Map(); + let stage: string | null = null; + try { + if (entry === output) throw new Error("output path must differ from the entry source"); + if (outputKind === "exe" && !target.supports.exe) throw new NativeCodegenError("SC3002", `${target.name} supports library archives, not executables`); + if (options.sanitize && (outputKind === "asm" || outputKind === "obj")) { + throw new NativeCodegenError("SC3002", `--sanitize is not supported with --emit=${outputKind}; AddressSanitizer instrumentation parity is not available in the LLVM native helper yet`); + } + if (options.sanitize) sanitizerDriver(toolchain); + let ffi: FfiProfile | null = null; + if (options.ffiProfilePath !== undefined) { + const loaded = loadFfiProfile(resolve(options.ffiProfilePath)); + if (!loaded.ok) return { ok: false, diagnostics: loaded.diagnostics, sourceTexts }; + ffi = loaded.profile; + } + const prepared = prepareNativeExecutable(entry, options, ffi, toolchain, this.frontend, this.cache); + if (!prepared.ok) return prepared; + sourceTexts = new Map(prepared.sources); + mkdirSync(dirname(output), { recursive: true }); + stage = mkdtempSync(join(dirname(output), ".scriptc-native-")); + const stagedOutput = join(stage, basename(output)); + const stem = basename(entry).replace(/\.(ts|mts|cts|js|mjs|cjs)$/, ""); + const llvmPath = join(options.outDir, `${stem}.ll`); + if (outputKind === "ir") { + writeFileSync(stagedOutput, prepared.ir!); + renameSync(stagedOutput, output); + return { ok: true, artifact: { kind: "ir", path: output } }; + } + const { llvm, features } = prepared; + const inputDirectory = join(stage, "input"); + mkdirSync(inputDirectory); + const llvmInput = outputKind === "llvm" ? stagedOutput : join(inputDirectory, "program.ll"); + writeFileSync(llvmInput, llvm); + if (outputKind === "llvm") { + renameSync(stagedOutput, output); + return { ok: true, artifact: { kind: "llvm", path: output } }; + } + const object = outputKind === "exe" ? join(inputDirectory, "program" + target.outputSuffixes.obj) : stagedOutput; + const executablePack = outputKind === "exe" + ? selectNativeRuntimePack(toolchain.runtimePackRoot, target, toolchain.compilerVersion, features, optimization) : null; + const executableCache = executablePack === null ? null + : openNativeExecutableCache(this.cache, toolchain, llvm, options, ffi, executablePack); + const restored = executableCache?.restore(stagedOutput) ?? false; + if (!restored) { + if (options.sanitize) { + runNativeTool(sanitizerDriver(toolchain), [...sanitizerFlags(toolchain, optimization), "-c", llvmInput, "-o", object]); + requireNativeArtifact(object); + } else if (outputKind === "exe") this.emitProgramObject(llvmInput, object, entry, optimization, stage, false); + else this.emitObject(llvmInput, object, entry, optimization, outputKind === "asm" ? "asm" : "obj"); + if (outputKind === "asm" || outputKind === "obj") { + const pack = outputKind === "obj" && options.nativeLinkInfo + ? selectNativeRuntimePack(toolchain.runtimePackRoot, target, toolchain.compilerVersion, features, optimization) : null; + if (pack !== null) stageNativeRuntimeSelection(pack, join(stage, "runtime")); + const info = pack === null ? undefined : formatNativeLinkInfo({ programObject: output, target, ffi }, toolchain.compilerVersion, pack); + renameSync(stagedOutput, output); + return { ok: true, artifact: { kind: outputKind, path: output, ...(info === undefined ? {} : { nativeLinkInfo: info }) } }; + } + const pack = executablePack!; + const runtime = options.sanitize + ? buildSanitizedRuntime(toolchain, pack, join(stage, "runtime"), "executable", this.cache) + : stageNativeRuntimeSelection(pack, join(stage, "runtime")); + const plan = executableLinkInputs({ + target, programObject: object, ffiLibraries: ffi?.libraries ?? [], ffiSystemLibraries: ffi?.systemLibraries ?? [], + ffiFrameworks: ffi?.frameworks ?? [], runtimeObjects: runtime.runtimeObjects, runtimeArchives: runtime.archives, + runtimeSystemLibraries: runtime.systemLibraries, optimization, strip: options.strip ?? false, + ...(options.windowsSubsystem === undefined ? {} : { windowsSubsystem: options.windowsSubsystem }), + }); + const linkArgs = [...(options.sanitize ? ["-fsanitize=address"] : toolchain.linkerArgs), ...plan.driverFlags, ...plan.inputs, + ...plan.systemLibraries.map((name) => `-l${name}`), "-o", stagedOutput]; + const cacheable = executableCache?.trace(linkArgs, stage) ?? false; + runNativeTool(options.sanitize ? sanitizerDriver(toolchain) : toolchain.linker, linkArgs); + requireNativeArtifact(stagedOutput); + if (target.platform === "darwin" && optimization === "dev" && !options.strip) { + runNativeTool(toolchain.dsymutil, [stagedOutput, "-o", stagedOutput + ".dSYM"]); + } + if (cacheable) executableCache?.publish(stagedOutput); + } + if (target.platform === "darwin" && optimization === "dev" && !options.strip) { + rmSync(output + ".dSYM", { recursive: true, force: true }); + renameSync(stagedOutput + ".dSYM", output + ".dSYM"); + } + mkdirSync(options.outDir, { recursive: true }); + writeFileSync(llvmPath, llvm); + let irPath: string | undefined; + if (options.emitIr) { + irPath = join(options.outDir, `${stem}.ir.json`); + writeFileSync(irPath, prepared.ir!); + } + renameSync(stagedOutput, output); + if (target.platform === "darwin" && (optimization !== "dev" || options.strip)) rmSync(output + ".dSYM", { recursive: true, force: true }); + return { ok: true, artifact: { kind: "exe", path: output, translationUnitPath: llvmPath, backend: "llvm" }, + binaryPath: output, llvmPath, backend: "llvm", ...(irPath === undefined ? {} : { irPath }) }; + } catch (error) { + // Keep subclass reads at the catch boundary, where the thrown object + // retains its class identity and source location. + if (error instanceof LlvmUnsupportedError) return { ok: false, diagnostics: [llvmRefusalDiag(error, entry)], sourceTexts }; + if (error instanceof NativeCodegenError) return { ok: false, diagnostics: [nativeCodegenDiag(error.diagnosticCode, error.message, entry)], sourceTexts }; + if (error instanceof RuntimePackError) return { ok: false, diagnostics: [nativeCodegenDiag(error.code === "unsupported" ? "SC3002" : "SC3003", error.message, entry)], sourceTexts }; + return failure(error, entry, sourceTexts); + } + finally { + if (stage !== null) rmSync(stage, { recursive: true, force: true }); + this.cache?.prune(); + } + } + + compileLibrary(options: CompileLibraryOptions): CompileLibraryResult { + clearFenceEvalCaches(); + const toolchain = this.toolchain; + let sourceTexts = new Map(); + let stage: string | null = null; + try { + const loaded = loadLibraryProfile(resolve(options.profilePath)); + if (!loaded.ok) return { ok: false, diagnostics: loaded.diagnostics, sourceTexts }; + const profile = loaded.profile; + const wasm = toolchain.target.platform === "wasi"; + if (wasm) { + const refusal = libraryWasmRefusal(profile, options.sanitize ?? false); + if (refusal !== null) return { ok: false, diagnostics: [refusal], sourceTexts }; + } + if (options.sanitize) sanitizerDriver(toolchain); + const stem = basename(profile.entry).replace(/\.(ts|mts|cts|js|mjs|cjs)$/, ""); + const archivePath = resolve(options.outPath ?? join(options.outDir, `${stem}${wasm ? ".wasm" : ".lib.a"}`)); + const llvmPath = join(options.outDir, `${stem}.lib.ll`); + const prepared = prepareNativeLibrary(profile, options, archivePath, toolchain, this.frontend, this.cache); + if (!prepared.ok) return prepared; + sourceTexts = new Map(prepared.sources); + if (archivePath === profile.entry) throw new Error("output path must differ from the entry source"); + mkdirSync(dirname(archivePath), { recursive: true }); + mkdirSync(options.outDir, { recursive: true }); + stage = mkdtempSync(join(dirname(archivePath), ".scriptc-library-")); + const llvm = prepared.llvm; + const llvmInput = join(stage, "program.ll"); + writeFileSync(llvmInput, llvm); + const features = prepared.features; + const pack = selectNativeRuntimePack(toolchain.runtimePackRoot, toolchain.target, toolchain.compilerVersion, + features, profile.optimization, profile.instancePerThread ? "library-thread" : "library"); + const runtime = options.sanitize + ? buildSanitizedRuntime(toolchain, pack, join(stage, "runtime"), profile.instancePerThread ? "library-thread" : "library", this.cache) + : stageNativeRuntimeSelection(pack, join(stage, "runtime")); + const stagedArchive = join(stage, wasm ? "output.wasm" : "output.a"); + const localizeSymbols = libraryLocalizeSymbols(profile); + const program = join(stage, "program" + toolchain.target.outputSuffixes.obj); + if (options.sanitize) { + runNativeTool(sanitizerDriver(toolchain), [...sanitizerFlags(toolchain, profile.optimization), "-c", llvmInput, "-o", program]); + requireNativeArtifact(program); + } else this.emitProgramObject(llvmInput, program, profile.entry, profile.optimization, stage, true); + if (wasm) { + linkNativeWasmLibrary({ toolchain, programObject: program, outputPath: stagedArchive, + runtime, optimization: profile.optimization, exports: libraryWasmExports(profile) }); + } else { + archiveNativeLibrary({ toolchain, programObject: program, outputPath: stagedArchive, + stage, runtime, ...(localizeSymbols === undefined ? {} : { localizeSymbols }) }); + } + writeFileSync(llvmPath, llvm); + let irPath: string | undefined; + if (options.emitIr) { + irPath = join(options.outDir, `${stem}.lib.ir.json`); + writeFileSync(irPath, prepared.ir!); + } + let sidecarPath: string | undefined; + if (prepared.sidecarJson !== null) { + sidecarPath = profile.sidecar!.path === null ? `${archivePath}.contract.json` : resolve(dirname(archivePath), profile.sidecar!.path); + writeFileSync(sidecarPath, prepared.sidecarJson); + } + renameSync(stagedArchive, archivePath); + return { ok: true, archivePath, llvmPath, backend: "llvm", + ...(irPath === undefined ? {} : { irPath }), ...(sidecarPath === undefined ? {} : { sidecarPath }) }; + } catch (error) { + // Keep subclass reads at the catch boundary, where the thrown object + // retains its class identity and source location. + if (error instanceof LlvmUnsupportedError) return { ok: false, diagnostics: [llvmRefusalDiag(error, options.profilePath)], sourceTexts }; + if (error instanceof NativeCodegenError) return { ok: false, diagnostics: [nativeCodegenDiag(error.diagnosticCode, error.message, options.profilePath)], sourceTexts }; + if (error instanceof RuntimePackError) return { ok: false, diagnostics: [nativeCodegenDiag(error.code === "unsupported" ? "SC3002" : "SC3003", error.message, options.profilePath)], sourceTexts }; + return failure(error, options.profilePath, sourceTexts); + } + finally { if (stage !== null) rmSync(stage, { recursive: true, force: true }); } + } +} diff --git a/packages/compiler/src/native/comptime.test.ts b/packages/compiler/src/native/comptime.test.ts new file mode 100644 index 00000000..f3860b09 --- /dev/null +++ b/packages/compiler/src/native/comptime.test.ts @@ -0,0 +1,68 @@ +import { execFileSync } from "node:child_process"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { afterAll, beforeAll, expect, test } from "vitest"; +import { nativeCodegenTarget } from "../backend/targets.js"; +import { ts7Executable } from "../frontend/ts7/rpc-api.js"; +import { evaluateNativeComptime } from "./comptime.js"; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), "../../.."); +const target = nativeCodegenTarget({}); +const stage = mkdtempSync(join(process.platform === "win32" ? tmpdir() : "/tmp", "scriptc-comptime-test-")); +const evaluator = join(stage, "comptime" + (process.platform === "win32" ? ".exe" : "")); + +beforeAll(() => { + if (target === null) throw new Error("a supported native host is required"); + const runtime = join(root, target.runtimePackPackage.replace("@scriptc/", "")); + execFileSync(target.defaultLinker, [...target.defaultLinkerArgs, "-std=c11", "-O2", "-Wall", "-Wextra", "-Werror", + "-I", join(root, "runtime/vendor/quickjs-ng"), join(root, "compiler/native/comptime.c"), + join(runtime, "artifacts/vendor/quickjs/libscriptc-quickjs.a"), "-lm", "-lpthread", "-o", evaluator]); +}); +afterAll(() => { rmSync(stage, { recursive: true, force: true }); }); + +function evaluate(source: string, timeout = 2000): unknown { + return evaluateNativeComptime(source, timeout, ts7Executable(), evaluator); +} + +test("native comptime erases TypeScript and returns the actual structured value", () => { + expect(evaluate(`() => { + interface Pair { value: number; text: string } + const make = (value: T): T => value; + const items: Pair[] = []; + for (let i = 0; i < 4; i++) items.push(make({ value: i * i, text: String(i) })); + return { items, zero: -0, text: "hello \\ud83c\\udf0d", lone: "\\ud800", empty: undefined }; + }`)).toEqual({ items: [0, 1, 4, 9].map((value, index) => ({ value, text: String(index) })), zero: -0, + text: "hello 🌍", lone: "\ud800", empty: undefined }); +}); + +test("native comptime retains non-finite values for shared result validation", () => { + const values = evaluate("() => [NaN, Infinity, -Infinity, -0, undefined, null, 123n]") as unknown[]; + expect(values).toEqual([NaN, Infinity, -Infinity, -0, undefined, null, 123n]); +}); + +test("callback changes to globals and serialization hooks cannot replace its result", () => { + expect(evaluate(`() => { + JSON.stringify = () => '"spoofed"'; + Array.prototype.toJSON = () => "spoofed"; + Object.prototype.toJSON = () => "spoofed"; + return { values: [1, 2], nested: { ok: true } }; + }`)).toEqual({ values: [1, 2], nested: { ok: true } }); +}); + +test("native comptime isolates callbacks and exposes no host bindings", () => { + expect(evaluate("() => { globalThis.saved = 1; return 2; }")).toBe(2); + expect(evaluate("() => [typeof saved, typeof process, typeof require, typeof console, typeof fetch]")).toEqual( + ["undefined", "undefined", "undefined", "undefined", "undefined"], + ); +}); + +test("throws, cycles, and runaway callbacks report bounded failures", () => { + expect(() => evaluate('() => { throw new Error("user failure") }')).toThrow("user failure"); + expect(() => evaluate("() => { const result = {}; result.self = result; return result; }")).toThrow("cyclic compile-time result"); + let failure: unknown; + try { evaluate("() => { while (true) {} }", 20); } catch (error) { failure = error; } + expect(failure).toMatchObject({ code: "ERR_SCRIPT_EXECUTION_TIMEOUT" }); + expect(evaluate("() => 42")).toBe(42); +}); diff --git a/packages/compiler/src/native/comptime.ts b/packages/compiler/src/native/comptime.ts new file mode 100644 index 00000000..10938904 --- /dev/null +++ b/packages/compiler/src/native/comptime.ts @@ -0,0 +1,64 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { runNativeTool } from "../backend/native-tools.js"; + +/** Tagged values preserve -0, non-finite numbers, and missing properties so + * shared result validation sees the value the callback actually returned. */ +export function decodeComptimeValue(wire: unknown): unknown { + if (!Array.isArray(wire) || typeof wire[0] !== "string") throw new Error("invalid compile-time result"); + const value = wire as unknown[]; + switch (value[0]) { + case "error": { + if (value[1] === "ERR_SCRIPT_EXECUTION_TIMEOUT") throw { code: "ERR_SCRIPT_EXECUTION_TIMEOUT", message: String(value[2]) }; + throw new Error(String(value[2])); + } + case "undefined": return undefined; + case "null": return null; + case "boolean": + if (typeof value[1] === "boolean") return value[1]; + break; + case "string": + if (typeof value[1] === "string") return value[1]; + break; + case "number": + if (typeof value[1] === "string") return value[1] === "-0" ? -0 : Number(value[1]); + break; + case "bigint": return BigInt(String(value[1])); + case "function": return () => {}; + case "symbol": throw new Error("compile-time result contains a symbol"); + case "array": + if (Array.isArray(value[1])) return value[1].map((item) => decodeComptimeValue(item)); + break; + case "object": { + if (!Array.isArray(value[1])) break; + const result: Record = Object.create(null) as Record; + for (const item of value[1]) { + if (!Array.isArray(item) || typeof item[0] !== "string") throw new Error("invalid compile-time property"); + result[item[0]] = decodeComptimeValue(item[1]); + } + return result; + } + } + throw new Error("invalid compile-time result"); +} + +export function comptimeScript(source: string): string { + return `(${source})();\n`; +} + +export function evaluateNativeComptime(source: string, timeoutMs: number, ts7: string, evaluator: string): unknown { + const stage = mkdtempSync(join(tmpdir(), "scriptc-comptime-")); + try { + const sourcePath = join(stage, "eval.ts"); + const output = join(stage, "output"); + mkdirSync(output); + writeFileSync(sourcePath, comptimeScript(source)); + // Type erasure remains TypeScript's job, including nested annotations, + // type assertions, generic functions, and enums in closed callbacks. + runNativeTool(ts7, ["--ignoreConfig", "--noCheck", "--target", "esnext", "--module", "esnext", + "--moduleDetection", "legacy", "--outDir", output, sourcePath]); + const result = runNativeTool(evaluator, [join(output, "eval.js"), String(timeoutMs)]); + return decodeComptimeValue(JSON.parse(result)); + } finally { rmSync(stage, { recursive: true, force: true }); } +} diff --git a/packages/compiler/src/native/driver.ts b/packages/compiler/src/native/driver.ts deleted file mode 100644 index 31094f23..00000000 --- a/packages/compiler/src/native/driver.ts +++ /dev/null @@ -1,142 +0,0 @@ -/** Static executable compiler. The TS7 parser/checker and native toolchain - * are external native processes; lowering, validation and LLVM emission run - * inside this binary. Installed paths are supplied by the distribution. */ -import { mkdirSync, mkdtempSync, renameSync, rmSync, writeFileSync } from "node:fs"; -import { basename, dirname, join, resolve } from "node:path"; -import { emitLlvmModule } from "../backend/llvm/emitter.js"; -import { executableLinkFeatures } from "../backend/executable-features.js"; -import { executableLinkInputs } from "../backend/link-plan-core.js"; -import type { NativeLinkFeatures } from "../backend/native-link-info.js"; -import { stageNativeRuntimePack } from "../backend/runtime-pack-native.js"; -import { emitNativeObject, requireNativeArtifact, runNativeTool } from "../backend/native-tools.js"; -import type { NativeHelperSpec, NativeTargetSpec } from "../backend/targets.js"; -import { loadFfiProfile, type FfiProfile } from "../ffi/ffi-manifest.js"; -import { runNativeFrontend } from "../frontend/pipeline-native.js"; -import type { LowerStats } from "../frontend/lowering/lowerer.js"; -import { validateModule } from "../ir/validate.js"; - -export interface NativeToolchain { - compilerVersion: string; - ts7Executable: string; - target: NativeTargetSpec; - helper: NativeHelperSpec; - helperExecutable: string; - helperPackageRoot: string; - runtimePackRoot: string; - linker: string; - linkerArgs: string[]; - dsymutil: string; -} - -export interface NativeBuildOptions { - entryPath: string; - outputPath: string; - backend: "llvm"; - outputKind: "exe" | "obj" | "llvm"; - optimization: "release" | "dev"; - strip: boolean; - keepLlvm?: boolean; - ffiProfilePath?: string; - npmStatic?: readonly string[] | "auto"; -} - -export interface NativeBuildResult { - outputPath: string; - llvmPath?: string; - stats: LowerStats; -} - -/** Keep the program graph and its TS7 session inside the frontend phase. - * Native object generation can need several GB of its own working memory. */ -function emitNativeInput( - options: NativeBuildOptions, toolchain: NativeToolchain, - entry: string, ffi: FfiProfile | null, path: string, -): { features: NativeLinkFeatures; stats: LowerStats } { - const frontend = runNativeFrontend(entry, toolchain.ts7Executable, options.npmStatic); - try { - if (frontend.preflight.length !== 0) throw new Error(JSON.stringify(frontend.preflight)); - const lowered = frontend.lower({ dynamic: false, targetPlatform: toolchain.target.platform, ffiImports: ffi?.functions ?? [] }); - if (lowered.module === null || lowered.stats.statementsFailed !== 0 || lowered.stats.statementsIsland !== 0 || lowered.stats.functionsSkipped !== 0) { - throw new Error(JSON.stringify(lowered.diagnostics)); - } - const module = lowered.module; - const errors = validateModule(module); - if (errors.length !== 0) throw new Error(JSON.stringify(errors)); - const features = executableLinkFeatures(module, false); - const debug = options.optimization === "dev" && !options.strip; - const debugSources = debug ? frontend.sourceTexts() : new Map(); - const llvm = emitLlvmModule(module, { - targetTriple: toolchain.target.llvmTriple, - pointerBits: toolchain.target.pointerBits, - wasi: toolchain.target.platform === "wasi", - runtimeAbiMarker: true, - ...(debug ? { debugSources } : {}), - }); - writeFileSync(path, llvm); - return { features, stats: lowered.stats }; - } finally { frontend.dispose(); } -} - -export function buildNative(options: NativeBuildOptions, toolchain: NativeToolchain): NativeBuildResult { - const entry = resolve(options.entryPath); - const output = resolve(options.outputPath); - if (entry === output) throw new Error("output path must differ from the entry source"); - let ffi: FfiProfile | null = null; - if (options.ffiProfilePath !== undefined) { - const loaded = loadFfiProfile(options.ffiProfilePath); - if (!loaded.ok) throw new Error(JSON.stringify(loaded.diagnostics)); - ffi = loaded.profile; - } - mkdirSync(dirname(output), { recursive: true }); - // A sibling temporary directory keeps installation on the same volume - // and preserves the basename used by Mach-O's ad-hoc signature. - const stage = mkdtempSync(join(dirname(output), ".scriptc-native-")); - try { - const outputDirectory = join(stage, "output"); - const inputDirectory = join(stage, "input"); - mkdirSync(outputDirectory); - mkdirSync(inputDirectory); - const stagedOutput = join(outputDirectory, basename(output)); - const sourceOutput = options.outputKind === "llvm"; - const input = sourceOutput ? stagedOutput : join(inputDirectory, "program.ll"); - const prepared = emitNativeInput(options, toolchain, entry, ffi, input); - if (!sourceOutput) { - const object = options.outputKind === "obj" ? stagedOutput : join(inputDirectory, "program" + toolchain.target.outputSuffixes.obj); - emitNativeObject({ - executable: toolchain.helperExecutable, packageRoot: toolchain.helperPackageRoot, - compilerVersion: toolchain.compilerVersion, target: toolchain.target, helper: toolchain.helper, - inputPath: input, outputPath: object, sourcePath: entry, optimization: options.optimization, - }); - if (options.outputKind === "exe") { - const runtime = stageNativeRuntimePack(toolchain.runtimePackRoot, join(stage, "runtime"), - toolchain.target, toolchain.compilerVersion, prepared.features, options.optimization); - const plan = executableLinkInputs({ - target: toolchain.target, programObject: object, - ffiLibraries: ffi?.libraries ?? [], ffiSystemLibraries: ffi?.systemLibraries ?? [], - ffiFrameworks: ffi?.frameworks ?? [], - runtimeObjects: runtime.runtimeObjects, runtimeArchives: runtime.archives, - runtimeSystemLibraries: runtime.systemLibraries, optimization: options.optimization, strip: options.strip, - }); - runNativeTool(toolchain.linker, [ - ...toolchain.linkerArgs, ...plan.driverFlags, ...plan.inputs, - ...plan.systemLibraries.map((name) => `-l${name}`), "-o", stagedOutput, - ]); - requireNativeArtifact(stagedOutput); - if (toolchain.target.platform === "darwin" && options.optimization === "dev" && !options.strip) { - runNativeTool(toolchain.dsymutil, [stagedOutput, "-o", stagedOutput + ".dSYM"]); - rmSync(output + ".dSYM", { recursive: true, force: true }); - renameSync(stagedOutput + ".dSYM", output + ".dSYM"); - } - } - } - if (options.keepLlvm && !sourceOutput) renameSync(join(inputDirectory, "program.ll"), output + ".ll"); - renameSync(stagedOutput, output); - if (options.outputKind === "exe" && toolchain.target.platform === "darwin" && (options.optimization !== "dev" || options.strip)) { - rmSync(output + ".dSYM", { recursive: true, force: true }); - } - return { - outputPath: output, stats: prepared.stats, - ...(options.keepLlvm && !sourceOutput ? { llvmPath: output + ".ll" } : {}), - }; - } finally { rmSync(stage, { recursive: true, force: true }); } -} diff --git a/packages/compiler/src/native/executable-cache.test.ts b/packages/compiler/src/native/executable-cache.test.ts new file mode 100644 index 00000000..3ee551cf --- /dev/null +++ b/packages/compiler/src/native/executable-cache.test.ts @@ -0,0 +1,159 @@ +import { spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, readFileSync, readdirSync, renameSync, rmSync, statSync, symlinkSync, utimesSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, test, vi } from "vitest"; +import { contentDigest, NativeCache } from "./cache.js"; +import { NativeExecutableCache } from "./executable-cache.js"; + +vi.mock("node:child_process", async (original) => ({ + ...await original(), spawnSync: vi.fn(), +})); + +const directories: string[] = []; +afterEach(() => { + vi.resetAllMocks(); + for (const path of directories.splice(0)) rmSync(path, { recursive: true, force: true }); +}); + +function fixture(debug = false) { + const root = mkdtempSync(join(tmpdir(), "scriptc-native-executable-")); + directories.push(root); + const inputs = join(root, "inputs"); + const sdk = join(root, "sdk"); + const stage = join(root, "stage"); + const restored = join(root, "restored"); + for (const directory of [inputs, sdk, stage, restored]) mkdirSync(directory); + const runtime = join(inputs, "runtime.o"); + const library = join(sdk, "libSystem.tbd"); + writeFileSync(runtime, "runtime"); + writeFileSync(library, "system library"); + const cache = new NativeCache(join(root, "cache")); + const key = contentDigest("llvm and options"); + const open = () => new NativeExecutableCache(cache, key, "clang", debug, [runtime]); + vi.mocked(spawnSync).mockReturnValue({ pid: 1, status: 0, signal: null, output: [], stdout: library + "\n", stderr: "" }); + const output = join(stage, "program"); + writeFileSync(output, Buffer.from([0, 128, 255, 10])); + if (debug) { + const contents = join(output + ".dSYM", "Contents"); + mkdirSync(join(contents, "Resources", "DWARF"), { recursive: true }); + writeFileSync(join(contents, "Info.plist"), "property list"); + writeFileSync(join(contents, "Resources", "DWARF", "program"), "debug information"); + } + const publish = () => { + const entry = open(); + expect(entry.trace(["-o", output], stage)).toBe(true); + entry.publish(output); + }; + return { root, inputs, sdk, stage, output, destination: join(restored, "program"), runtime, library, cache, key, open, publish }; +} + +test.each([false, true])("restores a complete executable and its debug information (%s)", (debug) => { + const f = fixture(debug); + f.publish(); + vi.mocked(spawnSync).mockClear(); + expect(f.open().restore(f.destination)).toBe(true); + expect(readFileSync(f.destination)).toEqual(readFileSync(f.output)); + if (process.platform !== "win32") expect(statSync(f.destination).mode & 0o111).toBe(0o111); + expect(spawnSync).not.toHaveBeenCalled(); + if (debug) { + expect(readFileSync(join(f.destination + ".dSYM/Contents/Resources/DWARF/program"), "utf8")).toBe("debug information"); + expect(readFileSync(join(f.destination + ".dSYM/Contents/Info.plist"), "utf8")).toBe("property list"); + } +}); + +test.each(["runtime", "library"] as const)("invalidates %s replacement even with restored size and mtime", (input) => { + const f = fixture(); + f.publish(); + const before = statSync(f[input]); + writeFileSync(f[input] + ".new", "x".repeat(before.size)); + utimesSync(f[input] + ".new", before.atime, before.mtime); + renameSync(f[input] + ".new", f[input]); + expect(f.open().restore(f.destination)).toBe(false); +}); + +test("adding a new SDK library search candidate invalidates the previous link", () => { + const f = fixture(); + f.publish(); + writeFileSync(join(f.sdk, "libSystem.dylib"), "new candidate"); + expect(f.open().restore(f.destination)).toBe(false); +}); + +test("a library search change during tracing prevents publication", () => { + const f = fixture(); + vi.mocked(spawnSync).mockImplementation(() => { + if (vi.mocked(spawnSync).mock.calls.length === 2) writeFileSync(join(f.sdk, "new-library.tbd"), "library"); + return { pid: 1, status: 0, signal: null, output: [], stdout: f.library + "\n", stderr: "" }; + }); + const entry = f.open(); + expect(entry.trace([], f.stage)).toBe(false); + entry.publish(f.output); + expect(f.open().restore(f.destination)).toBe(false); +}); + +test.skipIf(process.platform === "win32")("retargeting an input symlink invalidates the previous link", () => { + const f = fixture(); + const link = join(f.inputs, "current.o"); + symlinkSync(f.runtime, link); + const open = () => new NativeExecutableCache(f.cache, f.key, "clang", false, [link]); + const entry = open(); + expect(entry.trace([], f.stage)).toBe(true); + entry.publish(f.output); + expect(open().restore(f.destination)).toBe(true); + const replacement = join(f.inputs, "replacement.o"); + writeFileSync(replacement, "runtime"); + rmSync(link); + symlinkSync(replacement, link); + expect(open().restore(f.destination)).toBe(false); +}); + +test.each(["executable", "binary", "dsym"])("corrupt %s payloads are misses", (family) => { + const f = fixture(true); + f.publish(); + const directory = join(f.cache.root, family); + const key = readdirSync(directory).find((name) => /^[0-9a-f]{64}$/.test(name))!; + writeFileSync(join(directory, key), "corrupt"); + expect(f.open().restore(f.destination)).toBe(false); +}); + +test("invalid cache metadata is rejected even with a matching integrity digest", () => { + const f = fixture(); + f.publish(); + const bytes = f.cache.read("executable", f.key)!; + const entry = JSON.parse(bytes.toString()); + entry.inputs = []; + f.cache.write("executable", f.key, JSON.stringify(entry)); + expect(f.open().restore(f.destination)).toBe(false); +}); + +test("changing an input during emission prevents cache publication", () => { + const f = fixture(); + const entry = f.open(); + expect(entry.trace([], f.stage)).toBe(true); + writeFileSync(f.runtime, "replacement runtime"); + entry.publish(f.output); + expect(f.open().restore(f.destination)).toBe(false); +}); + +test("metadata cannot omit a required runtime input", () => { + const f = fixture(); + f.publish(); + const bytes = f.cache.read("executable", f.key)!; + const entry = JSON.parse(bytes.toString()); + entry.inputs = entry.inputs.filter((input: { path: string }) => input.path !== f.runtime); + f.cache.write("executable", f.key, JSON.stringify(entry)); + expect(f.open().restore(f.destination)).toBe(false); +}); + +test("a failed or empty link trace cannot publish a completed executable", () => { + const f = fixture(); + const entry = f.open(); + entry.publish(f.output); + expect(f.open().restore(f.destination)).toBe(false); + vi.mocked(spawnSync).mockReturnValue({ pid: 1, status: 0, signal: null, output: [], stdout: "", stderr: "" }); + expect(entry.trace([], f.stage)).toBe(false); + entry.publish(f.output); + expect(f.open().restore(f.destination)).toBe(false); + vi.mocked(spawnSync).mockReturnValue({ pid: 1, status: 1, signal: null, output: [], stdout: "", stderr: "failed" }); + expect(entry.trace([], f.stage)).toBe(false); +}); diff --git a/packages/compiler/src/native/executable-cache.ts b/packages/compiler/src/native/executable-cache.ts new file mode 100644 index 00000000..a1666493 --- /dev/null +++ b/packages/compiler/src/native/executable-cache.ts @@ -0,0 +1,210 @@ +import { spawnSync } from "node:child_process"; +import { chmodSync, mkdirSync, readFileSync, realpathSync, statSync, writeFileSync } from "node:fs"; +import { basename, delimiter, dirname, isAbsolute, join, resolve } from "node:path"; +import type { CompileRequestOptions } from "../compile-types.js"; +import type { FfiProfile } from "../ffi/ffi-manifest.js"; +import { driverTraceCandidates, linkTraceCandidate } from "../backend/link-trace.js"; +import { toolchainEnvironmentCachePolicy, toolchainEnvironmentFingerprint } from "../backend/toolchain-environment.js"; +import { runNativeTool } from "../backend/native-tools.js"; +import type { NativeRuntimeSelection } from "../backend/runtime-pack-native.js"; +import type { NativeToolchain } from "./toolchain.js"; +import { contentDigest, NativeCache } from "./cache.js"; + +interface InputIdentity { + path: string; + canonical: string; + kind: "file" | "directory"; + dev: number; + ino: number; + size: number; + mtime: number; + ctime: number; +} + +interface ExecutableEntry { + schema: "scriptc.native-executable.v1"; + inputs: InputIdentity[]; + binary: string; + symbols: string | null; +} + +function identity(path: string): InputIdentity { + path = resolve(path); + const info = statSync(path); + if (!info.isFile() && !info.isDirectory()) throw new Error(`unsupported native cache input: ${path}`); + return { path, canonical: realpathSync(path), kind: info.isFile() ? "file" : "directory", + dev: info.dev, ino: info.ino, size: info.size, mtime: info.mtimeMs, ctime: info.ctimeMs }; +} + +function stillMatches(inputs: InputIdentity[]): boolean { + try { return inputs.every((input) => JSON.stringify(identity(input.path)) === JSON.stringify(input)); } + catch { return false; } +} + +function commandPath(command: string): string { + if (command.includes("/") || command.includes("\\")) return resolve(command); + for (const directory of (process.env["PATH"] ?? "/usr/bin:/bin").split(delimiter)) { + const candidate = join(directory || process.cwd(), command); + try { if (statSync(candidate).isFile()) return candidate; } catch { /* Try the next PATH entry. */ } + } + throw new Error(`native command is unavailable: ${command}`); +} + +function toolOutput(executable: string, args: string[]): string { + const result = spawnSync(executable, args, { encoding: "utf8", stdio: "pipe" }); + if (result.error) throw result.error; + if (result.status !== 0 || result.signal !== null) throw new Error("could not trace native link inputs"); + return result.stdout + "\n" + result.stderr; +} + +function tracePaths(output: string, driver: boolean, roots: string[]): string[] { + const paths = new Set(); + for (const line of output.split(/\r?\n/)) { + for (const candidate of driver ? driverTraceCandidates(line) : linkTraceCandidate(line)) { + const option = driver ? ["-L", "-F", "--sysroot="].find((prefix) => candidate.startsWith(prefix)) : undefined; + const spelling = option === undefined ? candidate : candidate.slice(option.length); + if (!isAbsolute(spelling)) continue; + const path = resolve(spelling); + if (roots.some((root) => path === root || path.startsWith(root + "/"))) continue; + try { + const input = identity(path); + paths.add(path); + paths.add(dirname(path)); + if (input.kind === "directory" && path.endsWith(".sdk")) { + for (const name of ["SDKSettings.json", "SDKSettings.plist"]) { + const settings = join(path, name); + try { identity(settings); paths.add(settings); } catch { /* The SDK directory observes new settings files. */ } + } + } + } catch { /* Non-path trace tokens are not dependencies. */ } + } + } + return [...paths].sort(); +} + +function validEntry(value: unknown): value is ExecutableEntry { + if (value === null || typeof value !== "object") return false; + const entry = value as Partial; + return entry.schema === "scriptc.native-executable.v1" && typeof entry.binary === "string" && + /^[0-9a-f]{64}$/.test(entry.binary) && (entry.symbols === null || + (typeof entry.symbols === "string" && /^[0-9a-f]{64}$/.test(entry.symbols))) && + Array.isArray(entry.inputs) && entry.inputs.length > 0 && entry.inputs.every((input) => + input !== null && typeof input === "object" && typeof input.path === "string" && isAbsolute(input.path) && + typeof input.canonical === "string" && (input.kind === "file" || input.kind === "directory") && + [input.dev, input.ino, input.size, input.mtime, input.ctime].every((value) => typeof value === "number" && Number.isFinite(value))); +} + +function readSymbols(binary: string): Buffer { + const contents = join(binary + ".dSYM", "Contents"); + const plist = readFileSync(join(contents, "Info.plist")); + const dwarf = readFileSync(join(contents, "Resources", "DWARF", basename(binary))); + const header = Buffer.alloc(12); + header.write("SCDSYM01"); + header.writeUInt32LE(plist.length, 8); + return Buffer.concat([header, plist, dwarf]); +} + +function stageSymbols(bytes: Buffer, binary: string): void { + if (bytes.length < 12 || bytes.subarray(0, 8).toString() !== "SCDSYM01") throw new Error("invalid cached dSYM"); + const end = 12 + bytes.readUInt32LE(8); + if (end <= 12 || end >= bytes.length) throw new Error("invalid cached dSYM sizes"); + const contents = join(binary + ".dSYM", "Contents"); + const dwarf = join(contents, "Resources", "DWARF"); + mkdirSync(dwarf, { recursive: true }); + writeFileSync(join(contents, "Info.plist"), bytes.subarray(12, end)); + writeFileSync(join(dwarf, basename(binary)), bytes.subarray(end)); +} + +/** A completed executable is reusable only after the frontend has validated + * its source observations and emitted the same LLVM. Native inputs bracket + * helper verification, runtime staging and the actual link. */ +export class NativeExecutableCache { + private inputs: InputIdentity[]; + private traced = false; + + constructor(private readonly cache: NativeCache, private readonly key: string, + private readonly linker: string, private readonly debug: boolean, paths: string[]) { + this.inputs = [...new Set(paths)].sort().map(identity); + } + + restore(output: string): boolean { + try { + const metadata = this.cache.read("executable", this.key); + if (metadata === null) return false; + const entry: unknown = JSON.parse(metadata.toString("utf8")); + if (!validEntry(entry) || !stillMatches(entry.inputs) || !stillMatches(this.inputs)) return false; + if (!this.inputs.every((input) => entry.inputs.some((saved) => saved.path === input.path && + JSON.stringify(saved) === JSON.stringify(input)))) return false; + if (this.debug !== (entry.symbols !== null)) return false; + const binary = this.cache.read("binary", entry.binary); + const symbols = entry.symbols === null ? null : this.cache.read("dsym", entry.symbols); + if (binary === null || binary.length === 0 || (this.debug && symbols === null)) return false; + if (!stillMatches(entry.inputs)) return false; + if (symbols !== null) stageSymbols(symbols, output); + writeFileSync(output, binary); + chmodSync(output, 0o755); + return true; + } catch { return false; } + } + + /** Trace the real object-only link, including the SDK's transitive stubs. + * Directories from the driver's search line detect a newly added candidate. + * Private build inputs already belong to the LLVM/runtime cache identity. */ + trace(args: string[], stage: string): boolean { + this.traced = false; + try { + const roots = [resolve(stage), realpathSync(stage)]; + const dry = toolOutput(this.linker, [...args, "-###"]); + const driverInputs = tracePaths(dry, true, roots).map(identity); + const linked = toolOutput(this.linker, [...args, "-Wl,-t"]); + const paths = tracePaths(linked, false, roots); + if (paths.length === 0 || !stillMatches(this.inputs) || !stillMatches(driverInputs)) return false; + this.inputs.push(...driverInputs, ...paths.map(identity)); + this.traced = true; + return true; + } catch { return false; } + } + + publish(output: string): void { + try { + if (!this.traced || !stillMatches(this.inputs)) return; + const binary = readFileSync(output); + const symbols = this.debug ? readSymbols(output) : null; + const binaryKey = contentDigest(binary); + const symbolsKey = symbols === null ? null : contentDigest(symbols); + this.cache.write("binary", binaryKey, binary); + if (symbols !== null && symbolsKey !== null) this.cache.write("dsym", symbolsKey, symbols); + if (!stillMatches(this.inputs)) return; + const entry: ExecutableEntry = { schema: "scriptc.native-executable.v1", inputs: this.inputs, binary: binaryKey, symbols: symbolsKey }; + this.cache.write("executable", this.key, JSON.stringify(entry)); + } catch { /* Build artifacts remain valid when caching is unavailable. */ } + } +} + +export function openNativeExecutableCache(cache: NativeCache | null, toolchain: NativeToolchain, + llvm: string, options: CompileRequestOptions, ffi: FfiProfile | null, pack: NativeRuntimeSelection): NativeExecutableCache | null { + // Match the established complete-cache contract: default Apple driver, + // known runtime inputs, and no wrapper or caller-supplied library searches. + if (cache === null || toolchain.target.platform !== "darwin" || options.sanitize || + process.env["SCRIPTC_LINKER"] !== undefined || process.env["SCRIPTC_LLVM_HELPER"] !== undefined || + toolchain.linkerArgs.length !== 0 || ffi !== null || !toolchainEnvironmentCachePolicy().completeArtifacts) return null; + try { + const linker = commandPath(toolchain.linker); + if (realpathSync(linker) !== "/usr/bin/clang") return null; + const effective = commandPath(runNativeTool(linker, ["-print-prog-name=clang"]).trim()); + const debug = options.optimization === "dev" && !options.strip; + const paths = [linker, effective, toolchain.helperExecutable, join(toolchain.helperPackageRoot, "package.json"), + join(toolchain.runtimePackRoot, "package.json"), join(toolchain.runtimePackRoot, "runtime-pack.json"), + ...[...pack.selected.runtime, ...pack.selected.archives, ...pack.manifest.licenses].map((artifact) => join(pack.root, artifact.path))]; + if (debug) { + const dsymutil = commandPath(toolchain.dsymutil); + if (realpathSync(dsymutil) !== "/usr/bin/dsymutil") return null; + paths.push(dsymutil); + } + const key = contentDigest(JSON.stringify({ schema: 1, llvm: contentDigest(llvm), options, toolchain, + runtimeIdentity: pack.packageText, runtimeManifest: pack.manifestText, + linker: identity(linker), effective: identity(effective), environment: toolchainEnvironmentFingerprint(), + cwd: process.cwd(), path: process.env["PATH"] ?? null })); + return new NativeExecutableCache(cache, key, linker, debug, paths); + } catch { return null; } +} diff --git a/packages/compiler/src/native/file-identity.test.ts b/packages/compiler/src/native/file-identity.test.ts new file mode 100644 index 00000000..83b5875d --- /dev/null +++ b/packages/compiler/src/native/file-identity.test.ts @@ -0,0 +1,38 @@ +import { mkdtempSync, renameSync, rmSync, symlinkSync, utimesSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { nativeFileIdentity } from "./file-identity.js"; + +test("native tool identity invalidates same-size replacements with preserved mtime", () => { + const root = mkdtempSync(join(tmpdir(), "scriptc-file-identity-")); + try { + const path = join(root, "compiler"); + writeFileSync(path, "original"); + utimesSync(path, new Date(0), new Date(0)); + const first = nativeFileIdentity(path); + expect(nativeFileIdentity(path)).toBe(first); + const replacement = join(root, "new"); + writeFileSync(replacement, "replaced"); + utimesSync(replacement, new Date(0), new Date(0)); + renameSync(replacement, path); + expect(nativeFileIdentity(path)).not.toBe(first); + expect(() => nativeFileIdentity(root)).toThrow("not a file"); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test.skipIf(process.platform === "win32")("native tool identity follows a retargeted symlink", () => { + const root = mkdtempSync(join(tmpdir(), "scriptc-file-symlink-")); + try { + const current = join(root, "current"); + const first = join(root, "first"); + const second = join(root, "second"); + writeFileSync(first, "same"); + writeFileSync(second, "same"); + symlinkSync(first, current); + const before = nativeFileIdentity(current); + rmSync(current); + symlinkSync(second, current); + expect(nativeFileIdentity(current)).not.toBe(before); + } finally { rmSync(root, { recursive: true, force: true }); } +}); diff --git a/packages/compiler/src/native/file-identity.ts b/packages/compiler/src/native/file-identity.ts new file mode 100644 index 00000000..8bd6f19b --- /dev/null +++ b/packages/compiler/src/native/file-identity.ts @@ -0,0 +1,13 @@ +import { realpathSync, statSync } from "node:fs"; +import { resolve } from "node:path"; + +/** ctime and inode identity detect replaced or rewritten tools even when + * their size and mtime are preserved. Avoid hashing entire compiler binaries + * on every build; source observations still use their content digests. */ +export function nativeFileIdentity(path: string): string { + path = resolve(path); + const info = statSync(path); + if (!info.isFile()) throw new Error(`native compiler input is not a file: ${path}`); + return JSON.stringify({ path, canonical: realpathSync(path), dev: info.dev, ino: info.ino, + size: info.size, mtime: info.mtimeMs, ctime: info.ctimeMs }); +} diff --git a/packages/compiler/src/native/library.ts b/packages/compiler/src/native/library.ts new file mode 100644 index 00000000..b6609233 --- /dev/null +++ b/packages/compiler/src/native/library.ts @@ -0,0 +1,102 @@ +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { basename, join } from "node:path"; +import { requireNativeArtifact, runNativeTool } from "../backend/native-tools.js"; +import { localizeElfObject, mergeAndLocalizeCoffObjects } from "../backend/object-localize.js"; +import { RuntimePackError } from "../backend/runtime-pack-core.js"; +import type { NativeRuntimePack } from "../backend/runtime-pack-native.js"; +import { executableOptimizationLinkerArgs } from "../backend/targets.js"; +import type { NativeToolchain } from "./toolchain.js"; + +function archiver(toolchain: NativeToolchain): { executable: string; args: string[] } { + if (toolchain.archiver !== undefined) return { executable: toolchain.archiver, args: toolchain.archiverArgs ?? [] }; + const zig = /(?:^|[\\/])zig(?:\.exe)?$/.test(toolchain.linker); + return zig ? { executable: toolchain.linker, args: ["ar"] } : { executable: "ar", args: [] }; +} + +/** Archive member names are untrusted package data, even after digest verification. */ +export function runtimeArchiveMembers(listing: string): string[] { + const names = listing.trim().split(/\r?\n/).filter((name) => name !== "" && name !== "__.SYMDEF" && name !== "__.SYMDEF SORTED"); + if (new Set(names).size !== names.length || names.some((name) => !/^[A-Za-z0-9_.-]+\.o$/.test(name))) { + throw new RuntimePackError("runtime vendor archive has invalid or duplicate object members", "invalid"); + } + return names; +} + +/** Localize the runtime without changing the library's declared C ABI. */ +export function localizeNativeLibrary( + toolchain: NativeToolchain, stage: string, roots: string[], support: string[], keep: string[], +): string { + const combined = join(stage, "library.localized.o"); + const platform = toolchain.target.platform; + if (platform === "win32") { + writeFileSync(combined, mergeAndLocalizeCoffObjects( + roots.map((path) => readFileSync(path)), support.map((path) => readFileSync(path)), new Set(keep), + { roots: roots.map((path) => basename(path)), support: support.map((path) => basename(path)) }, + )); + return combined; + } + const ar = archiver(toolchain); + const supportArgs: string[] = []; + if (support.length > 0) { + const archive = join(stage, "support.a"); + runNativeTool(ar.executable, [...ar.args, "rcs", archive, ...support], undefined, { ...process.env, ZERO_AR_DATE: "1" }); + supportArgs.push(archive); + } + if (platform === "darwin") { + if (process.platform !== "darwin") throw new Error("Mach-O library localization requires a macOS host"); + const symbols = join(stage, "public.syms"); + writeFileSync(symbols, keep.map((name) => `_${name}\n`).join("")); + runNativeTool(toolchain.relocatableLinker ?? "ld", ["-r", ...roots, ...supportArgs, "-o", combined, "-exported_symbols_list", symbols]); + } else if (platform === "linux") { + runNativeTool(toolchain.linker, [...toolchain.linkerArgs, "-target", toolchain.target.linkerTargetTriple, + "-nostdlib", "-r", ...roots, ...supportArgs, "-o", combined]); + writeFileSync(combined, localizeElfObject(readFileSync(combined), new Set(keep))); + } else throw new Error(`library localization is unavailable for ${toolchain.target.name}`); + requireNativeArtifact(combined); + return combined; +} + +export function archiveNativeLibrary(options: { + toolchain: NativeToolchain; + programObject: string; + outputPath: string; + stage: string; + runtime: NativeRuntimePack; + localizeSymbols?: string[]; +}): void { + const { toolchain, stage } = options; + const ar = archiver(toolchain); + const program = options.programObject; + const support = [...options.runtime.runtimeObjects]; + for (let index = 0; index < options.runtime.archives.length; index++) { + const archive = options.runtime.archives[index]!; + const members = runtimeArchiveMembers(runNativeTool(ar.executable, [...ar.args, "t", archive])); + const directory = join(stage, `vendor-${index}`); + mkdirSync(directory); + if (members.length > 0) runNativeTool(ar.executable, [...ar.args, "x", archive, ...members], directory); + support.push(...members.map((name) => join(directory, name))); + } + const objects = options.localizeSymbols === undefined ? [program, ...support] + : [localizeNativeLibrary(toolchain, stage, [program], support, options.localizeSymbols)]; + runNativeTool(ar.executable, [...ar.args, "rcs", options.outputPath, ...objects], undefined, { ...process.env, ZERO_AR_DATE: "1" }); + requireNativeArtifact(options.outputPath); +} + +export function linkNativeWasmLibrary(options: { + toolchain: NativeToolchain; + programObject: string; + outputPath: string; + runtime: NativeRuntimePack; + optimization: "release" | "dev"; + exports: string[]; +}): void { + const { toolchain, runtime } = options; + runNativeTool(toolchain.linker, [ + ...toolchain.linkerArgs, "-target", toolchain.target.linkerTargetTriple, "-mexec-model=reactor", + ...executableOptimizationLinkerArgs("wasi", options.optimization), + ...options.exports.map((name) => `-Wl,--export=${name}`), + options.programObject, ...runtime.runtimeObjects, ...runtime.archives, + ...runtime.systemLibraries.map((name) => `-l${name}`), "-o", options.outputPath, + ]); + requireNativeArtifact(options.outputPath); +} diff --git a/packages/compiler/src/native/main.ts b/packages/compiler/src/native/main.ts deleted file mode 100644 index 60686196..00000000 --- a/packages/compiler/src/native/main.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { buildNative } from "./driver.js"; -import { loadNativeToolchain } from "./toolchain.js"; -import { NATIVE_HELP, parseNativeArguments } from "./arguments.js"; - -try { - const args = parseNativeArguments(process.argv.slice(2), process.execPath + ".json"); - if (args.help) console.log(NATIVE_HELP); - else { - const result = buildNative(args.build, loadNativeToolchain(args.toolchainPath)); - console.log(JSON.stringify(result)); - } -} catch (error) { - console.error(error instanceof Error ? error.message : String(error)); - process.exitCode = 1; -} diff --git a/packages/compiler/src/native/prepare.test.ts b/packages/compiler/src/native/prepare.test.ts new file mode 100644 index 00000000..384a36a7 --- /dev/null +++ b/packages/compiler/src/native/prepare.test.ts @@ -0,0 +1,71 @@ +import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, test } from "vitest"; +import { nativeCodegenTarget } from "../backend/targets.js"; +import { runFrontend, type FrontendFactory } from "../frontend/pipeline.js"; +import { loadProgram } from "../frontend/program-node.js"; +import { ts7Executable } from "../frontend/ts7/rpc-api.js"; +import { NativeCache } from "./cache.js"; +import { prepareNativeExecutable } from "./prepare.js"; +import type { NativeToolchain } from "./toolchain.js"; + +const scratch: string[] = []; +afterEach(() => { for (const path of scratch.splice(0)) rmSync(path, { recursive: true, force: true }); }); + +function fixture() { + const temporary = mkdtempSync(join(tmpdir(), "scriptc-native-frontend-")); + scratch.push(temporary); + const root = join(temporary, "project"); + mkdirSync(root); + const entry = join(root, "main.ts"); + const dependency = join(root, "value.ts"); + writeFileSync(entry, 'import { value } from "./value.js"; console.log(value);\n'); + writeFileSync(dependency, 'export const value = "first";\n'); + writeFileSync(join(root, "tsconfig.json"), JSON.stringify({ compilerOptions: { strict: true, types: [], target: "esnext", module: "nodenext" } })); + writeFileSync(join(root, "package.json"), '{"type":"module"}'); + const target = nativeCodegenTarget()!; + const toolchain: NativeToolchain = { + compilerVersion: "test", target, helper: target.helper, helperExecutable: "unused", helperPackageRoot: "unused", + runtimePackRoot: "unused", linker: "unused", linkerArgs: [], dsymutil: "unused", ts7Executable: ts7Executable(), + }; + const cache = new NativeCache(join(temporary, "cache")); + const options = { outPath: join(root, ".scriptc", "main"), outDir: join(root, ".scriptc"), optimization: "release" as const }; + let loads = 0; + const frontend: FrontendFactory = (path, npmStatic, externalTypes) => { + loads++; + return runFrontend(path, loadProgram, npmStatic, externalTypes); + }; + const prepare = () => prepareNativeExecutable(entry, options, null, toolchain, frontend, cache); + return { root, entry, dependency, cache, options, prepare, loads: () => loads }; +} + +test("warm frontend reuse survives output creation and invalidates imported source edits", () => { + const f = fixture(); + const first = f.prepare(); + expect(first.ok, JSON.stringify(first)).toBe(true); + if (!first.ok) return; + mkdirSync(f.options.outDir); + writeFileSync(f.options.outPath, "binary placeholder"); + writeFileSync(join(f.options.outDir, "main.ll"), first.llvm); + const warm = f.prepare(); + expect(warm).toEqual(first); + expect(f.loads()).toBe(1); + writeFileSync(f.dependency, 'export const value = "second";\n'); + const edited = f.prepare(); + expect(edited.ok).toBe(true); + if (edited.ok) expect(edited.llvm).not.toBe(first.llvm); + expect(f.loads()).toBe(2); +}); + +test("structurally invalid cached features cause a rebuild even with a valid digest", () => { + const f = fixture(); + expect(f.prepare().ok).toBe(true); + const family = join(f.cache.root, "frontend"); + const key = readdirSync(family).find((name) => /^[0-9a-f]{64}$/.test(name))!; + const payload = JSON.parse(readFileSync(join(family, key), "utf8")); + payload.input.features = {}; + f.cache.write("frontend", key, JSON.stringify(payload)); + expect(f.prepare().ok).toBe(true); + expect(f.loads()).toBe(2); +}); diff --git a/packages/compiler/src/native/prepare.ts b/packages/compiler/src/native/prepare.ts new file mode 100644 index 00000000..608751e7 --- /dev/null +++ b/packages/compiler/src/native/prepare.ts @@ -0,0 +1,178 @@ +import { basename, dirname, join, resolve } from "node:path"; +import type { CompileFailure, CompileLibraryOptions, CompileRequestOptions } from "../compile-types.js"; +import { emitLlvmModule } from "../backend/llvm/emitter.js"; +import { executableLinkFeatures } from "../backend/executable-features.js"; +import type { NativeLinkFeatures } from "../backend/native-link-info-core.js"; +import { prepareExecutableModule } from "../executable/prepare.js"; +import type { FfiProfile } from "../ffi/ffi-manifest.js"; +import type { FrontendFactory } from "../frontend/pipeline.js"; +import { FrontendInputTracker, frontendInputsStillMatch, validFrontendInputSnapshot, type FrontendInputSnapshot, type FrontendInputExclusions, trackedReadFile } from "../frontend/input-tracker.js"; +import { provenanceSources } from "../frontend/provenance-registry.js"; +import { prepareLibrary } from "../library/prepare.js"; +import type { LibraryProfile } from "../library/library-profile.js"; +import { serializeModule } from "../ir/serialize.js"; +import { contentDigest, type NativeCache } from "./cache.js"; +import type { NativeToolchain } from "./toolchain.js"; +import { nativeFileIdentity } from "./file-identity.js"; + +export interface NativeExecutableInput { + ok: true; + llvm: string; + ir: string | null; + sidecarJson: string | null; + features: NativeLinkFeatures; + sources: [string, string][]; +} + +interface FrontendCacheEntry { + schema: "scriptc.native-frontend.v2"; + probes: FrontendInputSnapshot; + input: NativeExecutableInput; +} + +function implementationIdentity(toolchain: NativeToolchain): string { + // Release versions alone cannot distinguish two development builds. + return contentDigest(JSON.stringify({ + compiler: nativeFileIdentity(process.execPath), + checker: nativeFileIdentity(toolchain.ts7Executable), + evaluator: toolchain.comptimeExecutable === undefined ? null : nativeFileIdentity(toolchain.comptimeExecutable), + version: toolchain.compilerVersion, + })); +} + +function validFeatures(value: unknown): value is NativeLinkFeatures { + if (value === null || typeof value !== "object" || Array.isArray(value)) return false; + const features = value as Record; + return ["dynamic", "regex", "copying", "textDecoderLegacy", "fileHandle", "fetch", "netIsland", "zlib", + "assert", "inspect", "dynInvoke", "dc", "dynAsync", "events", "emitter", "symbol", "bigint", "searchParams", + "qs", "parseArgs", "stream", "net", "http", "http2", "dgram", "watch", "foreignFfi", "nodeTest", "tls", "tlsCa"] + .every((name) => typeof features[name] === "boolean"); +} + +function validCachedInput(value: unknown): value is FrontendCacheEntry { + if (value === null || typeof value !== "object") return false; + const entry = value as Partial; + const input = entry.input; + if (entry.schema !== "scriptc.native-frontend.v2" || !validFrontendInputSnapshot(entry.probes) || + input === undefined || input === null || typeof input !== "object") return false; + return input.ok === true && typeof input.llvm === "string" && (input.ir === null || typeof input.ir === "string") && + (input.sidecarJson === null || typeof input.sidecarJson === "string") && + validFeatures(input.features) && + Array.isArray(input.sources) && input.sources.every((pair) => Array.isArray(pair) && pair.length === 2 && + typeof pair[0] === "string" && typeof pair[1] === "string"); +} + +/** Keep the checker and typed IR out of the native optimizer's live heap. */ +export function prepareNativeExecutable( + entry: string, options: CompileRequestOptions, ffi: FfiProfile | null, + toolchain: NativeToolchain, frontend: FrontendFactory, cache: NativeCache | null, +): NativeExecutableInput | CompileFailure { + const outputKind = options.outputKind ?? "exe"; + const stem = basename(entry).replace(/\.(ts|mts|cts|js|mjs|cjs)$/, ""); + const outputPaths = [resolve(options.outPath), resolve(options.outPath) + ".dSYM", + resolve(options.outDir, stem + ".ll"), resolve(options.outDir, stem + ".ir.json")]; + const directories = new Set(); + for (const output of outputPaths) { + for (let directory = dirname(output); ; directory = dirname(directory)) { + directories.add(directory); + if (dirname(directory) === directory) break; + } + } + const exclusions = { outputPaths, outputDirectories: [...directories] }; + let key: string | null = null; + if (cache !== null && provenanceSources() === null) { + try { + key = contentDigest(JSON.stringify({ schema: 1, entry, options, ffi, implementation: implementationIdentity(toolchain), + target: toolchain.target.name, declarations: toolchain.declarationsRoot ?? null, + environment: Object.entries(process.env).filter(([name]) => name.startsWith("SCRIPTC_")).sort(([a], [b]) => a.localeCompare(b)), + })); + const bytes = cache.read("frontend", key); + if (bytes !== null) { + const cached: unknown = JSON.parse(bytes.toString("utf8")); + if (validCachedInput(cached) && frontendInputsStillMatch(cached.probes, exclusions)) { + const input: NativeExecutableInput = cached.input; + return input; + } + } + } catch { key = null; } + } + const tracker = new FrontendInputTracker(); + const prepared = tracker.runSynchronous(() => prepareExecutableModule(entry, options, ffi, toolchain.target.platform, frontend)); + if (!prepared.ok) return prepared; + const ir = outputKind === "ir" || options.emitIr ? serializeModule(prepared.mod) : null; + const llvm = outputKind === "ir" ? "" : emitLlvmModule(prepared.mod, { + targetTriple: toolchain.target.llvmTriple, pointerBits: toolchain.target.pointerBits, + wasi: toolchain.target.platform === "wasi", runtimeAbiMarker: outputKind === "obj" || outputKind === "exe", + ...(options.optimization === "dev" && !options.strip ? { debugSources: prepared.sourceTexts } : {}), + }); + const input: NativeExecutableInput = { ok: true, llvm, ir, sidecarJson: null, + features: executableLinkFeatures(prepared.mod, options.dynamic ?? false), sources: [...prepared.sourceTexts] }; + const probes = tracker.snapshot(); + if (cache !== null && key !== null && probes.stable && frontendInputsStillMatch(probes, exclusions)) { + const saved: FrontendCacheEntry = { schema: "scriptc.native-frontend.v2", probes, input }; + cache.write("frontend", key, JSON.stringify(saved)); + } + return input; +} + +/** Library preparation also ends before the native optimizer starts. The + * cached form contains emitted artifacts, source frames, and the sidecar. */ +export function prepareNativeLibrary( + profile: LibraryProfile, options: CompileLibraryOptions, archivePath: string, + toolchain: NativeToolchain, frontend: FrontendFactory, cache: NativeCache | null, +): NativeExecutableInput | CompileFailure { + const stem = basename(profile.entry).replace(/\.(ts|mts|cts|js|mjs|cjs)$/, ""); + const outputPaths = [archivePath, resolve(options.outDir, stem + ".lib.ll"), resolve(options.outDir, stem + ".lib.ir.json")]; + if (profile.sidecar !== null) outputPaths.push(profile.sidecar.path === null + ? archivePath + ".contract.json" : resolve(dirname(archivePath), profile.sidecar.path)); + const directories = new Set(); + for (const output of outputPaths) { + for (let directory = dirname(output); ; directory = dirname(directory)) { + directories.add(directory); + if (dirname(directory) === directory) break; + } + } + const exclusions: FrontendInputExclusions = { outputPaths, outputDirectories: [...directories] }; + let key: string | null = null; + if (cache !== null && provenanceSources() === null) { + try { + key = contentDigest(JSON.stringify({ schema: 1, kind: "library", entry: profile.entry, profile: profile.profileBytes, + options, implementation: implementationIdentity(toolchain), target: toolchain.target.name, + declarations: toolchain.declarationsRoot ?? null, + environment: Object.entries(process.env).filter(([name]) => name.startsWith("SCRIPTC_")).sort(([a], [b]) => a.localeCompare(b)), + })); + const bytes = cache.read("frontend", key); + if (bytes !== null) { + const cached: unknown = JSON.parse(bytes.toString("utf8")); + if (validCachedInput(cached) && frontendInputsStillMatch(cached.probes, exclusions)) { + const input: NativeExecutableInput = cached.input; + return input; + } + } + } catch { key = null; } + } + const tracker = new FrontendInputTracker(); + const prepared = tracker.runSynchronous(() => { + for (let directory = dirname(profile.entry); ; directory = dirname(directory)) { + trackedReadFile(join(directory, "tsconfig.json")); + trackedReadFile(join(directory, "package.json")); + if (directory === dirname(resolve(options.profilePath)) || dirname(directory) === directory) break; + } + return prepareLibrary(profile, options.profilePath, toolchain.compilerVersion, toolchain.target.platform, frontend); + }); + if (!prepared.ok) return prepared; + const input: NativeExecutableInput = { + ok: true, llvm: emitLlvmModule(prepared.mod, { + targetTriple: toolchain.target.llvmTriple, pointerBits: toolchain.target.pointerBits, + wasi: toolchain.target.platform === "wasi", + }), + ir: options.emitIr ? serializeModule(prepared.mod) : null, sidecarJson: prepared.sidecarJson, + features: executableLinkFeatures(prepared.mod, false), sources: [...prepared.sourceTexts], + }; + const probes = tracker.snapshot(); + if (cache !== null && key !== null && probes.stable && frontendInputsStillMatch(probes, exclusions)) { + const saved: FrontendCacheEntry = { schema: "scriptc.native-frontend.v2", probes, input }; + cache.write("frontend", key, JSON.stringify(saved)); + } + return input; +} diff --git a/packages/compiler/src/native/sanitizer.ts b/packages/compiler/src/native/sanitizer.ts new file mode 100644 index 00000000..142efe4f --- /dev/null +++ b/packages/compiler/src/native/sanitizer.ts @@ -0,0 +1,128 @@ +import { createHash } from "node:crypto"; +import { mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; +import { basename, join } from "node:path"; +import { NativeCodegenError } from "../backend/native-codegen-core.js"; +import { requireNativeArtifact, runNativeTool } from "../backend/native-tools.js"; +import type { NativeRuntimePack, NativeRuntimeSelection } from "../backend/runtime-pack-native.js"; +import type { RuntimePackMode } from "../backend/runtime-pack-core.js"; +import { QJS_ENGINE_SOURCES, LRE_SOURCES, ZLIB_SOURCES } from "../backend/vendor-inputs.js"; +import { contentDigest, type NativeCache } from "./cache.js"; +import type { NativeToolchain } from "./toolchain.js"; + +/** Sanitizers retain the documented external Clang requirement. Normal + * builds use the installed precompiled runtime pack without compiling C. */ +export function sanitizerDriver(toolchain: NativeToolchain): string { + if (toolchain.target.platform !== process.platform || toolchain.target.architecture !== process.arch || + toolchain.target.name.startsWith("ios-") || toolchain.target.name.startsWith("android-")) { + throw new NativeCodegenError("SC3002", "--sanitize requires a native host target"); + } + return process.env["SCRIPTC_CC"] ?? "clang"; +} + +export function sanitizerFlags(toolchain: NativeToolchain, optimization: "release" | "dev"): string[] { + return ["-target", toolchain.target.llvmTriple, optimization === "dev" ? "-O0" : "-O1", "-g", "-fsanitize=address"]; +} + +function sourceFingerprint(root: string): string { + const hash = createHash("sha256"); + const visit = (directory: string): void => { + for (const entry of readdirSync(directory, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) { + if (entry.name.startsWith(".")) continue; + const path = join(directory, entry.name); + if (entry.isDirectory()) visit(path); + else if (/\.(?:c|h|inc|def)$/.test(entry.name)) hash.update(path).update("\0").update(readFileSync(path)).update("\0"); + } + }; + visit(join(root, "src")); + visit(join(root, "vendor")); + return hash.digest("hex"); +} + +/** The installed pack's matrix also selects sanitizer source units and + * feature defines, keeping native and seed runtime reachability aligned. */ +export function buildSanitizedRuntime( + toolchain: NativeToolchain, selection: NativeRuntimeSelection, stage: string, + mode: RuntimePackMode, cache: NativeCache | null, +): NativeRuntimePack { + const driver = sanitizerDriver(toolchain); + const sourceRoot = toolchain.runtimeSourceRoot; + if (sourceRoot === undefined) throw new NativeCodegenError("SC3003", "runtime development sources are missing from this installation"); + const identity = JSON.parse(readFileSync(join(sourceRoot, "package.json"), "utf8")) as { name?: string; version?: string }; + if (identity.name !== "@scriptc/runtime" || identity.version !== toolchain.compilerVersion) { + throw new NativeCodegenError("SC3003", "runtime development source version does not match this compiler"); + } + const sourceIdentity = sourceFingerprint(sourceRoot); + const compilerIdentity = runNativeTool(driver, ["--version"]); + const flags = sanitizerFlags(toolchain, selection.flavor); + const src = join(sourceRoot, "src"); + const vendor = join(sourceRoot, "vendor"); + const quickjs = join(vendor, "quickjs-ng"); + const zlib = join(vendor, "zlib"); + const mbedtls = join(vendor, "mbedtls"); + const cacheIdentity = contentDigest(JSON.stringify({ schema: 1, sourceIdentity, compilerIdentity, driver, + target: toolchain.target, environment: process.env })); + mkdirSync(stage, { recursive: true }); + const pending: { key: string; output: string }[] = []; + const compile = (source: string, name: string, args: string[]): string => { + const output = join(stage, name); + const key = contentDigest(JSON.stringify({ cacheIdentity, source, args: [...flags, ...args] })); + const cached = cache?.read("sanitizer", key); + if (cached !== null && cached !== undefined) writeFileSync(output, cached); + else { + runNativeTool(driver, [...flags, ...args, "-c", source, "-o", output]); + requireNativeArtifact(output); + pending.push({ key, output }); + } + return output; + }; + const key = mode === "executable" ? selection.flavor : mode + "-" + selection.flavor; + const flavors = selection.manifest.flavors; + const flavor = mode === "library" + ? selection.flavor === "dev" ? flavors["library-dev"] : flavors["library-release"] + : mode === "library-thread" + ? selection.flavor === "dev" ? flavors["library-thread-dev"] : flavors["library-thread-release"] + : selection.flavor === "dev" ? flavors.dev : flavors.release; + if (flavor === undefined) throw new NativeCodegenError("SC3003", `runtime pack lacks ${key}`); + const wanted = new Set(selection.selected.runtime.map((artifact) => artifact.path)); + const common = ["-std=c11", "-pthread", "-DSCR_RC_AUDIT", "-fno-math-errno", "-fno-strict-aliasing", + "-Wno-deprecated-declarations", "-I", src, "-I", quickjs, "-I", zlib, "-I", join(mbedtls, "include"), + ...(toolchain.target.platform === "linux" ? ["-D_GNU_SOURCE", "-ffunction-sections", "-fdata-sections"] : []), + ...(toolchain.target.name.endsWith("-musl") ? ["-DSCR_MUSL"] : [])]; + const runtimeObjects: string[] = []; + for (const unit of flavor.runtime_units) { + for (const variant of unit.variants) { + if (wanted.has(variant.path)) runtimeObjects.push(compile(join(src, unit.source), unit.source.replace(/\.c$/, ".o"), + [...common, ...variant.defines.map((define) => "-D" + define)])); + } + } + const archives: string[] = []; + for (const archive of selection.selected.archives) { + const name = basename(archive.path).replace(/^libscriptc-/, "").replace(/\.a$/, ""); + let sources: readonly string[]; + let directory: string; + let args: string[]; + if (name === "quickjs") { + sources = QJS_ENGINE_SOURCES; + directory = quickjs; + args = ["-std=gnu11", "-fvisibility=hidden", "-funsigned-char", "-DQUICKJS_NG_BUILD", "-D_GNU_SOURCE", "-DNDEBUG", "-I", quickjs]; + } else if (name === "libregexp") { + sources = LRE_SOURCES; directory = quickjs; args = ["-std=c11", "-I", quickjs]; + } else if (name === "zlib") { + sources = ZLIB_SOURCES; directory = zlib; args = ["-std=c11", "-I", zlib]; + } else if (name === "mbedtls") { + directory = join(mbedtls, "library"); + sources = readdirSync(directory).filter((file) => file.endsWith(".c") && !file.startsWith(".")).sort(); + args = ["-std=c11", "-I", join(mbedtls, "include"), "-I", directory]; + } else throw new NativeCodegenError("SC3003", `unsupported runtime vendor archive: ${name}`); + const objects = sources.map((source) => compile(join(directory, source), name + "-" + source.replace(/\.c$/, ".o"), args)); + const output = join(stage, "libscriptc-" + name + ".a"); + runNativeTool(toolchain.archiver ?? "ar", [...(toolchain.archiverArgs ?? []), "rcs", output, ...objects], undefined, { ...process.env, ZERO_AR_DATE: "1" }); + requireNativeArtifact(output); + archives.push(output); + } + if (sourceFingerprint(sourceRoot) !== sourceIdentity || runNativeTool(driver, ["--version"]) !== compilerIdentity) { + throw new NativeCodegenError("SC3004", "runtime sources or sanitizer compiler changed during the build; retry"); + } + for (const item of pending) cache?.write("sanitizer", item.key, readFileSync(item.output)); + return { runtimeObjects, archives, systemLibraries: selection.selected.systemLibraries }; +} diff --git a/packages/compiler/src/native/task.test.ts b/packages/compiler/src/native/task.test.ts new file mode 100644 index 00000000..5c6e739c --- /dev/null +++ b/packages/compiler/src/native/task.test.ts @@ -0,0 +1,18 @@ +import { expect, test } from "vitest"; +import { runCompilerTask } from "./task.js"; + +test("compiler work starts after the caller can suspend and preserves its result", async () => { + const events: string[] = []; + const result = { ok: true, diagnostics: [] }; + const pending = runCompilerTask(() => { events.push("compile"); return result; }); + events.push("caller"); + expect(events).toEqual(["caller"]); + expect(await pending).toBe(result); + expect(events).toEqual(["caller", "compile"]); +}); + +test("synchronous compiler failures reject the task without escaping its microtask", async () => { + const failure = new Error("compiler failed"); + await expect(runCompilerTask(() => { throw failure; })).rejects.toBe(failure); + expect(await runCompilerTask(() => 42)).toBe(42); +}); diff --git a/packages/compiler/src/native/task.ts b/packages/compiler/src/native/task.ts new file mode 100644 index 00000000..f7bb6bec --- /dev/null +++ b/packages/compiler/src/native/task.ts @@ -0,0 +1,11 @@ +/** Compiler traversals use the main stack; async fibers have small stacks + * intended for suspended application work. Queueing also preserves the + * promise rejection boundary for synchronous compiler failures. */ +export function runCompilerTask(work: () => T): Promise { + return new Promise((resolve, reject) => { + queueMicrotask(() => { + try { resolve(work()); } + catch (error) { reject(error); } + }); + }); +} diff --git a/packages/compiler/src/native/toolchain.ts b/packages/compiler/src/native/toolchain.ts index c5adc827..e045ac03 100644 --- a/packages/compiler/src/native/toolchain.ts +++ b/packages/compiler/src/native/toolchain.ts @@ -1,7 +1,26 @@ import { readFileSync } from "node:fs"; import { dirname, isAbsolute, join, resolve } from "node:path"; -import { NATIVE_TARGETS, type NativeTargetSpec } from "../backend/targets.js"; -import type { NativeToolchain } from "./driver.js"; +import { NATIVE_TARGETS, selectNativeTarget, type NativeTargetSpec, type NativeHelperSpec } from "../backend/targets.js"; + +export interface NativeToolchain { + compilerVersion: string; + ts7Executable: string; + target: NativeTargetSpec; + helper: NativeHelperSpec; + helperExecutable: string; + helperPackageRoot: string; + runtimePackRoot: string; + runtimeSourceRoot?: string; + linker: string; + linkerArgs: string[]; + dsymutil: string; + archiver?: string; + archiverArgs?: string[]; + relocatableLinker?: string; + declarationsRoot?: string; + comptimeExecutable?: string; + wasiNodeRunner?: string; +} /** Distribution paths are relative to this manifest, so moving an installed * compiler does not bake the seed machine's paths into the native binary. */ @@ -12,9 +31,17 @@ export interface NativeToolchainManifest { ts7: string; llvm_package: string; runtime_pack: string; + runtime_sources?: string; linker: string; linker_args: string[]; dsymutil: string; + declarations?: string; + runtime_packs?: { target: string; path: string }[]; + archiver?: string; + archiver_args?: string[]; + relocatable_linker?: string; + comptime?: string; + wasi_node_runner?: string; } function pathFrom(root: string, value: string): string { @@ -25,7 +52,7 @@ function commandFrom(root: string, value: string): string { return value.includes("/") || value.includes("\\") ? pathFrom(root, value) : value; } -export function loadNativeToolchain(path: string): NativeToolchain { +export function loadNativeToolchain(path: string, env: NodeJS.ProcessEnv = {}): NativeToolchain { const raw: unknown = JSON.parse(readFileSync(path, "utf8")); if (raw === null || typeof raw !== "object" || Array.isArray(raw)) throw new Error("invalid native toolchain manifest"); const value = raw as Record; @@ -39,18 +66,46 @@ export function loadNativeToolchain(path: string): NativeToolchain { } } const manifest = raw as NativeToolchainManifest; + for (const name of ["runtime_sources", "declarations", "archiver", "relocatable_linker", "comptime", "wasi_node_runner"]) { + if (value[name] !== undefined && (typeof value[name] !== "string" || value[name] === "")) throw new Error(`native toolchain requires a nonempty string for ${name}`); + } + const archiverArgs: unknown = value.archiver_args; + const runtimePacks: unknown = value.runtime_packs; + if (archiverArgs !== undefined && (!Array.isArray(archiverArgs) || !archiverArgs.every((arg) => typeof arg === "string"))) { + throw new Error("native toolchain requires a string array for archiver_args"); + } + if (runtimePacks !== undefined && (!Array.isArray(runtimePacks) || !runtimePacks.every((item) => { + if (item === null || typeof item !== "object") return false; + const pack = item as Record; + return typeof pack.target === "string" && typeof pack.path === "string" && pack.path !== ""; + }))) { + throw new Error("native toolchain requires target/path entries for runtime_packs"); + } const targets: readonly NativeTargetSpec[] = NATIVE_TARGETS; - const target = targets.find((item) => item.name === manifest.target); - if (target === undefined || target.platform === "wasi") throw new Error(`unsupported native compiler target: ${manifest.target}`); + const host = targets.find((item) => item.name === manifest.target); + if (host === undefined || host.platform === "wasi") throw new Error(`unsupported native compiler target: ${manifest.target}`); + const target = selectNativeTarget(env["SCRIPTC_TARGET"] ?? "", host); + if (target === null) throw new Error(`unsupported native compiler target: ${env["SCRIPTC_TARGET"]}`); const root = dirname(resolve(path)); - const helperPackageRoot = pathFrom(root, manifest.llvm_package); + const helperPackageRoot = pathFrom(root, env["SCRIPTC_LLVM_PACKAGE"] ?? manifest.llvm_package); + const runtimePath = manifest.runtime_packs?.find((pack) => pack.target === target.name)?.path + ?? (target.name === host.name ? manifest.runtime_pack : join(dirname(manifest.runtime_pack), target.runtimePackPackage.replace("@scriptc/", ""))); + const linker = env["SCRIPTC_LINKER"] ?? (target.name === host.name ? manifest.linker : target.defaultLinker); + const linkerArgs = env["SCRIPTC_LINKER"] !== undefined ? [] : target.name === host.name ? manifest.linker_args : [...target.defaultLinkerArgs]; return { compilerVersion: manifest.compiler_version, ts7Executable: pathFrom(root, manifest.ts7), - target, helper: target.helper, helperPackageRoot, - helperExecutable: join(helperPackageRoot, "bin", target.platform === "win32" ? "scriptc-llvm-codegen.exe" : "scriptc-llvm-codegen"), - runtimePackRoot: pathFrom(root, manifest.runtime_pack), - linker: commandFrom(root, manifest.linker), linkerArgs: manifest.linker_args, - dsymutil: commandFrom(root, manifest.dsymutil), + target, helper: host.helper, helperPackageRoot, + helperExecutable: env["SCRIPTC_LLVM_HELPER"] ?? join(helperPackageRoot, "bin", host.platform === "win32" ? "scriptc-llvm-codegen.exe" : "scriptc-llvm-codegen"), + runtimePackRoot: pathFrom(root, env["SCRIPTC_RUNTIME_PACK"] ?? runtimePath), + ...(manifest.runtime_sources === undefined ? {} : { runtimeSourceRoot: pathFrom(root, manifest.runtime_sources) }), + linker: commandFrom(root, linker), linkerArgs, + dsymutil: commandFrom(root, env["SCRIPTC_DSYMUTIL"] ?? manifest.dsymutil), + ...(manifest.declarations === undefined ? {} : { declarationsRoot: pathFrom(root, manifest.declarations) }), + ...(manifest.archiver === undefined ? {} : { archiver: commandFrom(root, manifest.archiver) }), + ...(manifest.archiver === undefined ? {} : { archiverArgs: manifest.archiver_args ?? [] }), + ...(manifest.relocatable_linker === undefined ? {} : { relocatableLinker: commandFrom(root, manifest.relocatable_linker) }), + ...(manifest.comptime === undefined ? {} : { comptimeExecutable: pathFrom(root, manifest.comptime) }), + ...(manifest.wasi_node_runner === undefined ? {} : { wasiNodeRunner: pathFrom(root, manifest.wasi_node_runner) }), }; } diff --git a/packages/compiler/test/ts7/baselines/order-parity.json b/packages/compiler/test/ts7/baselines/order-parity.json index fa78acd8..3d9974fe 100644 --- a/packages/compiler/test/ts7/baselines/order-parity.json +++ b/packages/compiler/test/ts7/baselines/order-parity.json @@ -7277,18 +7277,42 @@ ], "diags": [] }, + "/tests/corpus/3068-url-checked-identity.ts": { + "order": [ + "/tests/corpus/3068-url-checked-identity.ts" + ], + "diags": [] + }, + "/tests/corpus/3069-set-checked-seed.js": { + "order": [ + "/tests/corpus/3069-set-checked-seed.js" + ], + "diags": [] + }, "/tests/corpus/3069-tuple-spread.ts": { "order": [ "/tests/corpus/3069-tuple-spread.ts" ], "diags": [] }, + "/tests/corpus/3070-checked-object-accessors.js": { + "order": [ + "/tests/corpus/3070-checked-object-accessors.js" + ], + "diags": [] + }, "/tests/corpus/3070-collection-views.ts": { "order": [ "/tests/corpus/3070-collection-views.ts" ], "diags": [] }, + "/tests/corpus/3071-checked-specialization-narrowing.js": { + "order": [ + "/tests/corpus/3071-checked-specialization-narrowing.js" + ], + "diags": [] + }, "/tests/corpus/3071-map-contextual-record-seeds.ts": { "order": [ "/tests/corpus/3071-map-contextual-record-seeds.ts" @@ -7301,6 +7325,18 @@ ], "diags": [] }, + "/tests/corpus/3072-weak-process-streams.js": { + "order": [ + "/tests/corpus/3072-weak-process-streams.js" + ], + "diags": [] + }, + "/tests/corpus/3073-optional-callable-module.js": { + "order": [ + "/tests/corpus/3073-optional-callable-module.js" + ], + "diags": [] + }, "/tests/corpus/3073-union-keyed-reads.ts": { "order": [ "/tests/corpus/3073-union-keyed-reads.ts" @@ -7655,6 +7691,12 @@ ], "diags": [] }, + "/tests/corpus/3124-array-prototype-borrow.js": { + "order": [ + "/tests/corpus/3124-array-prototype-borrow.js" + ], + "diags": [] + }, "/tests/corpus/400-fib.ts": { "order": [ "/tests/corpus/400-fib.ts" @@ -7679,6 +7721,18 @@ ], "diags": [] }, + "/tests/corpus/4030-runtime-object-truthiness.ts": { + "order": [ + "/tests/corpus/4030-runtime-object-truthiness.ts" + ], + "diags": [] + }, + "/tests/corpus/4031-event-emitter-long-tuples.ts": { + "order": [ + "/tests/corpus/4031-event-emitter-long-tuples.ts" + ], + "diags": [] + }, "/tests/corpus/404-rc-stress.ts": { "order": [ "/tests/corpus/404-rc-stress.ts" @@ -8608,36 +8662,229 @@ ], "diags": [] }, + "/tests/corpus/array-buffer-conversions.js": { + "order": [ + "/tests/corpus/array-buffer-conversions.js" + ], + "diags": [] + }, + "/tests/corpus/array-buffer-storage.ts": { + "order": [ + "/tests/corpus/array-buffer-storage.ts" + ], + "diags": [] + }, + "/tests/corpus/array-fill-unit-values.ts": { + "order": [ + "/tests/corpus/array-fill-unit-values.ts" + ], + "diags": [] + }, + "/tests/corpus/array-optional-number-write-index.ts": { + "order": [ + "/tests/corpus/array-optional-number-write-index.ts" + ], + "diags": [] + }, + "/tests/corpus/array-refined-storage.ts": { + "order": [ + "/tests/corpus/array-refined-storage.ts" + ], + "diags": [] + }, + "/tests/corpus/array-sort-unknown-comparator.ts": { + "order": [ + "/tests/corpus/array-sort-unknown-comparator.ts" + ], + "diags": [] + }, + "/tests/corpus/async-backend-factory.mjs": { + "order": [ + "/tests/corpus/async-backend-factory.mjs" + ], + "diags": [] + }, "/tests/corpus/async-local-typed-results.ts": { "order": [ "/tests/corpus/async-local-typed-results.ts" ], "diags": [] }, + "/tests/corpus/bigint-checked-casts.ts": { + "order": [ + "/tests/corpus/bigint-checked-casts.ts" + ], + "diags": [] + }, + "/tests/corpus/bigint-checked-storage.js": { + "order": [ + "/tests/corpus/bigint-checked-storage.js" + ], + "diags": [] + }, "/tests/corpus/bigint-union-conversion.ts": { "order": [ "/tests/corpus/bigint-union-conversion.ts" ], "diags": [] }, + "/tests/corpus/boxed-error-rethrows.mjs": { + "order": [ + "/tests/corpus/boxed-error-rethrows.mjs" + ], + "diags": [] + }, "/tests/corpus/buffer-primitive-predicate.ts": { "order": [ "/tests/corpus/buffer-primitive-predicate.ts" ], "diags": [] }, + "/tests/corpus/callable-module-members.mjs": { + "order": [ + "/tests/corpus/callable-module-members.mjs" + ], + "diags": [] + }, + "/tests/corpus/callable-properties.mjs": { + "order": [ + "/tests/corpus/callable-properties.mjs" + ], + "diags": [] + }, + "/tests/corpus/callable-property-cycles.js": { + "order": [ + "/tests/corpus/callable-property-cycles.js" + ], + "diags": [] + }, "/tests/corpus/checked-addition.js": { "order": [ "/tests/corpus/checked-addition.js" ], "diags": [] }, + "/tests/corpus/checked-builder-cycles.js": { + "order": [ + "/tests/corpus/checked-builder-cycles.js" + ], + "diags": [] + }, + "/tests/corpus/checked-byte-storage/main.ts": { + "order": [ + "/tests/corpus/checked-byte-storage/values.js", + "/tests/corpus/checked-byte-storage/main.ts" + ], + "diags": [] + }, + "/tests/corpus/checked-object-module-alias.js": { + "order": [ + "/tests/corpus/checked-object-module-alias.js" + ], + "diags": [] + }, + "/tests/corpus/checked-renderer-defaults.js": { + "order": [ + "/tests/corpus/checked-renderer-defaults.js" + ], + "diags": [] + }, + "/tests/corpus/class-accessor-returned-function.js": { + "order": [ + "/tests/corpus/class-accessor-returned-function.js" + ], + "diags": [] + }, + "/tests/corpus/class-array-optional-return.ts": { + "order": [ + "/tests/corpus/class-array-optional-return.ts" + ], + "diags": [] + }, + "/tests/corpus/class-bracket-method-calls.js": { + "order": [ + "/tests/corpus/class-bracket-method-calls.js" + ], + "diags": [] + }, + "/tests/corpus/class-brand-overload.js": { + "order": [ + "/tests/corpus/class-brand-overload.js" + ], + "diags": [] + }, + "/tests/corpus/class-callback-replacement.js": { + "order": [ + "/tests/corpus/class-callback-replacement.js" + ], + "diags": [] + }, + "/tests/corpus/class-checked-upcast.ts": { + "order": [ + "/tests/corpus/class-checked-upcast.ts" + ], + "diags": [] + }, + "/tests/corpus/class-data-descriptors.js": { + "order": [ + "/tests/corpus/class-data-descriptors.js" + ], + "diags": [] + }, + "/tests/corpus/class-instance-constructor.js": { + "order": [ + "/tests/corpus/class-instance-constructor.js" + ], + "diags": [] + }, "/tests/corpus/class-interface-refinements.ts": { "order": [ "/tests/corpus/class-interface-refinements.ts" ], "diags": [] }, + "/tests/corpus/class-prototype-data.js": { + "order": [ + "/tests/corpus/class-prototype-data.js" + ], + "diags": [] + }, + "/tests/corpus/class-static-data.js": { + "order": [ + "/tests/corpus/class-static-data.js" + ], + "diags": [] + }, + "/tests/corpus/closure-nullable-union-return.ts": { + "order": [ + "/tests/corpus/closure-nullable-union-return.ts" + ], + "diags": [] + }, + "/tests/corpus/collection-array-cycles.ts": { + "order": [ + "/tests/corpus/collection-array-cycles.ts" + ], + "diags": [] + }, + "/tests/corpus/collection-array-maps.ts": { + "order": [ + "/tests/corpus/collection-array-maps.ts" + ], + "diags": [] + }, + "/tests/corpus/collection-array-missing.ts": { + "order": [ + "/tests/corpus/collection-array-missing.ts" + ], + "diags": [] + }, + "/tests/corpus/collection-array-sets.ts": { + "order": [ + "/tests/corpus/collection-array-sets.ts" + ], + "diags": [] + }, "/tests/corpus/collection-copy-identities.ts": { "order": [ "/tests/corpus/collection-copy-identities.ts" @@ -8650,12 +8897,36 @@ ], "diags": [] }, + "/tests/corpus/compound-presence-guards.ts": { + "order": [ + "/tests/corpus/compound-presence-guards.ts" + ], + "diags": [] + }, "/tests/corpus/console-imports.cjs": { "order": [ "/tests/corpus/console-imports.cjs" ], "diags": [] }, + "/tests/corpus/contextual-array-union-write.ts": { + "order": [ + "/tests/corpus/contextual-array-union-write.ts" + ], + "diags": [] + }, + "/tests/corpus/contextual-return-layouts.ts": { + "order": [ + "/tests/corpus/contextual-return-layouts.ts" + ], + "diags": [] + }, + "/tests/corpus/crypto-hash-optional-input.ts": { + "order": [ + "/tests/corpus/crypto-hash-optional-input.ts" + ], + "diags": [] + }, "/tests/corpus/declared-index-spread.ts": { "order": [ "/tests/corpus/declared-index-spread.ts" @@ -8668,12 +8939,42 @@ ], "diags": [] }, + "/tests/corpus/destructure-call-assignment.js": { + "order": [ + "/tests/corpus/destructure-call-assignment.js" + ], + "diags": [] + }, + "/tests/corpus/destructure-inferred-row-defaults.js": { + "order": [ + "/tests/corpus/destructure-inferred-row-defaults.js" + ], + "diags": [] + }, + "/tests/corpus/destructure-union-arrays.ts": { + "order": [ + "/tests/corpus/destructure-union-arrays.ts" + ], + "diags": [] + }, + "/tests/corpus/dynamic-function-descriptors.js": { + "order": [ + "/tests/corpus/dynamic-function-descriptors.js" + ], + "diags": [] + }, "/tests/corpus/error-cause-writes.js": { "order": [ "/tests/corpus/error-cause-writes.js" ], "diags": [] }, + "/tests/corpus/expression-runtime-values.ts": { + "order": [ + "/tests/corpus/expression-runtime-values.ts" + ], + "diags": [] + }, "/tests/corpus/field-assignment-values.ts": { "order": [ "/tests/corpus/field-assignment-values.ts" @@ -8686,12 +8987,48 @@ ], "diags": [] }, + "/tests/corpus/file-url-percent-decoding.ts": { + "order": [ + "/tests/corpus/file-url-percent-decoding.ts" + ], + "diags": [] + }, + "/tests/corpus/for-multiple-bindings.ts": { + "order": [ + "/tests/corpus/for-multiple-bindings.ts" + ], + "diags": [] + }, + "/tests/corpus/fresh-array-union-layout.js": { + "order": [ + "/tests/corpus/fresh-array-union-layout.js" + ], + "diags": [] + }, + "/tests/corpus/frozen-checked-dictionaries.mjs": { + "order": [ + "/tests/corpus/frozen-checked-dictionaries.mjs" + ], + "diags": [] + }, "/tests/corpus/fs-native-realpath.ts": { "order": [ "/tests/corpus/fs-native-realpath.ts" ], "diags": [] }, + "/tests/corpus/fs-remove-readonly.ts": { + "order": [ + "/tests/corpus/fs-remove-readonly.ts" + ], + "diags": [] + }, + "/tests/corpus/fs-write-string-bytes-union.ts": { + "order": [ + "/tests/corpus/fs-write-string-bytes-union.ts" + ], + "diags": [] + }, "/tests/corpus/generic-binding-ownership.ts": { "order": [ "/tests/corpus/generic-binding-ownership.ts" @@ -8711,12 +9048,30 @@ ], "diags": [] }, + "/tests/corpus/generic-record-index.ts": { + "order": [ + "/tests/corpus/generic-record-index.ts" + ], + "diags": [] + }, "/tests/corpus/generic-visitor-void.ts": { "order": [ "/tests/corpus/generic-visitor-void.ts" ], "diags": [] }, + "/tests/corpus/global-runtime-probes.js": { + "order": [ + "/tests/corpus/global-runtime-probes.js" + ], + "diags": [] + }, + "/tests/corpus/global-stored-probes.js": { + "order": [ + "/tests/corpus/global-stored-probes.js" + ], + "diags": [] + }, "/tests/corpus/global-symbol-cache/main.js": { "order": [ "/tests/corpus/global-symbol-cache/remote.js", @@ -8730,18 +9085,67 @@ ], "diags": [] }, + "/tests/corpus/indexed-assignment-values.js": { + "order": [ + "/tests/corpus/indexed-assignment-values.js" + ], + "diags": [] + }, "/tests/corpus/instanceof-class-record-union.ts": { "order": [ "/tests/corpus/instanceof-class-record-union.ts" ], "diags": [] }, + "/tests/corpus/intl-grapheme-conformance/main.ts": { + "order": [ + "/tests/corpus/intl-grapheme-conformance/cases.ts", + "/tests/corpus/intl-grapheme-conformance/main.ts" + ], + "diags": [] + }, + "/tests/corpus/intl-grapheme.ts": { + "order": [ + "/tests/corpus/intl-grapheme.ts" + ], + "diags": [] + }, "/tests/corpus/js-arrow-bind.cjs": { "order": [ "/tests/corpus/js-arrow-bind.cjs" ], "diags": [] }, + "/tests/corpus/js-checked-array-reference.js": { + "order": [ + "/tests/corpus/js-checked-array-reference.js" + ], + "diags": [] + }, + "/tests/corpus/js-class-added-parameters.js": { + "order": [ + "/tests/corpus/js-class-added-parameters.js" + ], + "diags": [] + }, + "/tests/corpus/js-class-computed-fields.js": { + "order": [ + "/tests/corpus/js-class-computed-fields.js" + ], + "diags": [] + }, + "/tests/corpus/js-class-fluent-overrides.js": { + "order": [ + "/tests/corpus/js-class-fluent-overrides.js" + ], + "diags": [] + }, + "/tests/corpus/js-class-subclass-properties.js": { + "order": [ + "/tests/corpus/js-class-subclass-properties.js" + ], + "diags": [] + }, "/tests/corpus/js-declared-fields.cjs": { "order": [ "/tests/corpus/js-declared-fields.cjs" @@ -8756,24 +9160,78 @@ ], "diags": [] }, + "/tests/corpus/js-derived-constructor-hoisting.js": { + "order": [ + "/tests/corpus/js-derived-constructor-hoisting.js" + ], + "diags": [] + }, + "/tests/corpus/js-dormant-overrides.js": { + "order": [ + "/tests/corpus/js-dormant-overrides.js" + ], + "diags": [] + }, "/tests/corpus/js-expando-records.js": { "order": [ "/tests/corpus/js-expando-records.js" ], "diags": [] }, + "/tests/corpus/js-inherited-parameter-abi.js": { + "order": [ + "/tests/corpus/js-inherited-parameter-abi.js" + ], + "diags": [] + }, "/tests/corpus/js-module-function-values.js": { "order": [ "/tests/corpus/js-module-function-values.js" ], "diags": [] }, + "/tests/corpus/js-nested-native-descriptors.js": { + "order": [ + "/tests/corpus/js-nested-native-descriptors.js" + ], + "diags": [] + }, + "/tests/corpus/js-object-shared-arrays.js": { + "order": [ + "/tests/corpus/js-object-shared-arrays.js" + ], + "diags": [] + }, + "/tests/corpus/js-omitted-typed-arguments.js": { + "order": [ + "/tests/corpus/js-omitted-typed-arguments.js" + ], + "diags": [] + }, + "/tests/corpus/js-optional-class-fields.js": { + "order": [ + "/tests/corpus/js-optional-class-fields.js" + ], + "diags": [] + }, + "/tests/corpus/js-primitive-union-addition.js": { + "order": [ + "/tests/corpus/js-primitive-union-addition.js" + ], + "diags": [] + }, "/tests/corpus/js-scalar-iterables.js": { "order": [ "/tests/corpus/js-scalar-iterables.js" ], "diags": [] }, + "/tests/corpus/js-specialized-field-method.js": { + "order": [ + "/tests/corpus/js-specialized-field-method.js" + ], + "diags": [] + }, "/tests/corpus/js-static-registries.cjs": { "order": [ "/tests/corpus/js-static-registries.cjs" @@ -8786,6 +9244,48 @@ ], "diags": [] }, + "/tests/corpus/llvm-read-receiver-lifetime.ts": { + "order": [ + "/tests/corpus/llvm-read-receiver-lifetime.ts" + ], + "diags": [] + }, + "/tests/corpus/local-class-call-spreads.js": { + "order": [ + "/tests/corpus/local-class-call-spreads.js" + ], + "diags": [] + }, + "/tests/corpus/local-class-evaluation.ts": { + "order": [ + "/tests/corpus/local-class-evaluation.ts" + ], + "diags": [] + }, + "/tests/corpus/local-class-lifetimes.ts": { + "order": [ + "/tests/corpus/local-class-lifetimes.ts" + ], + "diags": [] + }, + "/tests/corpus/local-class-specialization.js": { + "order": [ + "/tests/corpus/local-class-specialization.js" + ], + "diags": [] + }, + "/tests/corpus/local-class-unknown.ts": { + "order": [ + "/tests/corpus/local-class-unknown.ts" + ], + "diags": [] + }, + "/tests/corpus/local-class-values.ts": { + "order": [ + "/tests/corpus/local-class-values.ts" + ], + "diags": [] + }, "/tests/corpus/logical-assignment-dynamic.js": { "order": [ "/tests/corpus/logical-assignment-dynamic.js" @@ -8798,12 +9298,30 @@ ], "diags": [] }, + "/tests/corpus/logical-runtime-optional-record.ts": { + "order": [ + "/tests/corpus/logical-runtime-optional-record.ts" + ], + "diags": [] + }, + "/tests/corpus/map-checked-seed.js": { + "order": [ + "/tests/corpus/map-checked-seed.js" + ], + "diags": [] + }, "/tests/corpus/map-seeds-and-spreads.ts": { "order": [ "/tests/corpus/map-seeds-and-spreads.ts" ], "diags": [] }, + "/tests/corpus/math-function-values.js": { + "order": [ + "/tests/corpus/math-function-values.js" + ], + "diags": [] + }, "/tests/corpus/module-fallback-functions/main.js": { "order": [ "/tests/corpus/module-fallback-functions/platform.js", @@ -8817,18 +9335,126 @@ ], "diags": [] }, + "/tests/corpus/native-array-borrowed-find-last.js": { + "order": [ + "/tests/corpus/native-array-borrowed-find-last.js" + ], + "diags": [] + }, + "/tests/corpus/native-array-borrowed-slice.js": { + "order": [ + "/tests/corpus/native-array-borrowed-slice.js" + ], + "diags": [] + }, + "/tests/corpus/native-array-copying-methods.js": { + "order": [ + "/tests/corpus/native-array-copying-methods.js" + ], + "diags": [] + }, + "/tests/corpus/native-array-fill-copywithin.js": { + "order": [ + "/tests/corpus/native-array-fill-copywithin.js" + ], + "diags": [] + }, + "/tests/corpus/native-array-frozen-noops.js": { + "order": [ + "/tests/corpus/native-array-frozen-noops.js" + ], + "diags": [] + }, + "/tests/corpus/native-array-index-boundaries.js": { + "order": [ + "/tests/corpus/native-array-index-boundaries.js" + ], + "diags": [] + }, + "/tests/corpus/native-array-integrity.js": { + "order": [ + "/tests/corpus/native-array-integrity.js" + ], + "diags": [] + }, + "/tests/corpus/native-array-length.js": { + "order": [ + "/tests/corpus/native-array-length.js" + ], + "diags": [] + }, + "/tests/corpus/native-array-named-properties.js": { + "order": [ + "/tests/corpus/native-array-named-properties.js" + ], + "diags": [] + }, + "/tests/corpus/native-array-reduce-find-last.js": { + "order": [ + "/tests/corpus/native-array-reduce-find-last.js" + ], + "diags": [] + }, + "/tests/corpus/native-array-search-from-index.js": { + "order": [ + "/tests/corpus/native-array-search-from-index.js" + ], + "diags": [] + }, + "/tests/corpus/native-array-splice-coercion.js": { + "order": [ + "/tests/corpus/native-array-splice-coercion.js" + ], + "diags": [] + }, + "/tests/corpus/native-array-splice-flat.js": { + "order": [ + "/tests/corpus/native-array-splice-flat.js" + ], + "diags": [] + }, "/tests/corpus/native-buffer-coercion.js": { "order": [ "/tests/corpus/native-buffer-coercion.js" ], "diags": [] }, + "/tests/corpus/native-class-array-boundaries.ts": { + "order": [ + "/tests/corpus/native-class-array-boundaries.ts" + ], + "diags": [] + }, "/tests/corpus/native-class-cache.js": { "order": [ "/tests/corpus/native-class-cache.js" ], "diags": [] }, + "/tests/corpus/native-class-properties.cjs": { + "order": [ + "/tests/corpus/native-class-properties.cjs" + ], + "diags": [] + }, + "/tests/corpus/native-class-property-cycles.cjs": { + "order": [ + "/tests/corpus/native-class-property-cycles.cjs" + ], + "diags": [] + }, + "/tests/corpus/native-class-property-order.cjs": { + "order": [ + "/tests/corpus/native-class-property-order.cjs" + ], + "diags": [] + }, + "/tests/corpus/native-class-type-predicates.ts": { + "order": [ + "/tests/corpus/native-class-type-predicates.ts" + ], + "diags": [] + }, "/tests/corpus/native-factory-records.js": { "order": [ "/tests/corpus/native-factory-records.js" @@ -8847,6 +9473,30 @@ ], "diags": [] }, + "/tests/corpus/native-object-descriptors.js": { + "order": [ + "/tests/corpus/native-object-descriptors.js" + ], + "diags": [] + }, + "/tests/corpus/native-object-for-in-minimal.js": { + "order": [ + "/tests/corpus/native-object-for-in-minimal.js" + ], + "diags": [] + }, + "/tests/corpus/native-object-property-attributes.js": { + "order": [ + "/tests/corpus/native-object-property-attributes.js" + ], + "diags": [] + }, + "/tests/corpus/native-object-prototype.js": { + "order": [ + "/tests/corpus/native-object-prototype.js" + ], + "diags": [] + }, "/tests/corpus/native-property-names.ts": { "order": [ "/tests/corpus/native-property-names.ts" @@ -8877,12 +9527,48 @@ ], "diags": [] }, + "/tests/corpus/nullish-long-chain.ts": { + "order": [ + "/tests/corpus/nullish-long-chain.ts" + ], + "diags": [] + }, + "/tests/corpus/number-predicates-unknown.ts": { + "order": [ + "/tests/corpus/number-predicates-unknown.ts" + ], + "diags": [] + }, + "/tests/corpus/object-enumeration-array-reads.ts": { + "order": [ + "/tests/corpus/object-enumeration-array-reads.ts" + ], + "diags": [] + }, + "/tests/corpus/object-from-checked-entries.js": { + "order": [ + "/tests/corpus/object-from-checked-entries.js" + ], + "diags": [] + }, + "/tests/corpus/object-spread-computed.ts": { + "order": [ + "/tests/corpus/object-spread-computed.ts" + ], + "diags": [] + }, "/tests/corpus/opentui-byte-queue.js": { "order": [ "/tests/corpus/opentui-byte-queue.js" ], "diags": [] }, + "/tests/corpus/optional-native-callables.js": { + "order": [ + "/tests/corpus/optional-native-callables.js" + ], + "diags": [] + }, "/tests/corpus/optional-node-list-iteration.ts": { "order": [ "/tests/corpus/optional-node-list-iteration.ts" @@ -8895,18 +9581,108 @@ ], "diags": [] }, + "/tests/corpus/path-to-file-url-options.ts": { + "order": [ + "/tests/corpus/path-to-file-url-options.ts" + ], + "diags": [] + }, + "/tests/corpus/primitive-and-function-descriptors.js": { + "order": [ + "/tests/corpus/primitive-and-function-descriptors.js" + ], + "diags": [] + }, + "/tests/corpus/process-builtin-values.js": { + "order": [ + "/tests/corpus/process-builtin-values.js" + ], + "diags": [] + }, "/tests/corpus/process-env-computed-keys.js": { "order": [ "/tests/corpus/process-env-computed-keys.js" ], "diags": [] }, + "/tests/corpus/process-named-signals.cjs": { + "order": [ + "/tests/corpus/process-named-signals.cjs" + ], + "diags": [] + }, + "/tests/corpus/process-uncaught-handler-throws.cjs": { + "order": [ + "/tests/corpus/process-uncaught-handler-throws.cjs" + ], + "diags": [] + }, + "/tests/corpus/process-uncaught-loop.cjs": { + "order": [ + "/tests/corpus/process-uncaught-loop.cjs" + ], + "diags": [] + }, + "/tests/corpus/process-uncaught-module.mjs": { + "order": [ + "/tests/corpus/process-uncaught-module.mjs" + ], + "diags": [] + }, + "/tests/corpus/process-uncaught-monitor.cjs": { + "order": [ + "/tests/corpus/process-uncaught-monitor.cjs" + ], + "diags": [] + }, + "/tests/corpus/process-uncaught-rejection.cjs": { + "order": [ + "/tests/corpus/process-uncaught-rejection.cjs" + ], + "diags": [] + }, + "/tests/corpus/process-uncaught-sync.cjs": { + "order": [ + "/tests/corpus/process-uncaught-sync.cjs" + ], + "diags": [] + }, + "/tests/corpus/process-versions-storage.ts": { + "order": [ + "/tests/corpus/process-versions-storage.ts" + ], + "diags": [] + }, + "/tests/corpus/prototype-named-data-records.ts": { + "order": [ + "/tests/corpus/prototype-named-data-records.ts" + ], + "diags": [] + }, + "/tests/corpus/record-optional-json-presence.ts": { + "order": [ + "/tests/corpus/record-optional-json-presence.ts" + ], + "diags": [] + }, "/tests/corpus/record-unknown-values.ts": { "order": [ "/tests/corpus/record-unknown-values.ts" ], "diags": [] }, + "/tests/corpus/regex-checked-casts.ts": { + "order": [ + "/tests/corpus/regex-checked-casts.ts" + ], + "diags": [] + }, + "/tests/corpus/regex-checked-storage.js": { + "order": [ + "/tests/corpus/regex-checked-storage.js" + ], + "diags": [] + }, "/tests/corpus/regex-optional-templates.ts": { "order": [ "/tests/corpus/regex-optional-templates.ts" @@ -8919,6 +9695,18 @@ ], "diags": [] }, + "/tests/corpus/regexp-optional-constructor.ts": { + "order": [ + "/tests/corpus/regexp-optional-constructor.ts" + ], + "diags": [] + }, + "/tests/corpus/regexp-string-coercion.js": { + "order": [ + "/tests/corpus/regexp-string-coercion.js" + ], + "diags": [] + }, "/tests/corpus/renderer-export-dictionaries.js": { "order": [ "/tests/corpus/renderer-export-dictionaries.js" @@ -8937,6 +9725,36 @@ ], "diags": [] }, + "/tests/corpus/rest-optional-array-arguments.ts": { + "order": [ + "/tests/corpus/rest-optional-array-arguments.ts" + ], + "diags": [] + }, + "/tests/corpus/return-void-through-finally.ts": { + "order": [ + "/tests/corpus/return-void-through-finally.ts" + ], + "diags": [] + }, + "/tests/corpus/set-checked-methods.js": { + "order": [ + "/tests/corpus/set-checked-methods.js" + ], + "diags": [] + }, + "/tests/corpus/set-checked-storage.ts": { + "order": [ + "/tests/corpus/set-checked-storage.ts" + ], + "diags": [] + }, + "/tests/corpus/set-optional-seeds.ts": { + "order": [ + "/tests/corpus/set-optional-seeds.ts" + ], + "diags": [] + }, "/tests/corpus/static-field-updates/main.ts": { "order": [ "/tests/corpus/static-field-updates/counter.ts", @@ -8944,6 +9762,24 @@ ], "diags": [] }, + "/tests/corpus/stdio-values.mjs": { + "order": [ + "/tests/corpus/stdio-values.mjs" + ], + "diags": [] + }, + "/tests/corpus/stdio-write-errors.mjs": { + "order": [ + "/tests/corpus/stdio-write-errors.mjs" + ], + "diags": [] + }, + "/tests/corpus/stdio-write-overrides.mjs": { + "order": [ + "/tests/corpus/stdio-write-overrides.mjs" + ], + "diags": [] + }, "/tests/corpus/stored-native-builtins.js": { "order": [ "/tests/corpus/stored-native-builtins.js" @@ -9007,18 +9843,66 @@ ], "diags": [] }, + "/tests/corpus/tuple-array-union.ts": { + "order": [ + "/tests/corpus/tuple-array-union.ts" + ], + "diags": [] + }, "/tests/corpus/tuple-callback-visitation.ts": { "order": [ "/tests/corpus/tuple-callback-visitation.ts" ], "diags": [] }, + "/tests/corpus/typed-array-byte-io.js": { + "order": [ + "/tests/corpus/typed-array-byte-io.js" + ], + "diags": [] + }, + "/tests/corpus/typed-array-constructor-identity.js": { + "order": [ + "/tests/corpus/typed-array-constructor-identity.js" + ], + "diags": [] + }, + "/tests/corpus/typed-array-constructor-union.ts": { + "order": [ + "/tests/corpus/typed-array-constructor-union.ts" + ], + "diags": [] + }, + "/tests/corpus/typed-array-dynamic-methods.js": { + "order": [ + "/tests/corpus/typed-array-dynamic-methods.js" + ], + "diags": [] + }, + "/tests/corpus/typed-array-integer-widths.ts": { + "order": [ + "/tests/corpus/typed-array-integer-widths.ts" + ], + "diags": [] + }, + "/tests/corpus/typed-array-unknown.ts": { + "order": [ + "/tests/corpus/typed-array-unknown.ts" + ], + "diags": [] + }, "/tests/corpus/typed-rest-callables.ts": { "order": [ "/tests/corpus/typed-rest-callables.ts" ], "diags": [] }, + "/tests/corpus/typed-rest-filtered-return.ts": { + "order": [ + "/tests/corpus/typed-rest-filtered-return.ts" + ], + "diags": [] + }, "/tests/corpus/typedarray-native-inputs.js": { "order": [ "/tests/corpus/typedarray-native-inputs.js" @@ -9031,6 +9915,42 @@ ], "diags": [] }, + "/tests/corpus/union-discriminant-width.ts": { + "order": [ + "/tests/corpus/union-discriminant-width.ts" + ], + "diags": [] + }, + "/tests/corpus/union-literal-scalar-discriminants.ts": { + "order": [ + "/tests/corpus/union-literal-scalar-discriminants.ts" + ], + "diags": [] + }, + "/tests/corpus/union-literal-slot-conversion.ts": { + "order": [ + "/tests/corpus/union-literal-slot-conversion.ts" + ], + "diags": [] + }, + "/tests/corpus/union-nested-layout-discriminant.ts": { + "order": [ + "/tests/corpus/union-nested-layout-discriminant.ts" + ], + "diags": [] + }, + "/tests/corpus/union-optional-semantic-identity.ts": { + "order": [ + "/tests/corpus/union-optional-semantic-identity.ts" + ], + "diags": [] + }, + "/tests/corpus/union-record-clone.ts": { + "order": [ + "/tests/corpus/union-record-clone.ts" + ], + "diags": [] + }, "/tests/corpus/union-record-structural-copy.ts": { "order": [ "/tests/corpus/union-record-structural-copy.ts" @@ -9061,6 +9981,30 @@ ], "diags": [] }, + "/tests/corpus/util-parseargs-wrapped/main.ts": { + "order": [ + "/tests/corpus/util-parseargs-wrapped/main.ts" + ], + "diags": [] + }, + "/tests/corpus/weak-map-array-buffer.ts": { + "order": [ + "/tests/corpus/weak-map-array-buffer.ts" + ], + "diags": [] + }, + "/tests/corpus/weak-map-native.js": { + "order": [ + "/tests/corpus/weak-map-native.js" + ], + "diags": [] + }, + "/tests/corpus/weak-set-native.js": { + "order": [ + "/tests/corpus/weak-set-native.js" + ], + "diags": [] + }, "/tests/diagnostics/accessors.ts": { "order": [ "/tests/diagnostics/accessors.ts" @@ -9236,6 +10180,12 @@ ], "diags": [] }, + "/tests/diagnostics/class-prototype-reflection.ts": { + "order": [ + "/tests/diagnostics/class-prototype-reflection.ts" + ], + "diags": [] + }, "/tests/diagnostics/class-values.ts": { "order": [ "/tests/diagnostics/class-values.ts" @@ -9801,6 +10751,12 @@ ], "diags": [] }, + "/tests/diagnostics/intl-segmenter.ts": { + "order": [ + "/tests/diagnostics/intl-segmenter.ts" + ], + "diags": [] + }, "/tests/diagnostics/intl.ts": { "order": [ "/tests/diagnostics/intl.ts" @@ -9904,6 +10860,12 @@ ], "diags": [] }, + "/tests/diagnostics/local-class-boundaries.ts": { + "order": [ + "/tests/diagnostics/local-class-boundaries.ts" + ], + "diags": [] + }, "/tests/diagnostics/local-require-binding-esm.js": { "order": [ "/tests/diagnostics/local-require-binding-esm.js" @@ -10414,6 +11376,12 @@ ], "diags": [] }, + "/tests/diagnostics/union-clone-override-width.ts": { + "order": [ + "/tests/diagnostics/union-clone-override-width.ts" + ], + "diags": [] + }, "/tests/diagnostics/unions.ts": { "order": [ "/tests/diagnostics/unions.ts" @@ -10969,920 +11937,6 @@ "/tests/fixtures/node-types/url-getters.ts" ], "diags": [] - }, - "/tests/corpus/array-buffer-conversions.js": { - "order": [ - "/tests/corpus/array-buffer-conversions.js" - ], - "diags": [] - }, - "/tests/corpus/array-buffer-storage.ts": { - "order": [ - "/tests/corpus/array-buffer-storage.ts" - ], - "diags": [] - }, - "/tests/corpus/checked-byte-storage/main.ts": { - "order": [ - "/tests/corpus/checked-byte-storage/values.js", - "/tests/corpus/checked-byte-storage/main.ts" - ], - "diags": [] - }, - "/tests/corpus/expression-runtime-values.ts": { - "order": [ - "/tests/corpus/expression-runtime-values.ts" - ], - "diags": [] - }, - "/tests/corpus/global-runtime-probes.js": { - "order": [ - "/tests/corpus/global-runtime-probes.js" - ], - "diags": [] - }, - "/tests/corpus/global-stored-probes.js": { - "order": [ - "/tests/corpus/global-stored-probes.js" - ], - "diags": [] - }, - "/tests/corpus/indexed-assignment-values.js": { - "order": [ - "/tests/corpus/indexed-assignment-values.js" - ], - "diags": [] - }, - "/tests/corpus/intl-grapheme-conformance/main.ts": { - "order": [ - "/tests/corpus/intl-grapheme-conformance/cases.ts", - "/tests/corpus/intl-grapheme-conformance/main.ts" - ], - "diags": [] - }, - "/tests/corpus/intl-grapheme.ts": { - "order": [ - "/tests/corpus/intl-grapheme.ts" - ], - "diags": [] - }, - "/tests/corpus/object-spread-computed.ts": { - "order": [ - "/tests/corpus/object-spread-computed.ts" - ], - "diags": [] - }, - "/tests/corpus/optional-native-callables.js": { - "order": [ - "/tests/corpus/optional-native-callables.js" - ], - "diags": [] - }, - "/tests/corpus/process-builtin-values.js": { - "order": [ - "/tests/corpus/process-builtin-values.js" - ], - "diags": [] - }, - "/tests/corpus/process-versions-storage.ts": { - "order": [ - "/tests/corpus/process-versions-storage.ts" - ], - "diags": [] - }, - "/tests/corpus/regexp-optional-constructor.ts": { - "order": [ - "/tests/corpus/regexp-optional-constructor.ts" - ], - "diags": [] - }, - "/tests/corpus/collection-array-cycles.ts": { - "order": [ - "/tests/corpus/collection-array-cycles.ts" - ], - "diags": [] - }, - "/tests/corpus/collection-array-maps.ts": { - "order": [ - "/tests/corpus/collection-array-maps.ts" - ], - "diags": [] - }, - "/tests/corpus/collection-array-missing.ts": { - "order": [ - "/tests/corpus/collection-array-missing.ts" - ], - "diags": [] - }, - "/tests/corpus/collection-array-sets.ts": { - "order": [ - "/tests/corpus/collection-array-sets.ts" - ], - "diags": [] - }, - "/tests/corpus/prototype-named-data-records.ts": { - "order": [ - "/tests/corpus/prototype-named-data-records.ts" - ], - "diags": [] - }, - "/tests/corpus/set-optional-seeds.ts": { - "order": [ - "/tests/corpus/set-optional-seeds.ts" - ], - "diags": [] - }, - "/tests/diagnostics/intl-segmenter.ts": { - "order": [ - "/tests/diagnostics/intl-segmenter.ts" - ], - "diags": [] - }, - "/tests/corpus/local-class-call-spreads.js": { - "order": [ - "/tests/corpus/local-class-call-spreads.js" - ], - "diags": [] - }, - "/tests/corpus/local-class-evaluation.ts": { - "order": [ - "/tests/corpus/local-class-evaluation.ts" - ], - "diags": [] - }, - "/tests/corpus/local-class-lifetimes.ts": { - "order": [ - "/tests/corpus/local-class-lifetimes.ts" - ], - "diags": [] - }, - "/tests/corpus/local-class-specialization.js": { - "order": [ - "/tests/corpus/local-class-specialization.js" - ], - "diags": [] - }, - "/tests/corpus/local-class-unknown.ts": { - "order": [ - "/tests/corpus/local-class-unknown.ts" - ], - "diags": [] - }, - "/tests/corpus/local-class-values.ts": { - "order": [ - "/tests/corpus/local-class-values.ts" - ], - "diags": [] - }, - "/tests/diagnostics/local-class-boundaries.ts": { - "order": [ - "/tests/diagnostics/local-class-boundaries.ts" - ], - "diags": [] - }, - "/tests/corpus/3124-array-prototype-borrow.js": { - "order": [ - "/tests/corpus/3124-array-prototype-borrow.js" - ], - "diags": [] - }, - "/tests/corpus/union-discriminant-width.ts": { - "order": [ - "/tests/corpus/union-discriminant-width.ts" - ], - "diags": [] - }, - "/tests/corpus/number-predicates-unknown.ts": { - "order": [ - "/tests/corpus/number-predicates-unknown.ts" - ], - "diags": [] - }, - "/tests/corpus/typed-array-byte-io.js": { - "order": [ - "/tests/corpus/typed-array-byte-io.js" - ], - "diags": [] - }, - "/tests/corpus/typed-array-dynamic-methods.js": { - "order": [ - "/tests/corpus/typed-array-dynamic-methods.js" - ], - "diags": [] - }, - "/tests/corpus/typed-array-integer-widths.ts": { - "order": [ - "/tests/corpus/typed-array-integer-widths.ts" - ], - "diags": [] - }, - "/tests/corpus/typed-array-unknown.ts": { - "order": [ - "/tests/corpus/typed-array-unknown.ts" - ], - "diags": [] - }, - "/tests/corpus/return-void-through-finally.ts": { - "order": [ - "/tests/corpus/return-void-through-finally.ts" - ], - "diags": [] - }, - "/tests/corpus/union-literal-scalar-discriminants.ts": { - "order": [ - "/tests/corpus/union-literal-scalar-discriminants.ts" - ], - "diags": [] - }, - "/tests/corpus/union-literal-slot-conversion.ts": { - "order": [ - "/tests/corpus/union-literal-slot-conversion.ts" - ], - "diags": [] - }, - "/tests/corpus/union-optional-semantic-identity.ts": { - "order": [ - "/tests/corpus/union-optional-semantic-identity.ts" - ], - "diags": [] - }, - "/tests/corpus/async-backend-factory.mjs": { - "order": [ - "/tests/corpus/async-backend-factory.mjs" - ], - "diags": [] - }, - "/tests/corpus/destructure-call-assignment.js": { - "order": [ - "/tests/corpus/destructure-call-assignment.js" - ], - "diags": [] - }, - "/tests/corpus/destructure-inferred-row-defaults.js": { - "order": [ - "/tests/corpus/destructure-inferred-row-defaults.js" - ], - "diags": [] - }, - "/tests/corpus/map-checked-seed.js": { - "order": [ - "/tests/corpus/map-checked-seed.js" - ], - "diags": [] - }, - "/tests/corpus/object-from-checked-entries.js": { - "order": [ - "/tests/corpus/object-from-checked-entries.js" - ], - "diags": [] - }, - "/tests/corpus/weak-map-array-buffer.ts": { - "order": [ - "/tests/corpus/weak-map-array-buffer.ts" - ], - "diags": [] - }, - "/tests/corpus/weak-map-native.js": { - "order": [ - "/tests/corpus/weak-map-native.js" - ], - "diags": [] - }, - "/tests/corpus/weak-set-native.js": { - "order": [ - "/tests/corpus/weak-set-native.js" - ], - "diags": [] - }, - "/tests/corpus/destructure-union-arrays.ts": { - "order": [ - "/tests/corpus/destructure-union-arrays.ts" - ], - "diags": [] - }, - "/tests/corpus/fresh-array-union-layout.js": { - "order": [ - "/tests/corpus/fresh-array-union-layout.js" - ], - "diags": [] - }, - "/tests/corpus/bigint-checked-casts.ts": { - "order": [ - "/tests/corpus/bigint-checked-casts.ts" - ], - "diags": [] - }, - "/tests/corpus/bigint-checked-storage.js": { - "order": [ - "/tests/corpus/bigint-checked-storage.js" - ], - "diags": [] - }, - "/tests/corpus/checked-object-module-alias.js": { - "order": [ - "/tests/corpus/checked-object-module-alias.js" - ], - "diags": [] - }, - "/tests/corpus/dynamic-function-descriptors.js": { - "order": [ - "/tests/corpus/dynamic-function-descriptors.js" - ], - "diags": [] - }, - "/tests/corpus/regexp-string-coercion.js": { - "order": [ - "/tests/corpus/regexp-string-coercion.js" - ], - "diags": [] - }, - "/tests/corpus/frozen-checked-dictionaries.mjs": { - "order": [ - "/tests/corpus/frozen-checked-dictionaries.mjs" - ], - "diags": [] - }, - "/tests/corpus/regex-checked-casts.ts": { - "order": [ - "/tests/corpus/regex-checked-casts.ts" - ], - "diags": [] - }, - "/tests/corpus/regex-checked-storage.js": { - "order": [ - "/tests/corpus/regex-checked-storage.js" - ], - "diags": [] - }, - "/tests/corpus/set-checked-methods.js": { - "order": [ - "/tests/corpus/set-checked-methods.js" - ], - "diags": [] - }, - "/tests/corpus/set-checked-storage.ts": { - "order": [ - "/tests/corpus/set-checked-storage.ts" - ], - "diags": [] - }, - "/tests/corpus/checked-builder-cycles.js": { - "order": [ - "/tests/corpus/checked-builder-cycles.js" - ], - "diags": [] - }, - "/tests/corpus/checked-renderer-defaults.js": { - "order": [ - "/tests/corpus/checked-renderer-defaults.js" - ], - "diags": [] - }, - "/tests/corpus/contextual-return-layouts.ts": { - "order": [ - "/tests/corpus/contextual-return-layouts.ts" - ], - "diags": [] - }, - "/tests/corpus/compound-presence-guards.ts": { - "order": [ - "/tests/corpus/compound-presence-guards.ts" - ], - "diags": [] - }, - "/tests/corpus/union-record-clone.ts": { - "order": [ - "/tests/corpus/union-record-clone.ts" - ], - "diags": [] - }, - "/tests/corpus/generic-record-index.ts": { - "order": [ - "/tests/corpus/generic-record-index.ts" - ], - "diags": [] - }, - "/tests/corpus/path-to-file-url-options.ts": { - "order": [ - "/tests/corpus/path-to-file-url-options.ts" - ], - "diags": [] - }, - "/tests/corpus/file-url-percent-decoding.ts": { - "order": [ - "/tests/corpus/file-url-percent-decoding.ts" - ], - "diags": [] - }, - "/tests/diagnostics/union-clone-override-width.ts": { - "order": [ - "/tests/diagnostics/union-clone-override-width.ts" - ], - "diags": [] - }, - "/tests/corpus/callable-properties.mjs": { - "order": [ - "/tests/corpus/callable-properties.mjs" - ], - "diags": [] - }, - "/tests/corpus/callable-property-cycles.js": { - "order": [ - "/tests/corpus/callable-property-cycles.js" - ], - "diags": [] - }, - "/tests/corpus/stdio-values.mjs": { - "order": [ - "/tests/corpus/stdio-values.mjs" - ], - "diags": [] - }, - "/tests/corpus/stdio-write-errors.mjs": { - "order": [ - "/tests/corpus/stdio-write-errors.mjs" - ], - "diags": [] - }, - "/tests/corpus/stdio-write-overrides.mjs": { - "order": [ - "/tests/corpus/stdio-write-overrides.mjs" - ], - "diags": [] - }, - "/tests/corpus/boxed-error-rethrows.mjs": { - "order": [ - "/tests/corpus/boxed-error-rethrows.mjs" - ], - "diags": [] - }, - "/tests/corpus/callable-module-members.mjs": { - "order": [ - "/tests/corpus/callable-module-members.mjs" - ], - "diags": [] - }, - "/tests/corpus/array-refined-storage.ts": { - "order": [ - "/tests/corpus/array-refined-storage.ts" - ], - "diags": [] - }, - "/tests/corpus/class-array-optional-return.ts": { - "order": [ - "/tests/corpus/class-array-optional-return.ts" - ], - "diags": [] - }, - "/tests/corpus/contextual-array-union-write.ts": { - "order": [ - "/tests/corpus/contextual-array-union-write.ts" - ], - "diags": [] - }, - "/tests/corpus/nullish-long-chain.ts": { - "order": [ - "/tests/corpus/nullish-long-chain.ts" - ], - "diags": [] - }, - "/tests/corpus/tuple-array-union.ts": { - "order": [ - "/tests/corpus/tuple-array-union.ts" - ], - "diags": [] - }, - "/tests/corpus/typed-rest-filtered-return.ts": { - "order": [ - "/tests/corpus/typed-rest-filtered-return.ts" - ], - "diags": [] - }, - "/tests/corpus/class-accessor-returned-function.js": { - "order": [ - "/tests/corpus/class-accessor-returned-function.js" - ], - "diags": [] - }, - "/tests/corpus/class-bracket-method-calls.js": { - "order": [ - "/tests/corpus/class-bracket-method-calls.js" - ], - "diags": [] - }, - "/tests/corpus/native-array-borrowed-find-last.js": { - "order": [ - "/tests/corpus/native-array-borrowed-find-last.js" - ], - "diags": [] - }, - "/tests/corpus/native-array-borrowed-slice.js": { - "order": [ - "/tests/corpus/native-array-borrowed-slice.js" - ], - "diags": [] - }, - "/tests/corpus/native-array-copying-methods.js": { - "order": [ - "/tests/corpus/native-array-copying-methods.js" - ], - "diags": [] - }, - "/tests/corpus/native-array-fill-copywithin.js": { - "order": [ - "/tests/corpus/native-array-fill-copywithin.js" - ], - "diags": [] - }, - "/tests/corpus/native-array-frozen-noops.js": { - "order": [ - "/tests/corpus/native-array-frozen-noops.js" - ], - "diags": [] - }, - "/tests/corpus/native-array-index-boundaries.js": { - "order": [ - "/tests/corpus/native-array-index-boundaries.js" - ], - "diags": [] - }, - "/tests/corpus/native-array-integrity.js": { - "order": [ - "/tests/corpus/native-array-integrity.js" - ], - "diags": [] - }, - "/tests/corpus/native-array-length.js": { - "order": [ - "/tests/corpus/native-array-length.js" - ], - "diags": [] - }, - "/tests/corpus/native-array-named-properties.js": { - "order": [ - "/tests/corpus/native-array-named-properties.js" - ], - "diags": [] - }, - "/tests/corpus/native-array-reduce-find-last.js": { - "order": [ - "/tests/corpus/native-array-reduce-find-last.js" - ], - "diags": [] - }, - "/tests/corpus/native-array-search-from-index.js": { - "order": [ - "/tests/corpus/native-array-search-from-index.js" - ], - "diags": [] - }, - "/tests/corpus/native-array-splice-coercion.js": { - "order": [ - "/tests/corpus/native-array-splice-coercion.js" - ], - "diags": [] - }, - "/tests/corpus/native-array-splice-flat.js": { - "order": [ - "/tests/corpus/native-array-splice-flat.js" - ], - "diags": [] - }, - "/tests/corpus/native-object-descriptors.js": { - "order": [ - "/tests/corpus/native-object-descriptors.js" - ], - "diags": [] - }, - "/tests/corpus/native-object-for-in-minimal.js": { - "order": [ - "/tests/corpus/native-object-for-in-minimal.js" - ], - "diags": [] - }, - "/tests/corpus/native-object-property-attributes.js": { - "order": [ - "/tests/corpus/native-object-property-attributes.js" - ], - "diags": [] - }, - "/tests/corpus/native-object-prototype.js": { - "order": [ - "/tests/corpus/native-object-prototype.js" - ], - "diags": [] - }, - "/tests/corpus/primitive-and-function-descriptors.js": { - "order": [ - "/tests/corpus/primitive-and-function-descriptors.js" - ], - "diags": [] - }, - "/tests/corpus/3068-url-checked-identity.ts": { - "order": [ - "/tests/corpus/3068-url-checked-identity.ts" - ], - "diags": [] - }, - "/tests/corpus/3069-set-checked-seed.js": { - "order": [ - "/tests/corpus/3069-set-checked-seed.js" - ], - "diags": [] - }, - "/tests/corpus/3070-checked-object-accessors.js": { - "order": [ - "/tests/corpus/3070-checked-object-accessors.js" - ], - "diags": [] - }, - "/tests/corpus/3071-checked-specialization-narrowing.js": { - "order": [ - "/tests/corpus/3071-checked-specialization-narrowing.js" - ], - "diags": [] - }, - "/tests/corpus/3072-weak-process-streams.js": { - "order": [ - "/tests/corpus/3072-weak-process-streams.js" - ], - "diags": [] - }, - "/tests/corpus/3073-optional-callable-module.js": { - "order": [ - "/tests/corpus/3073-optional-callable-module.js" - ], - "diags": [] - }, - "/tests/corpus/native-class-properties.cjs": { - "order": [ - "/tests/corpus/native-class-properties.cjs" - ], - "diags": [] - }, - "/tests/corpus/native-class-property-cycles.cjs": { - "order": [ - "/tests/corpus/native-class-property-cycles.cjs" - ], - "diags": [] - }, - "/tests/corpus/native-class-property-order.cjs": { - "order": [ - "/tests/corpus/native-class-property-order.cjs" - ], - "diags": [] - }, - "/tests/corpus/process-named-signals.cjs": { - "order": [ - "/tests/corpus/process-named-signals.cjs" - ], - "diags": [] - }, - "/tests/corpus/process-uncaught-handler-throws.cjs": { - "order": [ - "/tests/corpus/process-uncaught-handler-throws.cjs" - ], - "diags": [] - }, - "/tests/corpus/process-uncaught-loop.cjs": { - "order": [ - "/tests/corpus/process-uncaught-loop.cjs" - ], - "diags": [] - }, - "/tests/corpus/process-uncaught-module.mjs": { - "order": [ - "/tests/corpus/process-uncaught-module.mjs" - ], - "diags": [] - }, - "/tests/corpus/process-uncaught-monitor.cjs": { - "order": [ - "/tests/corpus/process-uncaught-monitor.cjs" - ], - "diags": [] - }, - "/tests/corpus/process-uncaught-rejection.cjs": { - "order": [ - "/tests/corpus/process-uncaught-rejection.cjs" - ], - "diags": [] - }, - "/tests/corpus/process-uncaught-sync.cjs": { - "order": [ - "/tests/corpus/process-uncaught-sync.cjs" - ], - "diags": [] - }, - "/tests/corpus/4030-runtime-object-truthiness.ts": { - "order": [ - "/tests/corpus/4030-runtime-object-truthiness.ts" - ], - "diags": [] - }, - "/tests/corpus/4031-event-emitter-long-tuples.ts": { - "order": [ - "/tests/corpus/4031-event-emitter-long-tuples.ts" - ], - "diags": [] - }, - "/tests/corpus/logical-runtime-optional-record.ts": { - "order": [ - "/tests/corpus/logical-runtime-optional-record.ts" - ], - "diags": [] - }, - "/tests/corpus/union-nested-layout-discriminant.ts": { - "order": [ - "/tests/corpus/union-nested-layout-discriminant.ts" - ], - "diags": [] - }, - "/tests/corpus/llvm-read-receiver-lifetime.ts": { - "order": [ - "/tests/corpus/llvm-read-receiver-lifetime.ts" - ], - "diags": [] - }, - "/tests/corpus/class-brand-overload.js": { - "order": [ - "/tests/corpus/class-brand-overload.js" - ], - "diags": [] - }, - "/tests/corpus/class-prototype-data.js": { - "order": [ - "/tests/corpus/class-prototype-data.js" - ], - "diags": [] - }, - "/tests/corpus/js-omitted-typed-arguments.js": { - "order": [ - "/tests/corpus/js-omitted-typed-arguments.js" - ], - "diags": [] - }, - "/tests/corpus/math-function-values.js": { - "order": [ - "/tests/corpus/math-function-values.js" - ], - "diags": [] - }, - "/tests/diagnostics/class-prototype-reflection.ts": { - "order": [ - "/tests/diagnostics/class-prototype-reflection.ts" - ], - "diags": [] - }, - "/tests/corpus/rest-optional-array-arguments.ts": { - "order": [ - "/tests/corpus/rest-optional-array-arguments.ts" - ], - "diags": [] - }, - "/tests/corpus/class-data-descriptors.js": { - "order": [ - "/tests/corpus/class-data-descriptors.js" - ], - "diags": [] - }, - "/tests/corpus/class-static-data.js": { - "order": [ - "/tests/corpus/class-static-data.js" - ], - "diags": [] - }, - "/tests/corpus/for-multiple-bindings.ts": { - "order": [ - "/tests/corpus/for-multiple-bindings.ts" - ], - "diags": [] - }, - "/tests/corpus/js-dormant-overrides.js": { - "order": [ - "/tests/corpus/js-dormant-overrides.js" - ], - "diags": [] - }, - "/tests/corpus/js-primitive-union-addition.js": { - "order": [ - "/tests/corpus/js-primitive-union-addition.js" - ], - "diags": [] - }, - "/tests/corpus/js-specialized-field-method.js": { - "order": [ - "/tests/corpus/js-specialized-field-method.js" - ], - "diags": [] - }, - "/tests/corpus/typed-array-constructor-identity.js": { - "order": [ - "/tests/corpus/typed-array-constructor-identity.js" - ], - "diags": [] - }, - "/tests/corpus/typed-array-constructor-union.ts": { - "order": [ - "/tests/corpus/typed-array-constructor-union.ts" - ], - "diags": [] - }, - "/tests/corpus/class-callback-replacement.js": { - "order": [ - "/tests/corpus/class-callback-replacement.js" - ], - "diags": [] - }, - "/tests/corpus/class-instance-constructor.js": { - "order": [ - "/tests/corpus/class-instance-constructor.js" - ], - "diags": [] - }, - "/tests/corpus/js-class-computed-fields.js": { - "order": [ - "/tests/corpus/js-class-computed-fields.js" - ], - "diags": [] - }, - "/tests/corpus/js-derived-constructor-hoisting.js": { - "order": [ - "/tests/corpus/js-derived-constructor-hoisting.js" - ], - "diags": [] - }, - "/tests/corpus/js-inherited-parameter-abi.js": { - "order": [ - "/tests/corpus/js-inherited-parameter-abi.js" - ], - "diags": [] - }, - "/tests/corpus/js-nested-native-descriptors.js": { - "order": [ - "/tests/corpus/js-nested-native-descriptors.js" - ], - "diags": [] - }, - "/tests/corpus/js-class-fluent-overrides.js": { - "order": [ - "/tests/corpus/js-class-fluent-overrides.js" - ], - "diags": [] - }, - "/tests/corpus/js-class-subclass-properties.js": { - "order": [ - "/tests/corpus/js-class-subclass-properties.js" - ], - "diags": [] - }, - "/tests/corpus/js-object-shared-arrays.js": { - "order": [ - "/tests/corpus/js-object-shared-arrays.js" - ], - "diags": [] - }, - "/tests/corpus/js-optional-class-fields.js": { - "order": [ - "/tests/corpus/js-optional-class-fields.js" - ], - "diags": [] - }, - "/tests/corpus/native-class-array-boundaries.ts": { - "order": [ - "/tests/corpus/native-class-array-boundaries.ts" - ], - "diags": [] - }, - "/tests/corpus/native-class-type-predicates.ts": { - "order": [ - "/tests/corpus/native-class-type-predicates.ts" - ], - "diags": [] - }, - "/tests/corpus/array-sort-unknown-comparator.ts": { - "order": [ - "/tests/corpus/array-sort-unknown-comparator.ts" - ], - "diags": [] - }, - "/tests/corpus/class-checked-upcast.ts": { - "order": [ - "/tests/corpus/class-checked-upcast.ts" - ], - "diags": [] - }, - "/tests/corpus/js-checked-array-reference.js": { - "order": [ - "/tests/corpus/js-checked-array-reference.js" - ], - "diags": [] - }, - "/tests/corpus/js-class-added-parameters.js": { - "order": [ - "/tests/corpus/js-class-added-parameters.js" - ], - "diags": [] } } } diff --git a/packages/runtime/src/scr_lib.c b/packages/runtime/src/scr_lib.c index b8acaeca..bda45a95 100644 --- a/packages/runtime/src/scr_lib.c +++ b/packages/runtime/src/scr_lib.c @@ -2971,6 +2971,8 @@ void scr_fs_rename(ScrStr *oldpath, ScrStr *newpath) { if (error != 0) scr_fs_rename_error(error, oldpath, newpath); } +static int scr_rm_unlink(const char *path, size_t len); + void scr_fs_rm(ScrStr *path) { /* Node's rmSync: lstat first (a missing path reports the lstat syscall), * refuse directories (Node requires `recursive`, which the scriptc @@ -2985,7 +2987,7 @@ void scr_fs_rm(ScrStr *path) { scr_fs_throw(EISDIR, "rm", path); return; } - if (unlink(path->data) != 0) scr_fs_throw(errno, "unlink", path); + if (scr_rm_unlink(path->data, path->len) != 0) scr_fs_throw(errno, "unlink", path); } void scr_fs_rmdir(ScrStr *path) { @@ -3094,6 +3096,35 @@ static void scr_rm_fail_set(ScrRmFail *f, int err, const char *op, const char *p f->path = scr_str_new(path, len); } +static int scr_rm_unlink(const char *path, size_t len) { + if (unlink(path) == 0) return 0; +#ifdef _WIN32 + /* Node's Windows removal clears a file's read-only attribute before + * retrying. Private staged runtime objects intentionally use mode 0400. */ + const int original = errno; + if (original != EACCES && original != EPERM) return -1; + ScrStr *text = scr_str_new(path, len); + WCHAR *wide = scr_fs_win_wide(text); + scr_str_release(text); + if (!wide) { errno = original; return -1; } + const DWORD attributes = GetFileAttributesW(wide); + if (attributes != INVALID_FILE_ATTRIBUTES && (attributes & FILE_ATTRIBUTE_READONLY) && + SetFileAttributesW(wide, attributes & ~FILE_ATTRIBUTE_READONLY)) { + if (DeleteFileW(wide)) { free(wide); return 0; } + const DWORD error = GetLastError(); + SetFileAttributesW(wide, attributes); + free(wide); + errno = scr_fs_win_errno(error); + return -1; + } + free(wide); + errno = original; +#else + (void)len; +#endif + return -1; +} + /* Post-order tree removal for rmSync's recursive form. Stops at (and * records) the first failure, with the failing path and syscall name. */ static void scr_rm_tree_e(const char *path, size_t len, ScrRmFail *f) { @@ -3103,7 +3134,7 @@ static void scr_rm_tree_e(const char *path, size_t len, ScrRmFail *f) { return; } if (!S_ISDIR(st.st_mode)) { - if (unlink(path) != 0) scr_rm_fail_set(f, errno, "unlink", path, len); + if (scr_rm_unlink(path, len) != 0) scr_rm_fail_set(f, errno, "unlink", path, len); return; } DIR *d = opendir(path); @@ -3153,7 +3184,7 @@ static void scr_fs_rm_attempt(ScrStr *path, bool recursive, bool force, ScrRmFai scr_rm_tree_e(path->data, path->len, f); return; } - if (unlink(path->data) != 0) scr_rm_fail_set(f, errno, "unlink", path->data, path->len); + if (scr_rm_unlink(path->data, path->len) != 0) scr_rm_fail_set(f, errno, "unlink", path->data, path->len); } void scr_fs_rm_opts(ScrStr *path, bool recursive, bool force) { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5541d351..690afdc5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -45,10 +45,79 @@ importers: internal/compatibility: {} packages/cli: - dependencies: + devDependencies: '@scriptc/compiler': specifier: workspace:* version: link:../compiler + optionalDependencies: + '@scriptc/cli-darwin-arm64': + specifier: workspace:* + version: link:../cli-darwin-arm64 + '@scriptc/cli-darwin-x64': + specifier: workspace:* + version: link:../cli-darwin-x64 + '@scriptc/cli-linux-arm64-gnu': + specifier: workspace:* + version: link:../cli-linux-arm64-gnu + '@scriptc/cli-linux-arm64-musl': + specifier: workspace:* + version: link:../cli-linux-arm64-musl + '@scriptc/cli-linux-x64-gnu': + specifier: workspace:* + version: link:../cli-linux-x64-gnu + '@scriptc/cli-linux-x64-musl': + specifier: workspace:* + version: link:../cli-linux-x64-musl + '@scriptc/cli-win32-x64-msvc': + specifier: workspace:* + version: link:../cli-win32-x64-msvc + '@scriptc/runtime-android-arm64': + specifier: workspace:* + version: link:../runtime-android-arm64 + '@scriptc/runtime-darwin-arm64': + specifier: workspace:* + version: link:../runtime-darwin-arm64 + '@scriptc/runtime-darwin-x64': + specifier: workspace:* + version: link:../runtime-darwin-x64 + '@scriptc/runtime-ios-arm64': + specifier: workspace:* + version: link:../runtime-ios-arm64 + '@scriptc/runtime-ios-simulator-arm64': + specifier: workspace:* + version: link:../runtime-ios-simulator-arm64 + '@scriptc/runtime-linux-arm64-gnu': + specifier: workspace:* + version: link:../runtime-linux-arm64-gnu + '@scriptc/runtime-linux-arm64-musl': + specifier: workspace:* + version: link:../runtime-linux-arm64-musl + '@scriptc/runtime-linux-x64-gnu': + specifier: workspace:* + version: link:../runtime-linux-x64-gnu + '@scriptc/runtime-linux-x64-musl': + specifier: workspace:* + version: link:../runtime-linux-x64-musl + '@scriptc/runtime-wasm32-wasi': + specifier: workspace:* + version: link:../runtime-wasm32-wasi + '@scriptc/runtime-win32-x64-msvc': + specifier: workspace:* + version: link:../runtime-win32-x64-msvc + + packages/cli-darwin-arm64: {} + + packages/cli-darwin-x64: {} + + packages/cli-linux-arm64-gnu: {} + + packages/cli-linux-arm64-musl: {} + + packages/cli-linux-x64-gnu: {} + + packages/cli-linux-x64-musl: {} + + packages/cli-win32-x64-msvc: {} packages/compiler: dependencies: diff --git a/scripts/bench-builds.mjs b/scripts/bench-builds.mjs index c2d235f0..888ff23c 100644 --- a/scripts/bench-builds.mjs +++ b/scripts/bench-builds.mjs @@ -9,7 +9,10 @@ import { join } from "node:path"; import { fileURLToPath } from "node:url"; import { parseArgs } from "node:util"; -const { values } = parseArgs({ options: { iterations: { type: "string", default: "5" } } }); +const { values } = parseArgs({ options: { + iterations: { type: "string", default: "5" }, + compiler: { type: "string" }, +} }); const iterations = Number(values.iterations); if (!Number.isInteger(iterations) || iterations < 1 || iterations > 100) { throw new Error("--iterations must be an integer between 1 and 100"); @@ -17,7 +20,7 @@ if (!Number.isInteger(iterations) || iterations < 1 || iterations > 100) { if (process.env.SCRIPTC_TARGET && process.env.SCRIPTC_TARGET !== "native") { throw new Error("bench:builds runs host executables; unset SCRIPTC_TARGET"); } -const cli = fileURLToPath(new URL("../packages/cli/dist/bootstrap.js", import.meta.url)); +const cli = values.compiler ?? fileURLToPath(new URL("../packages/cli/dist/bootstrap.js", import.meta.url)); await access(cli).catch(() => { throw new Error("Build the workspace with pnpm build before running bench:builds"); }); const root = await mkdtemp(join(process.platform === "win32" ? tmpdir() : "/tmp", "scriptc-bench-builds-")); const cache = join(root, "cache"); @@ -46,7 +49,8 @@ function run(command, args) { function build(phase) { const start = performance.now(); - const result = run(process.execPath, [cli, "build", entry, "--optimization=dev", "-o", binary]); + const args = ["build", entry, "--optimization=dev", "-o", binary]; + const result = values.compiler ? run(cli, args) : run(process.execPath, [cli, ...args]); const ms = Math.round((performance.now() - start) * 10) / 10; assert.equal(result.status, 0, result.stderr); // Correctness checks are outside the timed build and run after EVERY edit, @@ -81,6 +85,7 @@ try { build("edit"); } process.stdout.write(JSON.stringify({ + compiler: values.compiler ?? "node", node: process.version, platform: process.platform, arch: process.arch, diff --git a/scripts/build-native-cli.mts b/scripts/build-native-cli.mts new file mode 100644 index 00000000..e46bcd79 --- /dev/null +++ b/scripts/build-native-cli.mts @@ -0,0 +1,67 @@ +/** Build the production CLI seed and its relocatable compiler assets. */ +import { execFileSync } from "node:child_process"; +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { basename, dirname, join, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { compile } from "../packages/compiler/src/index.js"; +import { NATIVE_TARGETS, nativeCodegenTarget } from "../packages/compiler/src/backend/targets.js"; +import { ts7Executable } from "../packages/compiler/src/frontend/ts7/rpc-api.js"; +import { stageNativeCliAssets } from "./native-cli-assets.mjs"; +import type { NativeToolchainManifest } from "../packages/compiler/src/native/toolchain.js"; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const output = resolve(process.argv[2] ?? join(root, ".scriptc/native-cli")); +const target = nativeCodegenTarget(); +if (target === null || target.platform === "wasi") throw new Error("a supported native host is required to build the compiler"); +const bin = join(output, "bin"); +const lib = join(output, "lib"); +const seed = join(dirname(output), ".scriptc", basename(output) + "-seed"); +for (const dir of [bin, lib, seed]) mkdirSync(dir, { recursive: true }); +const compiler = process.env["SCRIPTC_CC"] ?? target.defaultLinker; +const compilerArgs = process.env["SCRIPTC_CC"] === undefined ? [...target.defaultLinkerArgs] : []; +const nativeSources = join(root, "packages/compiler/native"); +const functions: unknown[] = []; +const libraries: string[] = []; +for (const name of ["ts7-process", "host"]) { + const object = join(seed, name + target.outputSuffixes.obj); + execFileSync(compiler, [ + ...compilerArgs, "-std=c11", "-Wall", "-Wextra", "-Werror", "-target", target.linkerTargetTriple, + "-c", join(nativeSources, name + ".c"), "-o", object, + ], { stdio: "inherit" }); + const manifest = JSON.parse(readFileSync(join(nativeSources, name + ".ffi.json"), "utf8")) as { functions: unknown[] }; + functions.push(...manifest.functions); + libraries.push(object); +} +const ffi = join(seed, "compiler.ffi.json"); +writeFileSync(ffi, JSON.stringify({ ffi_format: 6, functions, libraries }, null, 2) + "\n"); +const executable = join(bin, "scriptc" + target.outputSuffixes.exe); +const comptime = join(lib, "scriptc-comptime" + target.outputSuffixes.exe); +execFileSync(compiler, [ + ...compilerArgs, "-std=c11", "-O2", "-Wall", "-Wextra", "-Werror", "-target", target.linkerTargetTriple, + "-I", join(root, "packages/runtime/vendor/quickjs-ng"), join(nativeSources, "comptime.c"), + join(root, "packages", target.runtimePackPackage.replace("@scriptc/", ""), "artifacts/vendor/quickjs/libscriptc-quickjs.a"), + "-lm", "-lpthread", "-o", comptime, +], { stdio: "inherit" }); +const result = await compile(join(root, "packages/compiler/src/native/cli.ts"), { + outDir: seed, outPath: executable, backend: "llvm", + optimization: process.env["SCRIPTC_NATIVE_OPTIMIZATION"] === "dev" ? "dev" : "release", + strip: process.env["SCRIPTC_NATIVE_KEEP_SYMBOLS"] !== "1", dynamic: false, ffiProfilePath: ffi, sanitize: process.env["SCRIPTC_SAN"] === "1", +}); +if (!result.ok) throw new Error(result.diagnostics.map((item) => + `${item.loc ? `${item.loc.file}:${item.loc.start}: ` : ""}${item.code}: ${item.message}`, +).join("\n")); +const compilerVersion = (JSON.parse(readFileSync(join(root, "packages/compiler/package.json"), "utf8")) as { version: string }).version; +const assets = stageNativeCliAssets({ repository: root, output, target, ts7: ts7Executable(), compilerVersion }); +// A cross-libc seed may use Zig on the build host. The installed compiler +// selects the destination platform's ordinary linker and SDK. +const installedTarget = NATIVE_TARGETS.find((item) => item.name === target.name)!; +const manifest: NativeToolchainManifest = { + schema: "scriptc.native-toolchain.v1", compiler_version: compilerVersion, target: target.name, + ...assets, + linker: process.env["SCRIPTC_LINKER"] ?? installedTarget.defaultLinker, + linker_args: process.env["SCRIPTC_LINKER"] === undefined ? [...installedTarget.defaultLinkerArgs] : [], + dsymutil: process.env["SCRIPTC_DSYMUTIL"] ?? "dsymutil", + comptime: relative(bin, comptime), +}; +writeFileSync(executable + ".json", JSON.stringify(manifest, null, 2) + "\n"); +console.log(executable); diff --git a/scripts/build-native-compiler.mts b/scripts/build-native-compiler.mts deleted file mode 100644 index 2337f953..00000000 --- a/scripts/build-native-compiler.mts +++ /dev/null @@ -1,54 +0,0 @@ -/** Seed a native compiler using the Node-hosted compiler. Subsequent builds - * can invoke the resulting executable with the emitted FFI profile. */ -import { execFileSync } from "node:child_process"; -import { mkdirSync, readFileSync, realpathSync, writeFileSync } from "node:fs"; -import { dirname, join, relative, resolve } from "node:path"; -import { fileURLToPath } from "node:url"; -import { compile } from "../packages/compiler/src/index.js"; -import { nativeCodegenTarget } from "../packages/compiler/src/backend/targets.js"; -import { ts7Executable } from "../packages/compiler/src/frontend/ts7/rpc-api.js"; -import type { NativeToolchainManifest } from "../packages/compiler/src/native/toolchain.js"; - -const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); -const output = resolve(process.argv[2] ?? join(root, ".scriptc/native")); -const target = nativeCodegenTarget(); -if (target === null || target.platform === "wasi") throw new Error("a supported native host is required to build the compiler"); -mkdirSync(output, { recursive: true }); -const executable = join(output, "scriptc-native" + target.outputSuffixes.exe); -const object = join(output, "ts7-process" + target.outputSuffixes.obj); -const ffi = join(output, "ts7-process.ffi.json"); -const nativeSources = join(root, "packages/compiler/native"); -const compiler = process.env["SCRIPTC_CC"] ?? target.defaultLinker; -const compilerArgs = process.env["SCRIPTC_CC"] === undefined ? [...target.defaultLinkerArgs] : []; -const sanitize = process.env["SCRIPTC_SAN"] === "1"; -// This object and profile are reused by native rebuilds with packaged runtimes. -// Sanitizer instrumentation belongs to the seed compiler's own build below. -execFileSync(compiler, [ - ...compilerArgs, "-std=c11", "-Wall", "-Wextra", "-Werror", "-target", target.linkerTargetTriple, - "-c", join(nativeSources, "ts7-process.c"), "-o", object, -], { stdio: "inherit" }); -writeFileSync(ffi, JSON.stringify({ - ...JSON.parse(readFileSync(join(nativeSources, "ts7-process.ffi.json"), "utf8")), libraries: [object], -}, null, 2) + "\n"); -const result = await compile(join(root, "packages/compiler/src/native/main.ts"), { - outDir: output, outPath: executable, backend: "llvm", optimization: "release", strip: true, dynamic: false, ffiProfilePath: ffi, sanitize, -}); -if (!result.ok) throw new Error(result.diagnostics.map((item) => - `${item.loc ? `${item.loc.file}:${item.loc.start}: ` : ""}${item.code}: ${item.message}`, -).join("\n")); - -// The executable path can be canonical even when the build directory was -// reached through a symlink (for example /tmp on macOS). -const relativeRoot = realpathSync(output); -const packagePath = (name: string) => relative(relativeRoot, join(root, "packages", name.replace("@scriptc/", ""))); -const manifest: NativeToolchainManifest = { - schema: "scriptc.native-toolchain.v1", - compiler_version: (JSON.parse(readFileSync(join(root, "packages/compiler/package.json"), "utf8")) as { version: string }).version, - target: target.name, ts7: relative(relativeRoot, ts7Executable()), - llvm_package: packagePath(target.helper.packageName), runtime_pack: packagePath(target.runtimePackPackage), - linker: process.env["SCRIPTC_LINKER"] ?? target.defaultLinker, - linker_args: process.env["SCRIPTC_LINKER"] === undefined ? [...target.defaultLinkerArgs] : [], - dsymutil: process.env["SCRIPTC_DSYMUTIL"] ?? "dsymutil", -}; -writeFileSync(executable + ".json", JSON.stringify(manifest, null, 2) + "\n"); -console.log(executable); diff --git a/scripts/native-cli-assets.mts b/scripts/native-cli-assets.mts new file mode 100644 index 00000000..7a2c361a --- /dev/null +++ b/scripts/native-cli-assets.mts @@ -0,0 +1,73 @@ +import { cpSync, existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; +import { basename, dirname, join, relative } from "node:path"; +import type { NativeTargetSpec } from "../packages/compiler/src/backend/targets.js"; +import type { NativeToolchainManifest } from "../packages/compiler/src/native/toolchain.js"; + +/** Copy release payloads explicitly; development caches and build trees never ship. */ +function copyPackage(source: string, destination: string, members: readonly string[]): void { + mkdirSync(destination, { recursive: true }); + for (const member of ["package.json", ...members]) { + cpSync(join(source, member), join(destination, member), { + recursive: true, + filter: (path) => ![".cache", "node_modules", ".DS_Store"].includes(basename(path)), + }); + } + for (const name of readdirSync(source)) { + if (/^(?:LICENSE|COPYING|NOTICE|SCRIPTC_LICENSE|THIRD_PARTY_NOTICES)(?:\..*)?$/.test(name)) { + cpSync(join(source, name), join(destination, name)); + } + } +} + +export function stageNativeCliAssets(options: { + repository: string; + output: string; + target: NativeTargetSpec; + ts7: string; + compilerVersion: string; +}): Pick { + const { repository, output, target } = options; + const bin = join(output, "bin"); + const lib = join(output, "lib"); + mkdirSync(lib, { recursive: true }); + cpSync(join(repository, "LICENSE"), join(output, "LICENSE")); + const helper = join(lib, "llvm"); + const runtime = join(lib, target.runtimePackPackage.replace("@scriptc/", "")); + copyPackage(join(repository, "packages", target.helper.packageName.replace("@scriptc/", "")), helper, ["bin"]); + copyPackage(join(repository, "packages", target.runtimePackPackage.replace("@scriptc/", "")), runtime, ["artifacts", "runtime-pack.json"]); + const typescript = join(lib, "typescript"); + copyPackage(dirname(dirname(options.ts7)), typescript, ["lib"]); + const sources = join(lib, "runtime-sources"); + copyPackage(join(repository, "packages/runtime"), sources, ["src", "vendor"]); + const declarations = join(lib, "declarations"); + mkdirSync(declarations, { recursive: true }); + for (const name of ["scriptc.d.ts", "scriptc-overrides.d.ts", "scriptc-node-fallback.d.ts"]) { + cpSync(join(repository, "packages/compiler/ambient", name), join(declarations, name)); + } + // This optional Node host runs the built WASI module, never the compiler. + const wasi = join(lib, "wasi"); + mkdirSync(join(wasi, "cli"), { recursive: true }); + writeFileSync(join(wasi, "package.json"), '{"type":"module"}\n'); + for (const name of ["wasi-paths.js", "cli/wasi-paths.js", "cli/wasi-runner.js"]) { + const source = join(repository, "packages/compiler/dist", name); + if (!existsSync(source)) throw new Error("build the compiler workspace before packaging its WASI runner"); + cpSync(source, join(wasi, name)); + } + const identity = JSON.parse(readFileSync(join(typescript, "package.json"), "utf8")) as { version: string }; + writeFileSync(join(output, "THIRD_PARTY_NOTICES"), [ + `scriptc ${options.compilerVersion} includes the existing TypeScript ${identity.version}, LLVM, and C runtime dependencies.`, + "TypeScript licensing: lib/typescript/LICENSE and lib/typescript/NOTICE.txt.", + "LLVM licensing and notices: lib/llvm/LICENSE and lib/llvm/THIRD_PARTY_NOTICES.", + "Runtime dependency licenses: lib/runtime-sources/vendor/*/LICENSE*.", + "", + ].join("\n")); + return { + ts7: relative(bin, join(typescript, "lib", basename(options.ts7))), + llvm_package: relative(bin, helper), + runtime_pack: relative(bin, runtime), + runtime_packs: [{ target: target.name, path: relative(bin, runtime) }], + runtime_sources: relative(bin, sources), + declarations: relative(bin, declarations), + wasi_node_runner: relative(bin, join(wasi, "cli/wasi-runner.js")), + }; +} diff --git a/scripts/package-native-cli.mjs b/scripts/package-native-cli.mjs new file mode 100644 index 00000000..9597d4d1 --- /dev/null +++ b/scripts/package-native-cli.mjs @@ -0,0 +1,57 @@ +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { chmodSync, cpSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, dirname, join, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const repository = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const packages = resolve(process.argv[2] ?? join(repository, "packages")); +const output = resolve(process.argv[3] ?? join(repository, ".scriptc/releases")); +mkdirSync(output, { recursive: true }); +const compiler = JSON.parse(readFileSync(join(packages, "compiler/package.json"), "utf8")); +const runtimes = Object.keys(compiler.optionalDependencies).filter((name) => name.startsWith("@scriptc/runtime-")); +const sums = []; + +for (const name of readdirSync(packages).filter((name) => name.startsWith("cli-")).sort()) { + const source = join(packages, name); + const identity = JSON.parse(readFileSync(join(source, "package.json"), "utf8")); + if (identity.version !== compiler.version) throw new Error(`native CLI version mismatch: ${name}`); + const stage = mkdtempSync(join(process.platform === "win32" ? tmpdir() : "/tmp", "scriptc-release-")); + try { + cpSync(join(source, "package.json"), join(stage, "package.json")); + const distribution = join(stage, "dist"); + cpSync(join(source, "dist"), distribution, { recursive: true }); + const bin = join(distribution, "bin"); + const binary = join(bin, name.includes("win32-") ? "scriptc.exe" : "scriptc"); + const manifest = JSON.parse(readFileSync(binary + ".json", "utf8")); + const packs = new Map(); + for (const packageName of runtimes) { + const directory = packageName.replace("@scriptc/", ""); + const from = join(packages, directory); + const to = join(distribution, "lib", directory); + const runtime = JSON.parse(readFileSync(join(from, "runtime-pack.json"), "utf8")); + if (runtime.version !== identity.version || runtime.package !== packageName) throw new Error(`runtime version mismatch: ${packageName}`); + for (const member of ["package.json", "runtime-pack.json", "artifacts"]) { + cpSync(join(from, member), join(to, member), { recursive: true }); + } + packs.set(runtime.target.name, { target: runtime.target.name, path: relative(bin, to) }); + } + manifest.runtime_packs = [...packs.values()]; + writeFileSync(binary + ".json", JSON.stringify(manifest, null, 2) + "\n"); + // GitHub artifact transport drops executable modes. Restore the native + // tools before validating and creating the standalone archive. + for (const path of [binary, resolve(bin, manifest.ts7), resolve(bin, manifest.comptime), + join(resolve(bin, manifest.llvm_package), "bin", name.includes("win32-") ? "scriptc-llvm-codegen.exe" : "scriptc-llvm-codegen")]) { + chmodSync(path, 0o755); + } + execFileSync(process.execPath, [join(repository, "scripts/verify-native-cli.mjs"), stage], { stdio: "inherit" }); + const archive = join(output, `scriptc-${identity.version}-${name.slice(4)}.tar.gz`); + execFileSync("tar", ["-czf", archive, "-C", distribution, "."], { stdio: "inherit" }); + const digest = createHash("sha256").update(readFileSync(archive)).digest("hex"); + sums.push(`${digest} ${basename(archive)}`); + console.log(archive); + } finally { rmSync(stage, { recursive: true, force: true }); } +} +if (sums.length === 0) throw new Error("no native CLI packages found"); +writeFileSync(join(output, "SHA256SUMS"), sums.join("\n") + "\n"); diff --git a/scripts/sandbox-command.mjs b/scripts/sandbox-command.mjs index be9d186a..c29bba1a 100644 --- a/scripts/sandbox-command.mjs +++ b/scripts/sandbox-command.mjs @@ -2,6 +2,27 @@ export const MAX_INLINE_SANDBOX_COMMAND_BYTES = 768; // Outside the shell exit-code range, so an actual exit 125 stays a failure. export const REMOTE_COMMAND_PENDING = 256; +/** A transport failure cannot determine the remote command's exit status. */ +export async function waitForSandboxCommand(probe, { deadline, label, onPending }) { + let lastError; + for (;;) { + const remaining = deadline - Date.now(); + if (remaining <= 0) { + throw new Error(`${label} did not confirm completion before its timeout`, { cause: lastError }); + } + try { + await probe(remaining); + return; + } catch (error) { + if (error.remoteExitCode !== undefined && error.remoteExitCode !== REMOTE_COMMAND_PENDING) throw error; + lastError = error; + onPending(error); + const delay = Math.min(1000, deadline - Date.now()); + if (delay > 0) await new Promise((resolve) => setTimeout(resolve, delay)); + } + } +} + export const shellQuote = (value) => `'${value.replaceAll("'", `'"'"'`)}'`; /** Keep long shell programs out of the local CLI's argument vector. The diff --git a/scripts/sandbox-test.mjs b/scripts/sandbox-test.mjs index 58572f83..d47ba30b 100644 --- a/scripts/sandbox-test.mjs +++ b/scripts/sandbox-test.mjs @@ -23,7 +23,7 @@ import { filterExistingWorktreePaths, workspaceResetCommand, } from "./worktree-files.mjs"; -import { REMOTE_COMMAND_PENDING, sandboxCommand, sandboxStatusCommand, shellQuote } from "./sandbox-command.mjs"; +import { REMOTE_COMMAND_PENDING, sandboxCommand, sandboxStatusCommand, waitForSandboxCommand } from "./sandbox-command.mjs"; const root = fileURLToPath(new URL("../", import.meta.url)); const laneCaseShardedFiles = [ @@ -432,15 +432,10 @@ const execIn = async ( } catch (error) { if (error.remoteExitCode !== undefined) throw error; console.warn(`[${label}] CLI completion was not confirmed (${error.message}); checking the remote command status...`); - for (;;) { - const remaining = deadline - Date.now(); - if (remaining <= 0) { - await recoveredLog(); - throw new Error(`${label} did not confirm completion before its timeout`, { cause: error }); - } - const probeMarker = `__SCRIPTC_REMOTE_PROBE_${randomBytes(12).toString("hex")}__`; - const probeScript = sandboxStatusCommand(statusPath, probeMarker, Math.min(20, Math.floor(remaining / 1000))); - try { + try { + await waitForSandboxCommand(async (remaining) => { + const probeMarker = `__SCRIPTC_REMOTE_PROBE_${randomBytes(12).toString("hex")}__`; + const probeScript = sandboxStatusCommand(statusPath, probeMarker, Math.min(20, Math.floor(remaining / 1000))); await vercel( ["sandbox", "exec", "--timeout", "1m", "--workdir", workdir, worker.name, "sh", "-c", probeScript], { @@ -450,15 +445,14 @@ const execIn = async ( timeoutMs: Math.min(60_000, remaining), }, ); - await recoveredLog(); - return; - } catch (probeError) { - if (probeError.remoteExitCode !== REMOTE_COMMAND_PENDING) { - await recoveredLog(); - throw probeError; - } - console.log(`[${label}] remote command has not recorded completion; waiting...`); - } + }, { + deadline, label, + onPending: (probeError) => console.warn(probeError.remoteExitCode === REMOTE_COMMAND_PENDING + ? `[${label}] remote command has not recorded completion; waiting...` + : `[${label}] remote status probe was not confirmed (${probeError.message}); retrying...`), + }); + } finally { + await recoveredLog(); } } }; diff --git a/scripts/sync-versions.mjs b/scripts/sync-versions.mjs index f6b27c7a..905ee7d5 100644 --- a/scripts/sync-versions.mjs +++ b/scripts/sync-versions.mjs @@ -24,6 +24,9 @@ for (const pkg of [ "runtime-win32-x64-msvc", "runtime-wasm32-wasi", "runtime-ios-arm64", "runtime-ios-simulator-arm64", "runtime-android-arm64", "compiler", + "cli-darwin-arm64", "cli-darwin-x64", + "cli-linux-x64-gnu", "cli-linux-arm64-gnu", + "cli-linux-x64-musl", "cli-linux-arm64-musl", "cli-win32-x64-msvc", "llvm-darwin-arm64", "llvm-darwin-x64", "llvm-linux-x64-gnu", "llvm-linux-arm64-gnu", "llvm-linux-x64-musl", "llvm-linux-arm64-musl", "llvm-win32-x64-msvc", diff --git a/scripts/verify-native-cli.mjs b/scripts/verify-native-cli.mjs new file mode 100644 index 00000000..58fd1aae --- /dev/null +++ b/scripts/verify-native-cli.mjs @@ -0,0 +1,50 @@ +import { execFileSync } from "node:child_process"; +import { existsSync, readFileSync, readdirSync, realpathSync, statSync } from "node:fs"; +import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; + +const root = realpathSync(resolve(process.argv[2] ?? ".")); +const identity = JSON.parse(readFileSync(join(root, "package.json"), "utf8")); +const target = identity.name.replace("@scriptc/cli-", ""); +const distribution = join(root, "dist"); +const binary = join(distribution, "bin", target.startsWith("win32-") ? "scriptc.exe" : "scriptc"); +const toolchain = JSON.parse(readFileSync(binary + ".json", "utf8")); +if (toolchain.schema !== "scriptc.native-toolchain.v1" || toolchain.compiler_version !== identity.version) { + throw new Error("native CLI manifest version does not match the package"); +} +if (statSync(binary).size < 1024) throw new Error("native compiler payload is missing"); +if (!target.startsWith("win32-") && !(statSync(binary).mode & 0o111)) throw new Error("native compiler is not executable"); +const within = (path) => { + const child = relative(distribution, realpathSync(path)); + if (child === ".." || child.startsWith(".." + sep) || isAbsolute(child)) throw new Error(`distribution references external asset: ${path}`); +}; +for (const name of ["ts7", "llvm_package", "runtime_pack", "runtime_sources", "declarations", "comptime", "wasi_node_runner"]) { + const value = toolchain[name]; + if (typeof value !== "string" || isAbsolute(value)) throw new Error(`native toolchain needs a relative ${name}`); + within(resolve(dirname(binary), value)); +} +const pack = JSON.parse(readFileSync(resolve(dirname(binary), toolchain.runtime_pack, "runtime-pack.json"), "utf8")); +if (pack.version !== identity.version || pack.target.name !== toolchain.target) throw new Error("native compiler runtime identity mismatch"); +const seen = new Set(); +for (const runtime of toolchain.runtime_packs ?? []) { + if (seen.has(runtime.target)) throw new Error(`duplicate runtime target: ${runtime.target}`); + seen.add(runtime.target); + if (isAbsolute(runtime.path)) throw new Error("native runtime pack paths must be relative"); + const directory = resolve(dirname(binary), runtime.path); + within(directory); + const manifest = JSON.parse(readFileSync(join(directory, "runtime-pack.json"), "utf8")); + const packageIdentity = JSON.parse(readFileSync(join(directory, "package.json"), "utf8")); + if (manifest.schema !== "scriptc.runtime-pack.v1" || manifest.target.name !== runtime.target || + manifest.version !== identity.version || packageIdentity.version !== identity.version || + packageIdentity.name !== manifest.package) throw new Error(`native runtime pack identity mismatch: ${runtime.target}`); +} +for (const name of ["LICENSE", "THIRD_PARTY_NOTICES"]) { + if (!existsSync(join(distribution, name))) throw new Error(`native compiler is missing ${name}`); +} +for (const directory of [distribution, join(distribution, "lib/runtime-sources/vendor")]) { + if (readdirSync(directory).some((name) => ["node_modules", ".cache", "seed"].includes(name))) throw new Error(`development files present in ${directory}`); +} +if (process.argv.includes("--run")) { + const version = execFileSync(binary, ["--version"], { encoding: "utf8", env: { ...process.env, PATH: "" } }).trim(); + if (version !== identity.version) throw new Error(`native command reported ${version}, expected ${identity.version}`); +} +console.log(`${identity.name}@${identity.version}: native distribution verified`); diff --git a/tests/corpus/array-fill-unit-values.ts b/tests/corpus/array-fill-unit-values.ts new file mode 100644 index 00000000..67c4a9b1 --- /dev/null +++ b/tests/corpus/array-fill-unit-values.ts @@ -0,0 +1,10 @@ +const empty = new Array(3).fill(null); +console.log(JSON.stringify(empty), empty.map((value, index) => String(index) + ":" + String(value === null)).join(",")); +const mixed: (number | null)[] = [1, 2, 3, 4]; +console.log(JSON.stringify(mixed.fill(null, 1, -1))); +console.log(JSON.stringify(mixed.fill(7, 2))); +let calls = 0; +function nil(): null { calls++; return null; } +console.log(JSON.stringify(mixed.fill(nil())), calls); +const missing = new Array(2).fill(undefined); +console.log(JSON.stringify(missing), missing.map((value, index) => String(index) + ":" + String(value === undefined)).join(",")); diff --git a/tests/corpus/array-optional-number-write-index.ts b/tests/corpus/array-optional-number-write-index.ts new file mode 100644 index 00000000..4db5a342 --- /dev/null +++ b/tests/corpus/array-optional-number-write-index.ts @@ -0,0 +1,12 @@ +const order = [2, 0, 1]; +const mapped: number[] = new Array(3).fill(-1); +order.forEach((oldIndex, newIndex) => { mapped[oldIndex] = newIndex; }); +console.log(mapped.join(",")); + +const candidates = new Map(); +candidates.set("entry", [1, 2]); +const included = new Array(3).fill(false); +for (const candidate of candidates.get("entry") ?? new Array()) { + if (included[candidate] !== true) included[candidate] = true; +} +console.log(included.join(",")); diff --git a/tests/corpus/closure-nullable-union-return.ts b/tests/corpus/closure-nullable-union-return.ts new file mode 100644 index 00000000..9042cb87 --- /dev/null +++ b/tests/corpus/closure-nullable-union-return.ts @@ -0,0 +1,65 @@ +type ValueType = { kind: "string" } | { kind: "dyn" }; +type Expr = + | { kind: "literal"; value: string; type: ValueType } + | { kind: "object"; fields: { key: string; value: Expr }[]; type: ValueType } + | { kind: "call"; args: Expr[]; name: string; type: ValueType }; +const DYNAMIC: ValueType = { kind: "dyn" }; + +function assemble(groups: string[][]): Expr { + let fields: { key: string; value: Expr }[] = []; + let acc: Expr | null = null; + const flush = (): void => { + if (fields.length === 0) return; + const chunk: Expr = { kind: "object", fields, type: DYNAMIC }; + acc = acc === null ? chunk : { kind: "call", args: [acc, chunk], name: "merge", type: DYNAMIC }; + fields = []; + }; + for (const group of groups) { + for (const key of group) fields.push({ key, value: { kind: "literal", value: key, type: { kind: "string" } } }); + flush(); + if (group.length > 10) { + acc ??= { kind: "object", fields: [], type: DYNAMIC }; + acc = { kind: "call", args: [acc], name: "large", type: DYNAMIC }; + } + } + if (acc !== null) return acc; + return { kind: "object", fields: [], type: DYNAMIC }; +} + +for (const groups of [[], [["headers"]], [["a"], ["b", "c"]]]) { + const expression = assemble(groups); + console.log(expression.kind, expression.type.kind, JSON.stringify(expression)); +} + +function inspect(expression: Expr): void { + console.log(expression.kind, expression.type.kind, JSON.stringify(expression)); +} + +async function transfer(): Promise { + let value: Expr = { kind: "call", args: [], name: "initial", type: DYNAMIC }; + const replacement: Expr = { kind: "object", fields: [], type: DYNAMIC }; + const update = (): void => { value = replacement; }; + update(); + inspect(value); + let assigned: Expr = { kind: "literal", value: "initial", type: DYNAMIC }; + assigned = value; + inspect(assigned); + const values: Expr[] = [value]; + const holder: { value: Expr | null } = { value }; + inspect(values[0]!); + if (holder.value !== null) inspect(holder.value); + console.log(values[0] === replacement, holder.value === replacement); + return (value); +} + +inspect(await transfer()); + +function subset(value: Expr): Expr | null { + if (value.kind === "literal") { + const selected: Extract | null = value; + return selected; + } + return null; +} +const selected = subset({ kind: "literal", value: "subset", type: DYNAMIC }); +if (selected !== null) inspect(selected); diff --git a/tests/corpus/crypto-hash-optional-input.ts b/tests/corpus/crypto-hash-optional-input.ts new file mode 100644 index 00000000..1e161529 --- /dev/null +++ b/tests/corpus/crypto-hash-optional-input.ts @@ -0,0 +1,17 @@ +import { createHash, createHmac } from "node:crypto"; + +const environment: Record = { present: "hello" }; +for (const name of ["present", "missing"]) { + const value = environment[name]; + const text: string = value ?? ""; + console.log(createHash("sha256").update(text).digest("hex")); + console.log(createHmac("sha256", "secret").update(text).digest("hex")); + if (value !== undefined) console.log(createHash("sha256").update(value).digest("hex")); +} + +const buffers: (Buffer | undefined)[] = [Buffer.from("hello"), undefined]; +for (let index = 0; index < buffers.length; index++) { + const buffer = buffers[index] ?? Buffer.from(""); + console.log(createHash("sha256").update(buffer).digest("hex")); + console.log(createHmac("sha256", "secret").update(buffer).digest("hex")); +} diff --git a/tests/corpus/fs-remove-readonly.ts b/tests/corpus/fs-remove-readonly.ts new file mode 100644 index 00000000..64711c10 --- /dev/null +++ b/tests/corpus/fs-remove-readonly.ts @@ -0,0 +1,18 @@ +import { chmodSync, existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const root = mkdtempSync(join(tmpdir(), "scriptc-readonly-rm-")); +try { + const file = join(root, "readonly.txt"); + writeFileSync(file, "keep until removed"); + chmodSync(file, 0o400); + rmSync(file); + console.log(existsSync(file)); + const nested = join(root, "tree", "nested"); + mkdirSync(nested, { recursive: true }); + writeFileSync(join(nested, "payload"), "read-only payload"); + chmodSync(join(nested, "payload"), 0o400); + rmSync(join(root, "tree"), { recursive: true, force: true }); + console.log(existsSync(join(root, "tree"))); +} finally { rmSync(root, { recursive: true, force: true }); } diff --git a/tests/corpus/fs-write-string-bytes-union.ts b/tests/corpus/fs-write-string-bytes-union.ts new file mode 100644 index 00000000..ef04a497 --- /dev/null +++ b/tests/corpus/fs-write-string-bytes-union.ts @@ -0,0 +1,26 @@ +import { appendFileSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const directory = mkdtempSync(join(tmpdir(), "scriptc-write-union-")); +const path = join(directory, "data"); +function write(value: string | Uint8Array): void { writeFileSync(path, value); } +function append(value: string | Uint8Array): void { appendFileSync(path, value); } +let order = ""; +function destination(): string { order += "path "; return path; } +function data(bytes: boolean): string | Uint8Array { + order += "data "; + return bytes ? Buffer.from([0, 128, 255]) : "é"; +} +try { + for (const value of ["text", new Uint8Array([0, 128, 255]), Buffer.from("é"), "", new Uint8Array(0)]) { + write(value); + append("!"); + append(Buffer.from([0, 255])); + console.log(readFileSync(path).toString("hex")); + } + writeFileSync(destination(), data(true)); + appendFileSync(destination(), data(false)); + console.log(order); + console.log(readFileSync(path).toString("hex")); +} finally { rmSync(directory, { recursive: true, force: true }); } diff --git a/tests/corpus/object-enumeration-array-reads.ts b/tests/corpus/object-enumeration-array-reads.ts new file mode 100644 index 00000000..e281e50d --- /dev/null +++ b/tests/corpus/object-enumeration-array-reads.ts @@ -0,0 +1,9 @@ +const records: { first: number; second: number }[] = [{ first: 1, second: 2 }]; +for (const record of records) console.log(Object.keys(record).join(","), Object.values(record).join(","), JSON.stringify(Object.entries(record))); +const dictionaries: Record[] = [{ b: 2, a: 1 }]; +for (const dict of dictionaries) console.log(Object.keys(dict).join(","), Object.values(dict).join(","), JSON.stringify(Object.entries(dict))); +const empty: Record[] = []; +try { console.log(Object.values(empty[0])); } +catch (error) { console.log(error instanceof TypeError, error instanceof Error ? error.message : "unexpected"); } +try { console.log(Object.keys(records[20])); } +catch (error) { console.log(error instanceof TypeError, error instanceof Error ? error.message : "unexpected"); } diff --git a/tests/corpus/record-optional-json-presence.ts b/tests/corpus/record-optional-json-presence.ts new file mode 100644 index 00000000..c47a02f4 --- /dev/null +++ b/tests/corpus/record-optional-json-presence.ts @@ -0,0 +1,32 @@ +interface Entry { + name: string; + synthesized?: true; + count?: number; +} + +function inspect(value: unknown): void { + if (value === null || typeof value !== "object" || Array.isArray(value)) return; + const entry = value as Record; + console.log(entry.name, "synthesized" in entry, Object.hasOwn(entry, "synthesized"), entry.synthesized === true); + console.log(Object.keys(entry).join(","), JSON.stringify(entry)); +} + +function make(name: string, synthesized: boolean): Entry { + const entry: Entry = { name }; + if (synthesized) entry.synthesized = true; + return entry; +} + +const entries: Entry[] = [{ name: "declared" }, { name: "inline", synthesized: true }, make("built", false), make("generated", true)]; +for (const entry of entries) inspect(JSON.parse(JSON.stringify(entry))); +const document = { entries }; +const unknownDocument: unknown = JSON.parse(JSON.stringify(document)); +if (unknownDocument !== null && typeof unknownDocument === "object") { + const dict = unknownDocument as Record; + const items = dict.entries; + if (Array.isArray(items)) for (const item of items) inspect(item); +} + +// Boxing an explicitly present undefined value must retain its key. +inspect({ name: "undefined", synthesized: undefined }); +inspect({ name: "undefined-count", count: undefined }); diff --git a/tests/corpus/util-parseargs-wrapped/main.ts b/tests/corpus/util-parseargs-wrapped/main.ts new file mode 100644 index 00000000..1375adb4 --- /dev/null +++ b/tests/corpus/util-parseargs-wrapped/main.ts @@ -0,0 +1,26 @@ +import { parseArgs } from "node:util"; + +const options = { + output: { type: "string", short: "o" }, + verbose: { type: "boolean", default: false }, + keep: { type: "boolean", default: true }, + include: { type: "string", multiple: true }, +} as const; + +type Arguments = ReturnType>; +function parse(args: string[]): Arguments { + return parseArgs({ args, options, allowPositionals: true, allowNegative: true }); +} +function print(args: string[]): void { + const { values, positionals } = parse(args); + console.log(values.output ?? "none", values.verbose, values.keep); + console.log((values.include ?? []).join(":"), positionals.join(":")); +} +print(["build", "input.ts", "-o", "program", "--verbose", "--no-keep", "--include=one", "--include", "two"]); +print([]); +try { parse(["--unknown"]); } catch (error) { console.log(error instanceof Error ? error.message.split("\n")[0] : String(error)); } + +// Similar user declarations retain their own structural field types. +type ParsedResults = { values: number; positionals: boolean }; +function local(value: ParsedResults): number { return value.positionals ? value.values : 0; } +console.log(local({ values: 42, positionals: true })); diff --git a/tests/corpus/util-parseargs-wrapped/tsconfig.json b/tests/corpus/util-parseargs-wrapped/tsconfig.json new file mode 100644 index 00000000..27d492dd --- /dev/null +++ b/tests/corpus/util-parseargs-wrapped/tsconfig.json @@ -0,0 +1,3 @@ +{ + "compilerOptions": { "strict": true, "types": ["node"] } +} diff --git a/tests/fixtures/self-hosting/class-types.ts b/tests/fixtures/self-hosting/class-types.ts new file mode 100644 index 00000000..8e680f14 --- /dev/null +++ b/tests/fixtures/self-hosting/class-types.ts @@ -0,0 +1,26 @@ +import { F64, typeKey, type IrExpr, type IrModule, type IrType } from "../../../packages/compiler/src/ir/ir.js"; +import { sanitizeUnregisteredClassTypes } from "../../../packages/compiler/src/frontend/lowering/sanitize-class-types.js"; + +const loc = { file: "class-types.ts", start: 0, end: 1 }; +const missing: IrType = { kind: "object", className: "Fenced" }; +const kept: IrType = { kind: "object", className: "Registered" }; +const expr: IrExpr = { kind: "varRef", localId: "x", type: missing, loc }; +const nested: IrType = { kind: "func", params: [{ kind: "array", elem: missing }], + ret: { kind: "generator", yieldT: missing, retT: kept, nextT: missing } }; +const module: IrModule = { irVersion: 13, sourceFile: loc.file, entry: "main", + functions: [{ name: "main", params: [{ localId: "x", name: "x", type: missing }], + returnType: nested, locals: [], loc, body: [{ kind: "exprStmt", expr, loc }] }], + records: [{ id: "r", fields: [{ name: "callback", type: nested }], indexValue: missing }], + unions: [{ id: "u", arms: [missing, kept, { kind: "classval", className: "Fenced" }] }], +}; +sanitizeUnregisteredClassTypes(module, (name) => name === "Registered"); +console.log(module.functions[0]!.params[0]!.type.kind, expr.type.kind); +console.log(typeKey(nested)); +for (const record of module.records ?? []) { + for (const field of record.fields) console.log(typeKey(field.type)); + if (record.indexValue) console.log(typeKey(record.indexValue)); +} +for (const union of module.unions ?? []) console.log(union.arms.map(typeKey).join(",")); +console.log(missing.kind, kept.kind, F64.kind); +sanitizeUnregisteredClassTypes(module, (name) => name === "Registered"); +console.log(expr.type.kind); diff --git a/tests/fixtures/self-hosting/native-cache.ts b/tests/fixtures/self-hosting/native-cache.ts new file mode 100644 index 00000000..2cf12e0b --- /dev/null +++ b/tests/fixtures/self-hosting/native-cache.ts @@ -0,0 +1,24 @@ +import { readFileSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { NativeCache, contentDigest } from "../../../packages/compiler/src/native/cache.js"; +import { NativeExecutableCache } from "../../../packages/compiler/src/native/executable-cache.js"; +import { toolchainEnvironmentCachePolicy, toolchainEnvironmentFingerprint } from "../../../packages/compiler/src/backend/toolchain-environment.js"; + +const directory = process.argv[2]!; +const linker = process.argv[3]!; +const script = process.argv[4]!; +const library = process.argv[5]!; +const output = join(directory, "program"); +const cache = new NativeCache(join(directory, "cache")); +const key = contentDigest("native executable"); +const entry = new NativeExecutableCache(cache, key, linker, false, [library]); +console.log(toolchainEnvironmentFingerprint({ LANG: "en_US.UTF-8" })); +console.log(JSON.stringify(toolchainEnvironmentCachePolicy({ LIBRARY_PATH: "/fixture" }))); +console.log(entry.trace([script, library], directory)); +writeFileSync(output, "cached program"); +entry.publish(output); +rmSync(output); +console.log(entry.restore(output)); +console.log(readFileSync(output, "utf8")); +writeFileSync(library, "changed library"); +console.log(entry.restore(output)); diff --git a/tests/harness/library-dispatch.test.ts b/tests/harness/library-dispatch.test.ts index 8d23da83..1c7285a0 100644 --- a/tests/harness/library-dispatch.test.ts +++ b/tests/harness/library-dispatch.test.ts @@ -8,7 +8,7 @@ import { compileLibrary } from "@scriptc/compiler"; const fixture = join(import.meta.dirname, "../library-mode/wasm/dispatch.mjs"); const modes = ["native", "sanitized", "wasm-dev"] as const; -test.each(modes)("synchronous checked dispatch in a %s library matches Node", async (mode, context) => { +test.for(modes)("synchronous checked dispatch in a %s library matches Node", async (mode, context) => { if (mode === "wasm-dev" ? spawnSync("zig", ["version"]).status !== 0 : process.platform !== "darwin" && process.platform !== "linux") context.skip(); const directory = await mkdtemp("/tmp/scriptc-library-dispatch-"); const oldTarget = process.env["SCRIPTC_TARGET"]; diff --git a/tests/harness/sandbox-command.test.ts b/tests/harness/sandbox-command.test.ts index 9a03e67e..366b29be 100644 --- a/tests/harness/sandbox-command.test.ts +++ b/tests/harness/sandbox-command.test.ts @@ -2,17 +2,56 @@ import { execFile, execFileSync } from "node:child_process"; import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { promisify } from "node:util"; -import { afterEach, expect, test } from "vitest"; -import { REMOTE_COMMAND_PENDING, sandboxCommand, sandboxStatusCommand } from "../../scripts/sandbox-command.mjs"; +import { afterEach, expect, test, vi } from "vitest"; +import { REMOTE_COMMAND_PENDING, sandboxCommand, sandboxStatusCommand, waitForSandboxCommand } from "../../scripts/sandbox-command.mjs"; const roots: string[] = []; const statuses: string[] = []; afterEach(async () => { + vi.useRealTimers(); await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); await Promise.all(statuses.splice(0).map((path) => rm(path, { force: true }))); }); +test("status recovery retries lost probes and pending commands until a confirmed success", async () => { + vi.useFakeTimers(); + const probe = vi.fn() + .mockRejectedValueOnce(new Error("transport disconnected")) + .mockRejectedValueOnce(Object.assign(new Error("pending"), { remoteExitCode: REMOTE_COMMAND_PENDING })) + .mockResolvedValueOnce(undefined); + const onPending = vi.fn(); + const recovery = waitForSandboxCommand(probe, { deadline: Date.now() + 5000, label: "tests", onPending }); + await vi.runAllTimersAsync(); + await expect(recovery).resolves.toBeUndefined(); + expect(probe.mock.calls).toEqual([[5000], [4000], [3000]]); + expect(onPending).toHaveBeenCalledTimes(2); +}); + +test("status recovery preserves a confirmed remote failure after a transport failure", async () => { + vi.useFakeTimers(); + const failure = Object.assign(new Error("tests failed"), { remoteExitCode: 7 }); + const probe = vi.fn().mockRejectedValueOnce(new Error("disconnected")).mockRejectedValue(failure); + const recovery = expect(waitForSandboxCommand(probe, { + deadline: Date.now() + 5000, label: "tests", onPending: vi.fn(), + })).rejects.toBe(failure); + await vi.runAllTimersAsync(); + await recovery; + expect(probe).toHaveBeenCalledTimes(2); +}); + +test("unconfirmed status probes cannot extend the original command deadline", async () => { + vi.useFakeTimers(); + const disconnected = new Error("status timeout"); + const probe = vi.fn().mockRejectedValue(disconnected); + const recovery = expect(waitForSandboxCommand(probe, { + deadline: Date.now() + 1500, label: "tests", onPending: vi.fn(), + })).rejects.toMatchObject({ message: "tests did not confirm completion before its timeout", cause: disconnected }); + await vi.runAllTimersAsync(); + await recovery; + expect(probe.mock.calls).toEqual([[1500], [500]]); +}); + test("short commands stay inline and retain the remote exit contract", async () => { const marker = `__SCRIPTC_COMMAND_TEST_${process.pid}_SHORT__`; const prepared = sandboxCommand("sh", ["-c", "printf 'failure detail\\n' >&2; exit 7"], marker); diff --git a/tests/harness/self-hosting-class-types.test.ts b/tests/harness/self-hosting-class-types.test.ts new file mode 100644 index 00000000..b6585b3c --- /dev/null +++ b/tests/harness/self-hosting-class-types.test.ts @@ -0,0 +1,30 @@ +import { spawnSync } from "node:child_process"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { compileInChild } from "./self-hosting-compiler-process.js"; + +test("native class type cleanup updates shared IR without dynamic snapshots", async () => { + const root = join(import.meta.dirname, "../.."); + const entry = join(root, "tests/fixtures/self-hosting/class-types.ts"); + const directory = mkdtempSync(join(process.platform === "win32" ? tmpdir() : "/tmp", "scriptc-class-types-")); + try { + const built = await compileInChild(entry, { outDir: directory, + outPath: join(directory, "class-types" + (process.platform === "win32" ? ".exe" : "")), + optimization: "dev", dynamic: false, sanitize: process.env["SCRIPTC_SAN"] === "1" }); + if (!built.ok) throw new Error(JSON.stringify(built.diagnostics)); + const options = { cwd: root, encoding: "utf8" as const, timeout: 30_000 }; + const oracle = spawnSync(process.execPath, ["--import", "tsx", entry], options); + const native = spawnSync(built.binaryPath, [], options); + for (const result of [oracle, native]) { + expect(result.error).toBeUndefined(); + expect(result.signal, result.stderr).toBeNull(); + expect(result.status, result.stderr).toBe(0); + expect(result.stderr).toBe(""); + } + expect(native.stdout).toBe(oracle.stdout); + expect(native.stdout).toMatch(/^f64 f64\n/); + expect(native.stdout).toMatch(/object object f64\nf64\n$/); + } finally { rmSync(directory, { recursive: true, force: true }); } +}, 300_000); diff --git a/tests/harness/self-hosting-native-cache.test.ts b/tests/harness/self-hosting-native-cache.test.ts new file mode 100644 index 00000000..5ce0aa00 --- /dev/null +++ b/tests/harness/self-hosting-native-cache.test.ts @@ -0,0 +1,41 @@ +import { spawnSync } from "node:child_process"; +import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { compileInChild } from "./self-hosting-compiler-process.js"; + +const root = join(import.meta.dirname, "../.."); +const entry = join(root, "tests/fixtures/self-hosting/native-cache.ts"); + +test("native executable cache discovery, restoration and invalidation match Node", async () => { + const directory = mkdtempSync(join(process.platform === "win32" ? tmpdir() : "/tmp", "scriptc-native-cache-")); + try { + const built = await compileInChild(entry, { outDir: directory, + outPath: join(directory, "cache" + (process.platform === "win32" ? ".exe" : "")), + optimization: "dev", dynamic: false, sanitize: process.env["SCRIPTC_SAN"] === "1" }); + if (!built.ok) throw new Error(JSON.stringify(built.diagnostics)); + const linker = join(directory, "linker.mjs"); + writeFileSync(linker, 'console.error(JSON.stringify(process.argv[2]));\n'); + const sdk = join(directory, "sdk"); + mkdirSync(sdk); + const library = join(sdk, "system library.tbd"); + const results: string[] = []; + for (const native of [false, true]) { + const stage = join(directory, native ? "native" : "node"); + mkdirSync(stage); + writeFileSync(library, "system library"); + const args = [stage, process.execPath, linker, library]; + const result = spawnSync(native ? built.binaryPath : process.execPath, native ? args : ["--import", "tsx", entry, ...args], { + cwd: root, encoding: "utf8", timeout: 30_000, env: { ...process.env, ...(native ? { PATH: "" } : {}) }, + }); + expect(result.error).toBeUndefined(); + expect(result.signal, result.stderr).toBeNull(); + expect(result.status, result.stderr + result.stdout).toBe(0); + expect(result.stderr).toBe(""); + expect(result.stdout).toContain("true\ntrue\ncached program\nfalse\n"); + results.push(result.stdout); + } + expect(results[1]).toBe(results[0]); + } finally { rmSync(directory, { recursive: true, force: true }); } +}, 300_000); diff --git a/tests/harness/self-hosting-native-driver.test.ts b/tests/harness/self-hosting-native-driver.test.ts index 43b1aac0..88c2d93c 100644 --- a/tests/harness/self-hosting-native-driver.test.ts +++ b/tests/harness/self-hosting-native-driver.test.ts @@ -1,8 +1,9 @@ import { execFile, spawnSync } from "node:child_process"; -import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { existsSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { delimiter, join, resolve } from "node:path"; import { promisify } from "node:util"; +import { WASI } from "node:wasi"; import { expect, test } from "vitest"; import type { NativeToolchainManifest } from "../../packages/compiler/src/native/toolchain.js"; import { bootstrapStep } from "./self-hosting-timing.js"; @@ -29,14 +30,18 @@ function absoluteCommand(command: string): string { throw new Error(`native tool is not on PATH: ${command}`); } -test("the standalone compiler builds programs and rebuilds itself with Node unavailable", async () => { +test("the production CLI relocates, builds programs, and rebuilds itself with Node unavailable", async () => { const directory = mkdtempSync(join(process.platform === "win32" ? tmpdir() : "/tmp", "scriptc-native-bootstrap-")); const executable = (name: string) => join(directory, name + (process.platform === "win32" ? ".exe" : "")); const options = { cwd: root, timeout: 1_800_000, maxBuffer: 16 * 1024 * 1024 }; try { - await bootstrapStep("build native compiler seed", () => - exec(process.execPath, ["--max-old-space-size=8192", "--import", "tsx", join(root, "scripts/build-native-compiler.mts"), directory], options)); - const seed = executable("scriptc-native"); + const distribution = join(directory, "distribution"); + await bootstrapStep("build production CLI seed", () => + exec(process.execPath, ["--max-old-space-size=8192", "--import", "tsx", join(root, "scripts/build-native-cli.mts"), distribution], options)); + // All compiler assets must survive moving the complete distribution. + const relocated = join(directory, "relocated"); + renameSync(distribution, relocated); + const seed = join(relocated, "bin", "scriptc" + (process.platform === "win32" ? ".exe" : "")); const manifestPath = seed + ".json"; const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as NativeToolchainManifest; manifest.linker = absoluteCommand(manifest.linker); @@ -47,28 +52,27 @@ test("the standalone compiler builds programs and rebuilds itself with Node unav manifest.linker_args.push("--ld-path=" + absoluteCommand(linker.stdout.trim())); } if (process.platform === "darwin") manifest.dsymutil = absoluteCommand(manifest.dsymutil); + if (process.platform === "darwin") manifest.relocatable_linker = absoluteCommand("ld"); + manifest.archiver = absoluteCommand(process.platform === "win32" ? "zig" : "ar"); + manifest.archiver_args = process.platform === "win32" ? ["ar"] : []; writeFileSync(manifestPath, JSON.stringify(manifest)); - const nativeOptions = { ...options, env: { ...process.env, PATH: "" } }; + const nativeOptions = { ...options, env: { ...process.env, PATH: "", SCRIPTC_TOOLCHAIN: manifestPath, + SCRIPTC_CACHE_DIR: join(directory, "cache") } }; const invoke = async (compiler: string, args: string[]) => { - const result = await exec(compiler, [...args, "--toolchain", manifestPath], nativeOptions).catch((error: unknown) => { + const result = await exec(compiler, args, nativeOptions).catch((error: unknown) => { const failure = error as Error & { code?: string | number; signal?: string; stdout?: string; stderr?: string }; throw new Error(`${failure.message}\ncode=${failure.code} signal=${failure.signal}\n${failure.stderr ?? ""}\n${failure.stdout ?? ""}`, { cause: error }); }); expect(comparableStderr(result.stderr)).toBe(""); - const built = JSON.parse(result.stdout) as { outputPath: string; llvmPath?: string; stats: { - statementsTotal: number; statementsFailed: number; statementsIsland: number; functionsSkipped: number; - } }; - expect(built.stats.statementsFailed).toBe(0); - expect(built.stats.statementsIsland).toBe(0); - expect(built.stats.functionsSkipped).toBe(0); - return built; + return result.stdout; }; - expect((await exec(seed, ["--help"], nativeOptions)).stdout).toContain("Usage: scriptc-native"); - const checkProgram = async (compiler: string, source: string, backend: string) => { + expect(await invoke(seed, ["--help"])).toContain("scriptc build"); + expect((await invoke(seed, ["--version"])).trim()).toBe(manifest.compiler_version); + const checkProgram = async (compiler: string, source: string, extra: string[] = []) => { // This basename formerly collided with the driver's temporary object. const output = executable("program.o"); - const built = await invoke(compiler, ["build", source, "-o", output, "--backend", backend, "--dev", "--strip"]); - expect(built.outputPath).toBe(output); + const built = await invoke(compiler, ["build", source, "-o", output, "--optimization=dev", "--strip", ...extra]); + expect(built.trim()).toBe(output); const oracle = spawnSync(process.execPath, [source], options); const actual = spawnSync(output, [], nativeOptions); for (const result of [oracle, actual]) { @@ -82,41 +86,111 @@ test("the standalone compiler builds programs and rebuilds itself with Node unav const sample = join(root, "tests/corpus/class-array-optional-return.ts"); const unionSample = join(root, "tests/corpus/union-nested-layout-discriminant.ts"); const receiverSample = join(root, "tests/corpus/llvm-read-receiver-lifetime.ts"); - await checkProgram(seed, sample, "llvm"); - await checkProgram(seed, unionSample, "llvm"); - await checkProgram(seed, receiverSample, "llvm"); + await checkProgram(seed, sample); + await checkProgram(seed, unionSample); + await checkProgram(seed, receiverSample); + await checkProgram(seed, join(root, "tests/corpus/closure-nullable-union-return.ts")); + await checkProgram(seed, join(root, "tests/corpus/record-optional-json-presence.ts")); + await checkProgram(seed, join(root, "tests/corpus/1010-json-stringify-space.ts")); + await checkProgram(seed, join(root, "tests/corpus/fs-write-string-bytes-union.ts")); + + const fetchOptions = join(directory, "fetch-options.ts"); + writeFileSync(fetchOptions, 'async function probe() { const response = await fetch("https://example.invalid", { headers: { accept: "application/json" } }); console.log(response.status); } if (process.env["RUN_NATIVE_FETCH_PROBE"] === "1") await probe();\n'); + await checkProgram(seed, fetchOptions); + expect(await invoke(seed, ["coverage", fetchOptions])).toContain("(100%)"); + + const dynamic = join(directory, "dynamic.ts"); + writeFileSync(dynamic, 'const value: any = { answer: 42 }; console.log(`answer:${value.answer}`);\n'); + await checkProgram(seed, dynamic, ["--dynamic"]); + const comptime = join(directory, "comptime.ts"); + writeFileSync(comptime, 'const answer = comptime(() => [1, 2, 3].reduce((sum, value) => sum + value, 0) * 7); console.log(answer);\n'); + expect(await invoke(seed, ["run", comptime, "-o", executable("comptime"), "--strip"])).toBe("42\n"); + + const object = join(directory, "program.obj"); + const linkInfo = JSON.parse(await invoke(seed, ["build", sample, "-o", object, "--print=native-link-info"])); + expect(linkInfo.program.object).toBe(object); + expect(readFileSync(object).length).toBeGreaterThan(0); + + const profile = join(root, "tests/library-mode/contract-attest/profile.json"); + const archive = join(directory, "contract.a"); + const expectedArchive = join(directory, "contract-node.a"); + await exec(process.execPath, [join(root, "packages/cli/dist/main.js"), "build", "--lib", "--profile", profile, "-o", expectedArchive], options); + await invoke(seed, ["build", "--lib", "--profile", profile, "-o", archive]); + expect(JSON.parse(readFileSync(archive + ".contract.json", "utf8"))) + .toEqual(JSON.parse(readFileSync(expectedArchive + ".contract.json", "utf8"))); + + // The unmodified package exercises deep validator traversals. Compile it + // through the installed CLI, including its async command boundary. + const threeProfile = join(root, "tests/library-mode/wasm/three.json"); + await invoke(seed, ["build", "--lib", "--profile", threeProfile, "-o", join(directory, "three.a")]); + if (process.platform !== "win32" && spawnSync("zig", ["version"]).status === 0 && existsSync(join(root, "packages/runtime-wasm32-wasi/runtime-pack.json"))) { + const linker = join(directory, "zigcc"); + writeFileSync(linker, `#!/bin/sh\nexec '${absoluteCommand("zig").replaceAll("'", "'\\''")}' cc "$@"\n`, { mode: 0o755 }); + const output = join(directory, "three.wasm"); + const built = await exec(seed, ["build", "--lib", "--profile", threeProfile, "-o", output], { + ...nativeOptions, env: { ...nativeOptions.env, SCRIPTC_TARGET: "wasm32-wasi", SCRIPTC_LINKER: linker, + SCRIPTC_RUNTIME_PACK: join(root, "packages/runtime-wasm32-wasi"), SCRIPTC_NO_CACHE: "1" }, + }); + expect(built.stdout.trim()).toBe(output); + expect(comparableStderr(built.stderr)).toBe(""); + const vertices: number[][] = []; + const wasi = new WASI({ version: "preview1" }); + const instance = new WebAssembly.Instance(new WebAssembly.Module(readFileSync(output)), { + ...wasi.getImportObject(), scriptc: { + vertex: (...args: number[]) => vertices.push(args.map((value) => Number(value.toFixed(9)))), + panic: () => { throw new Error("unexpected Wasm panic"); }, + }, + }); + wasi.initialize(instance); + const api = instance.exports as Record number>; + api.app_init!(); + for (const time of [0, 123, 2000]) expect(api.app_frame!(time, 1.5)).toBe(24); + const oracle = await exec(process.execPath, ["--input-type=module", "--eval", ` + const vertices = []; + globalThis.vertex = (...args) => vertices.push(args.map(value => Number(value.toFixed(9)))); + const { frame } = await import(${JSON.stringify(join(root, "tests/library-mode/wasm/three.mjs"))}); + for (const time of [0, 123, 2000]) frame(time, 1.5); + console.log(JSON.stringify(vertices)); + `], options); + expect(vertices).toEqual(JSON.parse(oracle.stdout)); + expect(oracle.stderr).toBe(""); + api.app_collect!(); + } const badSource = join(directory, "bad.ts"); writeFileSync(badSource, 'const value: number = "wrong"; console.log(value);\n'); const retained = executable("retained"); writeFileSync(retained, "existing output"); - const failed = spawnSync(seed, [badSource, "-o", retained, "--toolchain", manifestPath], nativeOptions); + const failed = spawnSync(seed, ["build", badSource, "-o", retained], nativeOptions); expect(failed.status).toBe(1); expect(failed.stderr.toString()).toContain("not assignable"); expect(readFileSync(retained, "utf8")).toBe("existing output"); - const entry = join(root, "packages/compiler/src/native/main.ts"); - const profile = join(directory, "ts7-process.ffi.json"); + const entry = join(root, "packages/compiler/src/native/cli.ts"); + const ffi = join(directory, ".scriptc/distribution-seed/compiler.ffi.json"); const rebuilt = executable("scriptc-rebuilt"); // Optimize the compiler that will process the full graph again. Small // programs above and below still exercise development output. - const self = await bootstrapStep("native compiler rebuilds itself", () => - invoke(seed, [entry, "-o", rebuilt, "--backend=llvm", "--strip", "--keep-llvm", "--ffi", profile])); - expect(self.stats.statementsTotal).toBeGreaterThan(10_000); - expect(self.llvmPath).toBe(rebuilt + ".ll"); - console.log("native self-build", self.stats); - await checkProgram(rebuilt, join(root, "tests/corpus/nullish-long-chain.ts"), "llvm"); - await checkProgram(rebuilt, sample, "llvm"); - await checkProgram(rebuilt, unionSample, "llvm"); - await checkProgram(rebuilt, receiverSample, "llvm"); + const self = await bootstrapStep("production CLI rebuilds itself", () => + invoke(seed, ["build", entry, "-o", rebuilt, "--strip", "--keep-llvm", "--ffi", ffi])); + expect(self.trim()).toBe(rebuilt); + await checkProgram(rebuilt, join(root, "tests/corpus/nullish-long-chain.ts")); + await checkProgram(rebuilt, sample); + await checkProgram(rebuilt, unionSample); + await checkProgram(rebuilt, receiverSample); + await checkProgram(rebuilt, join(root, "tests/corpus/1010-json-stringify-space.ts")); // Compare the LLVM used to build the second generation with its own // output. Retaining the build input avoids repeating the seed's work. - if (!self.llvmPath) throw new Error("native self-build did not retain its LLVM"); - const seedLlvm = self.llvmPath; + const seedLlvm = join(directory, "cli.ll"); const rebuiltLlvm = join(directory, "rebuilt.ll"); await bootstrapStep("rebuilt compiler emits itself", () => - invoke(rebuilt, [entry, "--emit=llvm", "-o", rebuiltLlvm, "--ffi", profile])); - expect(readFileSync(seedLlvm).equals(readFileSync(rebuiltLlvm)), "native compiler generations must emit identical LLVM").toBe(true); + invoke(rebuilt, ["build", entry, "--emit=llvm", "-o", rebuiltLlvm, "--ffi", ffi])); + // Executable output adds a runtime ABI check; textual LLVM emission + // omits it. Compare all other output without changing either mode. + const withoutAbiCheck = (path: string): string => readFileSync(path, "utf8") + .replace(/^declare void @scr_runtime_abi_v\d+\(\)\n/m, "") + .replace(/^ call void @scr_runtime_abi_v\d+\(\)\n/m, ""); + expect(withoutAbiCheck(seedLlvm) === withoutAbiCheck(rebuiltLlvm), "native compiler generations must emit identical LLVM").toBe(true); } finally { rmSync(directory, { recursive: true, force: true }); } }, 5_400_000); diff --git a/tests/harness/self-hosting-native-toolchain.test.ts b/tests/harness/self-hosting-native-toolchain.test.ts index e73ba2c1..ea6032b5 100644 --- a/tests/harness/self-hosting-native-toolchain.test.ts +++ b/tests/harness/self-hosting-native-toolchain.test.ts @@ -63,8 +63,8 @@ for (const backend of ["llvm"] as const) { const toolchain = join(directory, "toolchain.json"); const config: NativeToolchainManifest = { schema: "scriptc.native-toolchain.v1", compiler_version: manifest.version, target: target.name, - ts7: "ts7", llvm_package: "llvm", runtime_pack: "pack", runtime_headers: "headers", - c_compiler: "cc", c_compiler_args: [], linker: "ld", linker_args: [], dsymutil: "dsymutil", + ts7: "ts7", llvm_package: "llvm", runtime_pack: "pack", + linker: "ld", linker_args: [], dsymutil: "dsymutil", }; writeFileSync(toolchain, JSON.stringify(config)); const input = join(directory, "request.json"); diff --git a/tests/harness/self-hosting-serialization.test.ts b/tests/harness/self-hosting-serialization.test.ts index e5716118..257a8c08 100644 --- a/tests/harness/self-hosting-serialization.test.ts +++ b/tests/harness/self-hosting-serialization.test.ts @@ -114,6 +114,7 @@ for (const backend of ["llvm"] as const) { "tests/corpus/3089-array-find-narrowing.ts", "tests/corpus/3091-json-recursive-discriminants.ts", "tests/corpus/3094-json-replacer-traversal.ts", + "tests/corpus/1010-json-stringify-space.ts", ]) { const path = join(dir, "emitted.json"); const emitted = await compile(join(root, source), { outDir: dir, outPath: path, outputKind: "ir", dynamic: false }); diff --git a/tests/harness/wasm-library.test.ts b/tests/harness/wasm-library.test.ts index eb672605..18eb652a 100644 --- a/tests/harness/wasm-library.test.ts +++ b/tests/harness/wasm-library.test.ts @@ -2,21 +2,49 @@ import { execFileSync, spawnSync } from "node:child_process"; import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { WASI } from "node:wasi"; -import { afterAll, beforeAll, describe, expect, test } from "vitest"; -import { compileLibrary } from "@scriptc/compiler"; +import { afterAll, beforeAll, describe, expect, test, vi } from "vitest"; +import { compileLibrary as compileNodeLibrary, type CompileLibraryOptions } from "@scriptc/compiler"; +import { NativeCompiler } from "../../packages/compiler/src/native/compiler.js"; +import { nativeCodegenTarget, WASM32_WASI_TARGET } from "../../packages/compiler/src/backend/targets.js"; +import { ts7Executable } from "../../packages/compiler/src/frontend/ts7/rpc-api.js"; + +// Run both compiler drivers against the same real checker transport in this +// harness. The production bootstrap separately exercises the native transport. +vi.mock("../../packages/compiler/src/frontend/pipeline-native.js", async () => { + const { runFrontend } = await import("../../packages/compiler/src/frontend/pipeline.js"); + const { loadProgram } = await import("../../packages/compiler/src/frontend/program-node.js"); + return { runNativeFrontend: (entry: string, _executable: string, npmStatic?: readonly string[] | "auto" | "lib", + externalTypes?: Readonly>, _evaluate?: unknown, libraryNpmStatic?: readonly string[]) => + runFrontend(entry, loadProgram, npmStatic, externalTypes, libraryNpmStatic) }; +}); const fixture = join(import.meta.dirname, "../library-mode/wasm"); const hasZig = spawnSync("zig", ["version"]).status === 0; type Api = Record number> & { memory: WebAssembly.Memory }; -describe.skipIf(!hasZig)("Wasm library embedding", () => { +describe.skipIf(!hasZig).each(["node", "native"] as const)("Wasm library embedding (%s driver)", (driver) => { let directory: string; let module: WebAssembly.Module; let oldTarget: string | undefined; let profile: any; + let native: NativeCompiler; + const compileLibrary = (options: CompileLibraryOptions) => driver === "node" + ? compileNodeLibrary(options) : Promise.resolve(native.compileLibrary(options)); beforeAll(async () => { oldTarget = process.env["SCRIPTC_TARGET"]; + if (driver === "native") { + const host = nativeCodegenTarget()!; + const root = join(import.meta.dirname, "../.."); + const helperPackageRoot = join(root, "packages", host.helper.packageName.replace("@scriptc/", "")); + const compilerPackage = JSON.parse(await readFile(join(root, "packages/compiler/package.json"), "utf8")); + native = new NativeCompiler({ + compilerVersion: compilerPackage.version, target: WASM32_WASI_TARGET, helper: host.helper, + helperPackageRoot, helperExecutable: join(helperPackageRoot, "bin", process.platform === "win32" ? "scriptc-llvm-codegen.exe" : "scriptc-llvm-codegen"), + runtimePackRoot: join(root, "packages/runtime-wasm32-wasi"), ts7Executable: ts7Executable(), + linker: "zig", linkerArgs: ["cc"], dsymutil: "unused", + }); + } process.env["SCRIPTC_TARGET"] = "wasm32-wasi"; directory = await mkdtemp("/tmp/scriptc-wasm-library-test-"); profile = JSON.parse(await readFile(join(fixture, "profile.json"), "utf8")); diff --git a/vitest.config.ts b/vitest.config.ts index 406b8a35..c11fc556 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -30,6 +30,11 @@ const workers = process.env["SCRIPTC_TEST_WORKERS"]; export default defineConfig({ resolve: { alias: { + "@scriptc/compiler/cli/command": fileURLToPath(new URL("./packages/compiler/src/cli/command.ts", import.meta.url)), + "@scriptc/compiler/cli/host": fileURLToPath(new URL("./packages/compiler/src/cli/host.ts", import.meta.url)), + "@scriptc/compiler/cli/paths": fileURLToPath(new URL("./packages/compiler/src/cli/paths.ts", import.meta.url)), + "@scriptc/compiler/cli/output-options": fileURLToPath(new URL("./packages/compiler/src/cli/output-options.ts", import.meta.url)), + "@scriptc/compiler/cli/usage": fileURLToPath(new URL("./packages/compiler/src/cli/usage.ts", import.meta.url)), // Tests run against compiler source directly — no build step needed. "@scriptc/compiler": fileURLToPath( new URL("./packages/compiler/src/index.ts", import.meta.url),